turi Namespace Reference

SKD. More...

Namespaces
	aggregate
	All the available groupby aggregators aggregators. See gl_sframe::groupby for details.
	groupby_aggregate_impl
	groupby_operators
	integer_pack
	join_impl
	memory_info
	Memory info namespace contains functions used to compute memory usage.
	query_eval
	random
	rolling_aggregate
	sframe_config
	sframe_saving_impl
	sgraph_compute
	v2_block_impl

Classes
struct	all_flexible_type_convertible
struct	all_variant_convertible
class	any
class	auto_close_sarrays
class	bad_alloc
class	bad_cast
class	blob
class	block_cache
class	blocking_queue
	Implements a blocking queue useful for producer/consumer models. More...
class	broadcast_queue
class	buffer_pool
class	buffered_writer
class	cancellable_barrier
class	conditional
struct	conditional_test
class	const_gl_sarray_reference
struct	csv_file_handling_options
struct	csv_line_tokenizer
class	csv_writer
class	dataframe_row_iterator
struct	dataframe_t
class	deferred_rwlock
class	dense_bitset
class	dir_archive
struct	edge_triple
struct	enum_to_type
class	fast_integer_power
class	file_download_cache
class	file_line_count_estimator
struct	first_nested_type
class	fixed_dense_bitset
struct	flex_date_time
class	flex_dict_view
struct	flex_undefined
class	flexible_type
struct	flexible_type_converter
class	flexible_type_parser
struct	flexible_type_parser_impl
struct	function_closure_info
class	function_output_iterator
class	general_ifstream
class	general_ofstream
class	gl_gframe
struct	gl_is_pod
	Tests if T is a POD type. More...
struct	gl_is_pod_or_scaler
class	gl_sarray
class	gl_sarray_range
class	gl_sarray_reference
class	gl_sarray_writer
class	gl_sframe
class	gl_sframe_range
class	gl_sframe_writer
class	gl_sgraph
class	global_startup
class	global_teardown
class	group_aggregate_value
struct	group_index_file_information
struct	GroupKeyHash
struct	has_direct_conversion_to_flexible_type
class	hash_bucket
class	hash_bucket_container
class	hash_value
class	hopscotch_map
class	hopscotch_set
class	hopscotch_table
class	iarchive
	The serialization input archive object which, provided with a reference to an istream, will read from the istream, providing deserialization capabilities. More...
class	iarchive_soft_fail
	When this archive is used to deserialize an object, and the object does not support serialization, failure will only occur at runtime. Otherwise equivalent to turi::iarchive. More...
class	image_type
struct	index_file_information
class	indexed_column_groupby
struct	is_flexible_type_convertible
struct	IS_POD_TYPE
	Inheriting from this type will force the serializer to treat the derived type as a POD type. More...
struct	is_valid_flex_type
struct	is_variant_convertible
class	l2_rescaling
class	lazy_eval_future
struct	lazy_eval_operation_base
class	lazy_eval_operation_dag
struct	list_objects_response
class	lockfree_push_back
struct	lru_cache
class	ml_data
struct	ml_data_entry
struct	ml_data_entry_global_index
struct	ml_data_full_entry
class	ml_data_iterator
class	ml_data_row_reference
class	ml_metadata
class	ml_model_base
class	model_base
class	model_proxy
class	mutable_queue
class	mutex
class	oarchive
	The serialization output archive object which, provided with a reference to an ostream, will write to the ostream, providing serialization capabilities. More...
class	oarchive_soft_fail
	When this archive is used to serialize an object, and the object does not support serialization, failure will only occur at runtime. Otherwise equivalent to turi::oarchive. More...
class	option_manager
struct	padded_integer
	an integer value padded to 64 bytes More...
class	padded_simple_spinlock
class	parallel_sframe_iterator
class	parallel_sframe_iterator_initializer
class	parallel_task_queue
class	process
struct	progress_time
class	queued_rw_lock
struct	rdtsc_time
class	read_caching_device
class	recursive_mutex
class	resizing_array_sink_ref
class	rwlock
class	s3_device
struct	s3url
class	safe_circular_char_buffer
class	sarray
class	sarray_block_iterator
class	sarray_format_reader_common_base
class	sarray_format_reader_v2
class	sarray_group_format_writer
class	sarray_group_format_writer_v2
class	sarray_iterator
class	sarray_reader
class	sarray_reader_buffer
class	sarray_sorted_buffer
class	scoped_finally
struct	second_nested_type
class	semaphore
class	Serializable
	Concept checks if a type T is serializable. More...
class	sframe
class	sframe_function_output_iterator
struct	sframe_index_file_information
class	sframe_iterator
class	sframe_reader
class	sframe_reader_buffer
class	sframe_rows
class	sgraph
class	simple_model
class	simple_spinlock
class	siterable
class	sparse_parallel_2d_array
class	sparse_similarity_lookup
class	spinrwlock
class	standardization_interface
class	statistics_tracker
class	swriter_base
class	symmetric_2d_array
class	table_printer
class	thread
class	thread_group
class	thread_pool
class	timer
	A simple class that can be used for benchmarking/timing up to microsecond resolution. More...
class	toolkit_class_registry
struct	toolkit_class_specification
struct	toolkit_function_invocation
class	toolkit_function_registry
struct	toolkit_function_response_type
struct	toolkit_function_specification
class	topk_indexer
struct	trace_count
struct	type_to_enum
class	unfair_lock
class	union_fstream
class	unity_global
class	unity_global_base
class	unity_sarray
class	unity_sarray_builder
class	unity_sframe
class	unity_sframe_builder
class	unity_sgraph
class	unity_sgraph_base
class	unity_sketch
struct	unsupported_serialize
	Inheritting from this class will prevent the serialization of the derived class. Used for debugging purposes. More...
class	value_container_mapper
struct	variant_converter
struct	variant_converter< gl_sarray, void >
struct	variant_converter< std::function< S(Args...)>, typename std::enable_if< is_variant_convertible< S >::value &&all_variant_convertible< Args... >::value >::type >
struct	variant_converter< std::map< std::string, T >, typename std::enable_if<!is_flexible_type_convertible< T >::value &&is_variant_convertible< T >::value &&!is_variant_member< std::map< std::string, T > >::value >::type >
struct	variant_converter< std::pair< S, T >, typename std::enable_if<!is_flexible_type_convertible< T >::value &&is_variant_convertible< T >::value >::type >
struct	variant_converter< std::shared_ptr< T >, typename std::enable_if< is_model_descendent< T >::value &&!std::is_same< T, model_base >::value &&!is_toolkit_builtin< T >::value >::type >
struct	variant_converter< std::shared_ptr< unity_sarray >, void >
struct	variant_converter< std::shared_ptr< unity_sframe >, void >
struct	variant_converter< std::shared_ptr< unity_sgraph >, void >
struct	variant_converter< std::tuple< Args... >, typename std::enable_if<!all_flexible_type_convertible< Args... >::value &&all_variant_convertible< Args... >::value >::type >
struct	variant_converter< std::unordered_map< std::string, T >, typename std::enable_if<!is_flexible_type_convertible< T >::value &&is_variant_convertible< T >::value >::type >
struct	variant_converter< std::vector< T >, typename std::enable_if<!is_flexible_type_convertible< std::vector< T > >::value &&is_variant_convertible< T >::value &&!is_variant_member< std::vector< T > >::value >::type >
struct	variant_converter< T, typename std::enable_if< is_flexible_type_convertible< T >::value >::type >
struct	variant_converter< T, typename std::enable_if< is_model_descendent< T >::value &&!std::is_same< T, model_base >::value &&!is_toolkit_builtin< T >::value >::type >
struct	variant_converter< T, typename std::enable_if<(is_variant_member< T >::value &&!std::is_same< T, flexible_type >::value)||std::is_same< T, variant_type >::value >::type >
struct	variant_converter< variant_type, void >
class	vector_index_mapper

Typedefs
typedef int64_t	flex_int
typedef double	flex_float
typedef std::string	flex_string
typedef std::vector< double >	flex_vec
typedef std::vector< flexible_type >	flex_list
typedef std::vector< std::pair< flexible_type, flexible_type > >	flex_dict
typedef image_type	flex_image
typedef flexible_type_impl::ndarray< double >	flex_nd_vec
typedef mutex	spinlock
	if spinlock not supported, it is typedef it to a mutex.
typedef conditional	condition_variable
typedef std::vector< flexible_type >	sgraph_vertex_data
typedef std::vector< flexible_type >	sgraph_edge_data
typedef boost::iostreams::stream< charstream_impl::resizing_array_sink< true > >	charstream
typedef std::map< std::string, flexible_type >	options_map_t
typedef std::function< void(edge_triple &)>	lambda_triple_apply_fn
typedef sframe	sframe_type
	The sframe type.
typedef boost::make_recursive_variant< flexible_type, std::shared_ptr< unity_sgraph_base >, dataframe_t, std::shared_ptr< model_base >, std::shared_ptr< unity_sframe_base >, std::shared_ptr< unity_sarray_base >, std::map< std::string, boost::recursive_variant_ >, std::vector< boost::recursive_variant_ >, boost::recursive_wrapper< function_closure_info > >::type	variant_type
template<typename T >
using	is_variant_member = boost::mpl::contains< variant_type::types, T >
template<typename T >
using	is_model_descendent = std::is_convertible< T *, model_base * >

Enumerations
enum	flex_type_enum : char {   flex_type_enum::INTEGER = 0, flex_type_enum::FLOAT = 1, flex_type_enum::STRING = 2, flex_type_enum::VECTOR = 3,   flex_type_enum::LIST = 4, flex_type_enum::DICT = 5, flex_type_enum::DATETIME = 6, flex_type_enum::UNDEFINED = 7,   flex_type_enum::IMAGE = 8, flex_type_enum::ND_VECTOR = 9 }
enum	Format : size_t { Format::JPG = 0, Format::PNG = 1, Format::RAW_ARRAY = 2, Format::UNDEFINED = 3 }

Functions
std::shared_ptr< turi::unity_server_initializer >	capi_server_initializer ()
bool	flex_type_is_convertible (flex_type_enum from, flex_type_enum to)
bool	flex_type_has_binary_op (flex_type_enum left, flex_type_enum right, char op)
const char *	flex_type_enum_to_name (flex_type_enum en)
flex_type_enum	flex_type_enum_from_name (const std::string &name)
flex_type_enum	get_common_type (const std::set< flex_type_enum > &types)
JSONNode	flexible_type_to_json (const flexible_type &val, std::string name)
void	unescape_string (std::string &cal, bool use_escape_char, char escape_char, char quote_char, bool double_quote)
void	unescape_string (std::string &cal, char escape_char, char quote_char, bool double_quote)
size_t	unescape_string (char *cal, size_t length, bool use_escape_char, char escape_char, char quote_char, bool double_quote)
size_t	unescape_string (char *cal, size_t length, char escape_char, char quote_char, bool double_quote)
void	escape_string (const std::string &val, char escape_char, bool use_escape_char, char quote_char, bool use_quote_char, bool double_quote, std::string &output, size_t &output_len)
void	decode_image_inplace (image_type &image)
void	encode_image_inplace (image_type &image)
void	write_image (const std::string &filename, char *data, size_t width, size_t height, size_t channels, Format format)
image_type	read_image (const std::string &url, const std::string &format_hint)
void	read_raw_image (const std::string &url, char **data, size_t &length, size_t &width, size_t &height, size_t &channels, Format &format, const std::string &format_hint)
void	parse_jpeg (const char *data, size_t length, size_t &width, size_t &height, size_t &channels)
void	parse_png (const char *data, size_t length, size_t &width, size_t &height, size_t &channels)
void	boost_parse_image (std::string filename, size_t &width, size_t &height, size_t &channels, Format &format, size_t &image_data_size, std::string format_string)
void	boost_read_image (std::string filename, char **out_data, size_t &width, size_t &height, size_t &channels, Format &format, size_t &image_data_size, std::string format_string)
std::ostream &	operator<< (std::ostream &out, const gl_sarray &other)
flex_type_enum	infer_type_of_list (const std::vector< flexible_type > &vec)
std::ostream &	operator<< (std::ostream &out, const gl_sframe &other)
void	begin_log_rotation (std::string log_file_name, size_t log_interval, size_t truncate_limit)
void	stop_log_rotation ()
template<typename T >
bool	atomic_compare_and_swap (T &a, T oldval, T newval)
template<typename T >
bool	atomic_compare_and_swap (volatile T &a, T oldval, T newval)
template<typename T >
T	atomic_compare_and_swap_val (T &a, T oldval, T newval)
template<typename T >
T	atomic_compare_and_swap_val (volatile T &a, T oldval, T newval)
template<>
bool	atomic_compare_and_swap (volatile double &a, double oldval, double newval)
template<>
bool	atomic_compare_and_swap (volatile float &a, float oldval, float newval)
template<typename T >
void	atomic_exchange (T &a, T &b)
	Atomically exchanges the values of a and b. More...
template<typename T >
void	atomic_exchange (volatile T &a, T &b)
	Atomically exchanges the values of a and b. More...
template<typename T >
T	fetch_and_store (T &a, const T &newval)
	Atomically sets a to the newval, returning the old value.
template<typename T >
T	atomic_set_max (T &max_value, T new_value)
template<typename T >
T	atomic_set_max (volatile T &max_value, T new_value)
template<typename T >
T	atomic_set_min (T &min_value, T new_value)
template<typename T >
T	atomic_set_min (volatile T &min_value, T new_value)
template<typename T , typename U = int>
static T	atomic_increment (T &value, const U &increment=1, typename std::enable_if< std::is_integral< T >::value &&std::is_integral< U >::value >::type *=0)
template<typename T , typename U = int>
static T	atomic_increment (volatile T &value, const U &increment=1, typename std::enable_if< std::is_integral< T >::value &&std::is_integral< U >::value >::type *=0)
std::exception_ptr	execute_task_in_native_thread (const std::function< void(void)> &fn)
template<typename F , typename ... Args>
std::result_of< F(Args...)>::type	run_as_native (F f, Args... args)
void	in_parallel (const std::function< void(size_t thread_id, size_t num_threads)> &fn)
thread_pool &	get_parfor_thread_pool ()
template<typename FunctionType >
void	parallel_for (size_t begin, size_t end, const FunctionType &fn)
template<typename FunctionType , typename ReduceType >
ReduceType	fold_reduce (size_t begin, size_t end, const FunctionType &fn, ReduceType base=ReduceType())
template<typename RandomAccessIterator , typename FunctionType >
void	parallel_for (RandomAccessIterator iter_begin, RandomAccessIterator iter_end, const FunctionType &fn, std::random_access_iterator_tag=typename std::iterator_traits< RandomAccessIterator >::iterator_category())
thread	launch_in_new_thread (const boost::function< void(void)> &f, size_t cpuid=size_t(-1))
	Runs f in a new thread. convenience function for creating a new thread quickly.
int	download_url (std::string url, std::string output_file)
std::tuple< int, bool, std::string >	download_url (std::string url)
std::string	get_curl_error_string (int status)
Aws::S3::S3Client	init_aws_sdk_with_turi_env (s3url &parsed_url)
std::string	get_s3_file_last_modified (const std::string &url)
list_objects_response	list_objects (std::string s3_url, std::string proxy="")
list_objects_response	list_directory (std::string s3_url, std::string proxy="")
std::string	delete_object (std::string s3_url, std::string proxy="")
std::string	delete_prefix (std::string s3_url, std::string proxy="")
std::string	sanitize_s3_url (const std::string &url)
bool	parse_s3url (const std::string &url, s3url &ret, std::string &err_msg)
void	set_upload_timeout (long timeout)
void	set_download_timeout (long timeout)
std::string	sanitize_url (std::string url)
std::string	get_system_user_name ()
std::string	get_temp_name (const std::string &prefix="", bool _prefer_hdfs=false)
std::string	get_temp_name_prefer_hdfs (const std::string &prefix="")
bool	delete_temp_file (std::string s)
void	delete_temp_files (std::vector< std::string > files)
void	reap_unused_temp_files ()
void	reap_current_process_temp_files ()
std::vector< std::string >	get_temp_directories ()
size_t	num_temp_directories ()
template<typename OutArcType , typename RandomAccessIterator >
void	serialize_iterator (OutArcType &oarc, RandomAccessIterator begin, RandomAccessIterator end)
	Serializes the contents between the iterators begin and end. More...
template<typename OutArcType , typename InputIterator >
void	serialize_iterator (OutArcType &oarc, InputIterator begin, InputIterator end, size_t vsize)
	Serializes the contents between the iterators begin and end. More...
template<typename InArcType , typename T , typename OutputIterator >
void	deserialize_iterator (InArcType &iarc, OutputIterator result)
	The accompanying function to serialize_iterator() Reads elements from the stream and writes it to the output iterator. More...
template<typename T >
std::string	serialize_to_string (const T &t)
	Serializes a object to a string. More...
template<typename T >
void	deserialize_from_string (const std::string &s, T &t)
	Deserializes a object from a string. More...
template<typename S , typename T , typename TransformFn , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	transform (S &&input, T &&output, TransformFn transformfn, std::set< size_t > constraint_segments=std::set< size_t >())
template<typename S , typename T , typename FilterFn , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	copy_if (S &&input, T &&output, FilterFn filterfn, std::set< size_t > constraint_segments=std::set< size_t >(), size_t random_seed=size_t(-1))
template<typename S , typename T , typename FilterFn , typename TransformFn , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	copy_transform_if (S &&input, T &&output, FilterFn filterfn, TransformFn transformfn, std::set< size_t > constraint_segments=std::set< size_t >(), size_t random_seed=size_t(-1))
template<typename S , typename T , typename FilterFn , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	split (S &&input, T &&output1, T &&output2, FilterFn filterfn, size_t random_seed=std::time(NULL))
template<typename Iterator , typename SWriter , typename = typename std::enable_if<sframe_impl::is_sarray_like<SWriter>::value>::type>
void	copy (Iterator begin, Iterator end, SWriter &&writer)
template<typename S , typename Iterator , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type>
void	copy (S &&array, Iterator output, size_t limit=(size_t)(-1))
template<typename ResultType , typename S , typename FunctionType , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type>
std::vector< ResultType >	reduce (S &&input, FunctionType f, ResultType init=ResultType())
template<typename S1 , typename S2 , typename T , typename TransformFn , typename = typename std::enable_if<sframe_impl::is_sarray_like<S1>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<S2>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	binary_transform (S1 &&input1, S2 &&input2, T &&output, TransformFn transformfn)
template<typename S , typename T , typename = typename std::enable_if<sframe_impl::is_sarray_like<S>::value>::type, typename = typename std::enable_if<sframe_impl::is_sarray_like<T>::value>::type>
void	copy_range (S &&input, T &&output, size_t start, size_t step, size_t end)
void	parallel_dataframe_iterate (const dataframe_t &df, std::function< void(dataframe_row_iterator &iter, size_t startrow, size_t endrow)> partialrowfn)
std::shared_ptr< group_aggregate_value >	get_builtin_group_aggregator (const std::string &)
sframe	group (sframe sframe_in, std::string key_column)
sframe	groupby_aggregate (const sframe &source, const std::vector< std::string > &keys, const std::vector< std::string > &group_output_columns, const std::vector< std::pair< std::vector< std::string >, std::shared_ptr< group_aggregate_value >>> &groups, size_t max_buffer_size=SFRAME_GROUPBY_BUFFER_NUM_ROWS)
std::istream &	eol_safe_getline (std::istream &is, std::string &t)
std::map< std::string, std::shared_ptr< sarray< flexible_type > > >	parse_csvs_to_sframe (const std::string &url, csv_line_tokenizer &tokenizer, csv_file_handling_options options, sframe &frame, std::string frame_sidx_file="")
index_file_information	read_index_file (std::string index_file)
group_index_file_information	read_array_group_index_file (std::string group_index_file)
void	write_array_group_index_file (std::string group_index_file, const group_index_file_information &info)
std::pair< std::string, size_t >	parse_v2_segment_filename (std::string fname)
template<typename T >
sarray_block_iterator< T >	make_sarray_block_iterator (const std::shared_ptr< sarray< T > > &data)
bool	sframe_fast_compact (const sframe &sf)
void	sframe_compact (sframe &sf, size_t segment_threshold)
template<typename T >
bool	sarray_fast_compact (sarray< T > &column)
template<typename T >
void	sarray_compact (sarray< T > &column, size_t segment_threshold)
sframe_index_file_information	read_sframe_index_file (std::string index_file)
void	write_sframe_index_file (std::string index_file, const sframe_index_file_information &info)
size_t	sframe_row_to_csv (const std::vector< flexible_type > &row, char *buf, size_t buflen)
void	sframe_row_to_json (const std::vector< std::string > &column_names, const std::vector< flexible_type > &column_values, JSONNode &node)
void	sframe_save_naive (const sframe &sf, std::string index_file)
void	sframe_save_blockwise (const sframe &sf, std::string index_file)
void	sframe_save (const sframe &sf, std::string index_file)
void	sframe_save_weak_reference (const sframe &sf, std::string index_file)
std::vector< sframe >	shuffle (sframe sframe_in, size_t n, std::function< size_t(const std::vector< flexible_type > &)> hash_fn, std::function< void(const std::vector< flexible_type > &, size_t)> emit_call_back=std::function< void(const std::vector< flexible_type > &, size_t)>())
std::pair< size_t, size_t >	hilbert_index_to_coordinate (size_t s, size_t n)
size_t	coordinate_to_hilbert_index (std::pair< size_t, size_t > coord, size_t n)
void	save_sgraph_to_json (const sgraph &g, std::string targetfile)
void	save_sgraph_to_csv (const sgraph &g, std::string targetdir)
uint32_t	get_local_ip (bool print=true)
std::string	get_local_ip_as_str (bool print=true)
std::pair< size_t, int >	get_free_tcp_port ()
size_t	get_parent_pid ()
size_t	get_my_pid ()
void	wait_for_parent_exit (size_t parent_pid)
bool	is_process_running (size_t pid)
boost::optional< std::string >	getenv_str (const char *variable_name)
rdtsc_type	estimate_ticks_per_second ()
rdtsc_type	rdtsc (void)
void EXPORT	configure_global_environment (std::string argv0)
template<typename T >
static bool	is_power_of_2 (const T &x, _ENABLE_IF_UINT(T))
template<typename T >
static bool	bit_on (T x, unsigned int bit, _ENABLE_IF_UINT(T))
template<typename T >
static bool	bit_off (T x, unsigned int bit, _ENABLE_IF_UINT(T))
template<typename T >
static void	set_bit_off (T &x, unsigned int bit, _ENABLE_IF_UINT(T))
template<typename T >
static void	set_bit_on (T &x, unsigned int bit, _ENABLE_IF_UINT(T))
template<typename T >
static void	flip_bit (T &x, unsigned int bit, _ENABLE_IF_UINT(T))
template<typename T >
static T	bit_mask (unsigned int n_bits, _ENABLE_IF_UINT(T))
template<typename T >
static T	bit_mask (size_t index_begin, unsigned int index_end, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	num_bits_on (T v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	n_trailing_zeros (T v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	n_trailing_ones (const T &v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	index_first_on_bit (const T &v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	n_leading_zeros (T v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	n_leading_ones (T v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	index_last_on_bit (const T &v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	bitwise_log2_ceil (const T &v, _ENABLE_IF_UINT(T))
template<typename T >
static unsigned int	bitwise_log2_floor (const T &v, _ENABLE_IF_UINT(T))
template<typename T >
static T	bitwise_pow2_mod (const T &v, unsigned pow2_idx, _ENABLE_IF_UINT(T))
template<typename T >
static bool	first_n_bits_on (const T &v, unsigned int top_bit, _ENABLE_IF_UINT(T))
static uint128_t	hash128 (const char *s, size_t len)
static uint128_t	hash128 (const std::string &s)
static uint128_t	hash128 (uint128_t v)
template<typename T >
static uint128_t	hash128 (const T &v, typename std::enable_if< std::is_integral< T >::value &&sizeof(T)<=8 >::type *=0)
static uint128_t	hash128 (uint64_t v)
uint128_t	hash128 (const flexible_type &v)
uint128_t	hash128 (const std::vector< flexible_type > &v)
static uint64_t	hash64 (const char *s, size_t len)
static uint64_t	hash64 (const std::string &s)
static uint64_t	hash64 (uint64_t v1, uint64_t v2)
static uint64_t	hash64 (uint64_t v1, uint64_t v2, uint64_t v3)
static uint64_t	hash64 (uint128_t v)
template<typename T >
static uint64_t	hash64 (const T &v, typename std::enable_if< std::is_integral< T >::value &&sizeof(T)<=8 >::type *=0)
uint64_t	hash64 (const flexible_type &v)
uint64_t	hash64 (const std::vector< flexible_type > &v)
static uint128_t	hash128_combine (uint128_t h1, uint128_t h2)
template<typename T >
static uint128_t	hash128_update (uint128_t h, const T &v)
static uint128_t	hash128 (const std::vector< std::string > &v)
static uint64_t	hash64_combine (uint64_t h1, uint64_t h2)
uint64_t	hash64_proportion_cutoff (double proportion)
template<typename T >
static uint64_t	hash64_update (uint64_t h1, const T &t)
static uint64_t	hash64 (const std::vector< std::string > &v)
static uint32_t	simple_random_mapping (size_t index, size_t seed)
static uint64_t	index_hash (uint64_t idx)
static uint64_t	reverse_index_hash (uint64_t idx)
template<typename T , typename LessThan >
void	extract_and_sort_top_k (std::vector< T > &v, size_t top_k, LessThan less_than)
template<typename T >
void	extract_and_sort_top_k (std::vector< T > &v, size_t top_k)
static GL_HOT_INLINE_FLATTEN double	sigmoid (double x)
static GL_HOT_INLINE_FLATTEN double	log1pe (double x)
static GL_HOT_INLINE_FLATTEN double	log1pen (double x)
static GL_HOT_INLINE_FLATTEN double	log1me (double x)
static GL_HOT_INLINE_FLATTEN double	log1men (double x)
static GL_HOT_INLINE_FLATTEN double	logem1 (double x)
static GL_HOT_INLINE_FLATTEN double	log1pe_deriviative (double x)
static GL_HOT_INLINE_FLATTEN double	log1pen_deriviative (double x)
std::string	md5 (std::string val)
std::string	md5_raw (std::string val)
template<typename T >
std::set< T >	set_union (const std::set< T > &a, const std::set< T > &b)
template<typename T >
std::set< T >	set_union (const std::set< T > &a, const T &b)
template<typename T >
std::set< T >	set_intersect (const std::set< T > &a, const std::set< T > &b)
template<typename T >
std::set< T >	set_difference (const std::set< T > &a, const std::set< T > &b)
template<typename T >
std::set< T >	set_difference (const std::set< T > &a, const T &b)
template<typename T >
std::pair< std::set< T >, std::set< T > >	set_partition (const std::set< T > &s, const std::set< T > &partition)
template<typename T >
bool	set_disjoint (const std::set< T > &a, const std::set< T > &b)
template<typename T >
bool	set_equal (const std::set< T > &a, const std::set< T > &b)
template<typename T >
bool	includes (const std::set< T > &a, const std::set< T > &b)
template<typename T >
bool	is_subset (const std::set< T > &a, const std::set< T > &b)
template<typename T >
bool	is_superset (const std::set< T > &a, const std::set< T > &b)
template<typename T >
boost::enable_if_c< boost::is_output_streamable< T >::value, std::ostream & >::type	operator<< (std::ostream &out, const std::set< T > &s)
template<typename T >
boost::enable_if_c< boost::is_output_streamable< T >::value, std::ostream & >::type	operator<< (std::ostream &out, const std::vector< T > &v)
template<typename Key , typename T >
const T &	safe_get (const std::map< Key, T > &map, const Key &key)
template<typename Key , typename T >
const T	safe_get (const std::map< Key, T > &map, const Key &key, const T default_value)
template<typename OldKey , typename NewKey , typename T >
std::map< NewKey, T >	rekey (const std::map< OldKey, T > &map, const std::map< OldKey, NewKey > &key_map)
template<typename Key , typename OldT , typename NewT >
std::map< Key, NewT >	remap (const std::map< Key, OldT > &map, const std::map< OldT, NewT > &val_map)
template<typename Key , typename T >
void	remap (std::map< Key, T > &map, const std::map< T, T > &val_map)
template<typename Key , typename T >
std::map< Key, T >	map_union (const std::map< Key, T > &a, const std::map< Key, T > &b)
template<typename Key , typename T >
std::map< Key, T >	map_intersect (const std::map< Key, T > &a, const std::map< Key, T > &b)
template<typename Key , typename T >
std::map< Key, T >	map_intersect (const std::map< Key, T > &m, const std::set< Key > &keys)
template<typename Key , typename T >
std::map< Key, T >	map_difference (const std::map< Key, T > &a, const std::map< Key, T > &b)
template<typename Key , typename T >
std::set< Key >	keys (const std::map< Key, T > &map)
template<typename Key , typename T >
std::vector< Key >	keys_as_vector (const std::map< Key, T > &map)
template<typename Key , typename T >
std::set< T >	values (const std::map< Key, T > &map)
template<typename Key , typename T >
std::vector< T >	values (const std::map< Key, T > &m, const std::set< Key > &keys)
template<typename Key , typename T >
std::vector< T >	values (const std::map< Key, T > &m, const std::vector< Key > &keys)
template<typename Key >
std::map< Key, Key >	make_identity_map (const std::set< Key > &keys)
template<typename Key , typename T >
std::ostream &	operator<< (std::ostream &out, const std::map< Key, T > &m)
	Writes a map to the supplied stream.
std::string	trim (const std::string &str)
template<typename T >
std::string	tostr (const T &t)
template<typename T >
T	fromstr (const std::string &str)
std::string	pad_number (const size_t number, const size_t npad, const char pad_value='0')
std::vector< std::string >	strsplit (const std::string &str, const std::string &splitchars, const bool auto_trim=false)
void	_add_directory_to_deleter (const std::string &name)
std::string	_get_unique_directory (const std::string &file, size_t line)
template<typename T , typename U >
void	_save_and_load_object (T &dest, const U &src, std::string dir)
std::vector< ml_data_entry >	translate_row_to_ml_data_entry (const std::shared_ptr< ml_metadata > &metadata, const std::vector< ml_data_entry_global_index > &row)
std::vector< ml_data_entry >	translate_row_to_ml_data_entry (const std::shared_ptr< ml_metadata > &metadata, const DenseVector &row)
std::vector< ml_data_entry >	translate_row_to_ml_data_entry (const std::shared_ptr< ml_metadata > &metadata, const SparseVector &v)
std::vector< flexible_type >	translate_row_to_original (const std::shared_ptr< ml_metadata > &metadata, const DenseVector &v)
std::vector< flexible_type >	translate_row_to_original (const std::shared_ptr< ml_metadata > &metadata, const SparseVector &v)
std::vector< flexible_type >	translate_row_to_original (const std::shared_ptr< ml_metadata > &metadata, const std::vector< ml_data_entry_global_index > &row)
std::pair< sframe, ml_data >	make_random_sframe_and_ml_data (size_t n_rows, std::string column_types, bool create_target_column=false, bool target_column_categorical=false)
static bool	ml_testing_equals (const flexible_type &v1, const flexible_type &v2)
std::map< std::string, flexible_type >	varmap_to_flexmap (const variant_map_type &map)
std::map< std::string, variant_type >	flexmap_to_varmap (const std::map< std::string, flexible_type > &map)
void	create_unity_global_singleton (toolkit_function_registry *_toolkit_functions, toolkit_class_registry *_classes)
std::shared_ptr< unity_global >	get_unity_global_singleton ()
std::string	get_variant_which_name (int i)
template<typename T >
static T &	variant_get_ref (variant_type &v)
template<typename T >
static const T &	variant_get_ref (const variant_type &v)
template<typename T >
void	variant_set_value (variant_type &v, const T &f)
template<typename T >
variant_type	to_variant (const T &f)
template<typename T >
std::decay< T >::type	variant_get_value (const variant_type &v)
void	variant_deep_save (const variant_type &v, oarchive &oarc)
template<typename T >
void	variant_deep_save (const T &v, oarchive &oarc)
void	variant_deep_load (variant_type &v, iarchive &iarc)
template<typename T >
void	variant_deep_load (T &v, iarchive &iarc)
void	add_metadata (CoreML::Specification::Model &model_spec, const std::map< std::string, flexible_type > &context)
std::shared_ptr< coreml::MLModelWrapper >	export_linear_regression_as_model_asset (const std::shared_ptr< ml_metadata > &metadata, const Eigen::Matrix< double, Eigen::Dynamic, 1 > &coefs, const std::map< std::string, flexible_type > &context)
std::shared_ptr< coreml::MLModelWrapper >	export_linear_svm_as_model_asset (const std::shared_ptr< ml_metadata > &metadata, const Eigen::Matrix< double, Eigen::Dynamic, 1 > &coefs, const std::map< std::string, flexible_type > &context)
std::shared_ptr< coreml::MLModelWrapper >	export_logistic_model_as_model_asset (const std::shared_ptr< ml_metadata > &metadata, const Eigen::Matrix< double, Eigen::Dynamic, 1 > &coefs, const std::map< std::string, flexible_type > &context)
void EXPORT	export_linear_regression_as_model_asset (const std::string &filename, const std::shared_ptr< ml_metadata > &metadata, const Eigen::Matrix< double, Eigen::Dynamic, 1 > &coefs, const std::map< std::string, flexible_type > &context)
void	setup_pipeline_from_mldata (CoreML::Pipeline &pipeline, std::shared_ptr< ml_metadata > metadata)
std::shared_ptr< coreml::MLModelWrapper >	export_object_detector_model (neural_net::pipeline_spec pipeline, size_t num_classes, size_t num_predictions, flex_list class_labels, float confidence_threshold, float iou_threshold, bool include_non_maximum_suppression, bool use_nms_layer, bool use_most_confident_class)
std::shared_ptr< coreml::MLModelWrapper >	export_activity_classifier_model (const neural_net::model_spec &nn_spec, size_t prediction_window, const flex_list &features, size_t lstm_hidden_layer_size, const flex_list &class_labels, const flex_string &target)
std::shared_ptr< coreml::MLModelWrapper >	export_style_transfer_model (const neural_net::model_spec &nn_spec, size_t image_width, size_t image_height, bool include_flexible_shape, std::string content_feature, std::string style_feature, size_t num_styles)
std::shared_ptr< coreml::MLModelWrapper >	export_drawing_classifier_model (const neural_net::model_spec &nn_spec, const flex_list &features, const flex_list &class_labels, const flex_string &target)
flex_dict	to_flat_dict (const flexible_type &dict_or_list, const flex_string &separator, const flex_string &undefined_string, std::function< flexible_type(const flexible_type &)> nonnumeric_value_handler)
flex_dict	to_flat_dict (const flexible_type &input, const flex_string &separator, const flex_string &undefined_string, const std::string &image_policy="error", const std::string &datetime_policy="error")
gl_sarray	to_sarray_of_flat_dictionaries (gl_sarray input, const flex_string &sep, const flex_string &undefined_string, const std::string &image_policy="error", const std::string &datetime_policy="error")
template<typename T >
GL_HOT_INLINE std::pair< size_t, size_t >	find_slice_boundary_indices (const std::vector< std::pair< size_t, T > > &v, size_t item_index_lb, size_t item_index_ub)
template<typename T , typename SliceInitFunction , typename RowProcessFunction , typename ElementProcessFunction , typename SliceFinalizeFunction >
void	iterate_through_sparse_item_array_by_slice (const std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > > &data, const std::vector< size_t > &slice_delimiters, SliceInitFunction &&init_slice, RowProcessFunction &&preprocess_row, ElementProcessFunction &&process_element, SliceFinalizeFunction &&finalize_slice)
template<typename T , typename RowProcessFunction >
void	iterate_through_sparse_item_array (const std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > > &data, RowProcessFunction &&process_row)
template<typename T >
std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > >	transpose_sparse_sarray (std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > > data, const std::vector< size_t > &item_counts, size_t max_memory_usage)
template<typename InputIterator1 , typename InputIterator2 , typename ComparisonFunction , typename AccumulateFunction >
static void	accumulate_intersection (InputIterator1 first1, const InputIterator1 &last1, InputIterator2 first2, const InputIterator2 &last2, const ComparisonFunction &less_than_operator, const AccumulateFunction &accumulate_matching_pair)
template<typename InputIterator1 , typename InputIterator2 , typename AccumulateFunction >
static void	accumulate_intersection (InputIterator1 first1, const InputIterator1 &last1, InputIterator2 first2, const InputIterator2 &last2, const AccumulateFunction &accumulate_matching_pair)
template<typename InputIterator1 , typename InputIterator2 >
static size_t	count_intersection (InputIterator1 first1, const InputIterator1 &last1, InputIterator2 first2, const InputIterator2 &last2)
template<typename InputIterator1 , typename InputIterator2 , typename ComparisonFunction >
static size_t	count_intersection (InputIterator1 first1, const InputIterator1 &last1, InputIterator2 first2, const InputIterator2 &last2, const ComparisonFunction &less_than_operator)
std::shared_ptr< neural_net::Image >	wrap_image (const image_type &image)
	Converts from the Turi image_type to the neural_net Image type.
std::vector< size_t >	get_unique_values (std::shared_ptr< sarray< flexible_type > > indexed_column)
std::shared_ptr< sarray< flexible_type > >	make_unique (std::shared_ptr< sarray< flexible_type > > indexed_column)
std::pair< sframe, sframe >	split_sframe_on_index (const sframe &src, std::function< bool(size_t)> switch_function)
std::shared_ptr< sarray< flexible_type > >	matrix_to_sarray (const Eigen::MatrixXd &m)
template<typename GenFunction >
sframe	sframe_from_ranged_generator (const std::vector< std::string > &column_names, const std::vector< flex_type_enum > &column_types, size_t num_rows, GenFunction &&generator_function)
void	print_training_device (std::vector< std::string > gpu_names)

Variables
static flexible_type	FLEX_UNDEFINED = flex_undefined()
const char *	DIR_ARCHIVE_INI_FILE
const char *	DIR_ARCHIVE_OBJECTS_BIN
size_t	SFRAME_DEFAULT_NUM_SEGMENTS
const size_t	DEFAULT_SARRAY_READER_BUFFER_SIZE
const size_t	SARRAY_FROM_FILE_BATCH_SIZE
const size_t	MIN_SEGMENT_LENGTH
const size_t	SFRAME_WRITER_BUFFER_HARD_LIMIT
size_t	SFRAME_FILE_HANDLE_POOL_SIZE
const size_t	SFRAME_BLOCK_MANAGER_BLOCK_BUFFER_COUNT
const float	COMPRESSION_DISABLE_THRESHOLD
size_t	SFRAME_DEFAULT_BLOCK_SIZE
const size_t	SARRAY_WRITER_INITAL_ELEMENTS_PER_BLOCK
const size_t	SARRAY_WRITER_MIN_ELEMENTS_PER_BLOCK
size_t	SFRAME_WRITER_MAX_BUFFERED_CELLS_PER_BLOCK
size_t	SFRAME_WRITER_MAX_BUFFERED_CELLS
size_t	SFRAME_MAX_BLOCKS_IN_CACHE
size_t	SFRAME_CSV_PARSER_READ_SIZE
size_t	SFRAME_GROUPBY_BUFFER_NUM_ROWS
size_t	SFRAME_SHUFFLE_BUCKET_SIZE
size_t	SFRAME_JOIN_BUFFER_NUM_CELLS
size_t	SFRAME_IO_READ_LOCK
const size_t	SFRAME_IO_LOCK_FILE_SIZE_THRESHOLD
size_t	SFRAME_SORT_PIVOT_ESTIMATION_SAMPLE_SIZE
size_t	SFRAME_SORT_MAX_SEGMENTS
size_t	SFRAME_COMPACTION_THRESHOLD
size_t	FAST_COMPACT_BLOCKS_IN_SMALL_SEGMENT
size_t	SGRAPH_TRIPLE_APPLY_LOCK_ARRAY_SIZE
size_t	SGRAPH_BATCH_TRIPLE_APPLY_LOCK_ARRAY_SIZE
size_t	SGRAPH_TRIPLE_APPLY_EDGE_BATCH_SIZE
size_t	SGRAPH_DEFAULT_NUM_PARTITIONS
size_t	SGRAPH_INGRESS_VID_BUFFER_SIZE
size_t	SGRAPH_HILBERT_CURVE_PARALLEL_FOR_NUM_THREADS
size_t	DEFAULT_NUM_PYLAMBDA_WORKERS
size_t	DEFAULT_NUM_GRAPH_LAMBDA_WORKERS
enum	ml_missing_value_action : int
enum	ml_column_mode : int
static GL_HOT_INLINE_FLATTEN bool	mode_is_categorical (ml_column_mode mode)
static GL_HOT_INLINE_FLATTEN bool	mode_has_fixed_size (ml_column_mode mode)
static GL_HOT_INLINE_FLATTEN bool	mode_is_indexed (ml_column_mode mode)
const char *	column_mode_enum_to_name (ml_column_mode mode)

Detailed Description

SKD.

This file implements a collection of routines that operate and behave uniformly on all supported protocols. (currently, HDFS, S3, local fs)

Provides a generic interface to call cython functions (which can in turn call python functions) from the C++ code and properly handle exceptions.

ml_data layout format

The rows are stored in segments of a vector of entry_value structs, where entry_value encloses a union of an index_value and a double_value. Thus it is 8 bytes. Each vector contains up to row_block_size rows. The data is stored as an sarray<vector<entry_value> >

Raw data layout

Each vector is simply laid out as

| row 1 | row 2 | ... | row n |

where n <= row_block_size.

Each row is laid out according to the "mode" of the columns, determined by column_metadata::mode. mode can be NUMERIC, CATEGORICAL, NUMERIC_VECTOR, CATEGORICAL_VECTOR, and7 DICTIONARY. The last three of these can hold multiple values.

ROW FORMAT (from ml_data.hpp)

ml_data loads data from an existing sframe, indexes it by mapping all categorical values to unique indices in 0, 1,2,...,n, and records statistics about the values. It then puts it into an efficient row-based data storage structure for use in regression and learning algorithms. The row based storage structure is designed for fast iteration through the rows and target.

The rows are stored in segments of a vector of entry_value structs, where entry_value encloses a union of an index_value and a double_value. Thus it is 8 bytes. Each vector contains up to ROW_BLOCK_SIZE rows. The data is stored as an sarray<vector<entry_value> >

Raw data layout

Each vector is simply laid out as

| <row_size> row 1 | <row_size> row 2 | ... | <row_size> row n |

The row size variable is present if CATEGORICAL_VECTOR or DICTIONARY are defined types, as these may induce variable sized rows. Otherwise, the row sizes are constant and determined from the metadata, thus this does not need to be there.

where n <= ROW_BLOCK_SIZE.

Each row is laid out according to the "mode" of the columns, determined by column_metadata::mode. mode can be NUMERIC, CATEGORICAL, NUMERIC_VECTOR, CATEGORICAL_VECTOR, and DICTIONARY. The last three of these can hold multiple values.

Row layout

Each row takes a block of data entries laid out according to mode:

NUMERIC:

... | value | ...

CATEGORICAL:

... | index | ...

NUMERIC_VECTOR:

... | value1 value2 ... | ...

CATEGORICAL_VECTOR:

... | length index1 index2 ... | ...

DICTIONARY:

... | length index1 value1 index2 value2 ... | ...

example: {0 : 2.1, 1 : 8.5} would be ... | 2 0 2.1 1 8.5 | ...

Since the number of columns, and the mode of each column is constant across rows, the values are all laid out sequentially; for example, a row with columns (numeric, numeric_vector, dictionary) would be

... || row_size | value | v1 v2 ... | length k1 v1 k2 v2 ... || ...

Thus a row of [1.0, [2.0, 4.0], {0 : 2.1, 1 : 8.5}] would appear as

... || 8 | 1.0 | 2.0 4.0 | 2 0 2.1 1 8.5 || ... = ... || 8 1.0 2.0 4.0 2 0 2.1 1 8.5 || ...

Similarly, a row with columns (numeric, numeric_vector, categorical) would be

... || value | v1 v2 ... | k1 || ...

• And a row of [1.0, [2.0, 4.0], 8] would appear as

... || 1.0 | 2 2.0 4.0 | 8 || ... = ... || 1.0 2 2.0 4.0 8 || ... Since ROW_BLOCK_SIZE are stored together in a vector<entry_value>, iterating through rows is extremely fast.

target access

If a target column is present, it is stored as the LAST element in exactly the same way as a regular numeric value.

TODO:

• Replace spmat with flex_dict for each document
• Parallelize over documents. Trim zeros at the end of each sample.
• Try map, hopscotch_map for spmat.
• change predict_counts and sample_counts API to be able to handle both training set and validation set? That way we aren't starting from scratch on the validation set.
• combine all the word alias computation into one method
• use Eigen Vector instead of matrices with one row.
• Make sure to use const auto& where appropriate.
• Choose whether to use w, s, t, d, psdw, etc.
• Track MH acceptance ratio
• Consider using row-order Eigen matrices and checking for speedup (at least for CGS word_topic_counts?)

Typedef Documentation

condition_variable

typedef conditional turi::condition_variable

Convenience typedef to be equivalent to the std::condition_variable

Definition at line 1099 of file pthread_tools.hpp.

is_model_descendent

template<typename T >

using turi::is_model_descendent = typedef std::is_convertible<T*, model_base*>

is_model_descendent<T>::value is true if *T is a descendent of model_base, or is model_base itself.

is_model_descendent<flexible_type>::value // false
is_model_descendent<svariant_converter_imple_model*>::value // true
is_model_descendent<svariant_converter_imple_model>::value // false
is_model_descendent<model_base*>::value // true

Definition at line 71 of file variant_converter.hpp.

is_variant_member

template<typename T >

using turi::is_variant_member = typedef boost::mpl::contains<variant_type::types, T>

is_variant_member<T>::value is true if T is a type contained by the variant_type, and is false otherwise.

is_variant_member<unity_sarray_base*>::value // true
is_variant_member<size_t>::value // false
is_variant_member<flexible_type>::value // true

Definition at line 57 of file variant_converter.hpp.

lambda_triple_apply_fn

typedef std::function<void(edge_triple&)> turi::lambda_triple_apply_fn

Type of the triple apply lambda function.

Definition at line 26 of file sgraph_triple_apply_typedefs.hpp.

variant_type

typedef boost::make_recursive_variant< flexible_type, std::shared_ptr<unity_sgraph_base>, dataframe_t, std::shared_ptr<model_base>, std::shared_ptr<unity_sframe_base>, std::shared_ptr<unity_sarray_base>, std::map<std::string, boost::recursive_variant_>, std::vector<boost::recursive_variant_>, boost::recursive_wrapper<function_closure_info> >::type turi::variant_type

A variant object that can be communicated between Python and C++ which contains either a

• flexible_type
• std::shared_ptr<unity_sgraph>
• dataframe_t
• model
• std::shared_ptr<unity_sframe>
• std::shared_ptr<unity_sarray>
• std::map<variant>
• std::vector<variant>

See the boost variant documentation for details.

The variant should not be accessed directly. See to_variant and variant_get_value for powerful ways to extract or store values from a variant.

Definition at line 24 of file variant.hpp.

Enumeration Type Documentation

flex_type_enum

enum turi::flex_type_enum : char

strong

An enumeration over the possible types the flexible type can contain.

Note

Renumber this at your own risk.

See also

flex_type_enum_from_name

flex_type_enum_to_name

Enumerator
INTEGER	Integer type. Stored type is the flex_int
FLOAT	64-bit floating point type. Stored type is the flex_float
STRING	String type. Stored type is the flex_string
VECTOR	Numeric vector type. Stored type is the flex_vec. This is deprecated.
LIST	Recursive List type. Stored type is the flex_list
DICT	Recursive Dictionary type. Stored type is the flex_dict
DATETIME	Date-Time type. Stored type is the flex_date_time
UNDEFINED	Undefined / Missing Value type. Stored type is the flex_undefined
IMAGE	Image type. Stored type is the flex_image
ND_VECTOR	Numeric vector type. Stored type is the flex_nd_vec

Examples:

/build/src/core/storage/sframe_interface/unity_sframe.hpp.

Definition at line 332 of file flexible_type_base_types.hpp.

Format

enum turi::Format : size_t

strong

Possible image formats stored in the image type

Enumerator
JPG	JPEG Compressed.
PNG	PNG Compressed.
RAW_ARRAY	Not Compressed.
UNDEFINED	Unknown.

Definition at line 22 of file image_type.hpp.

Function Documentation

_add_directory_to_deleter()

void turi::_add_directory_to_deleter

(

const std::string &

name

)

The directories we use for our temporary archives should be unique and everything, but we don't want hundreds of these lying around. Thus add them to a list with which we delete when the program exits; this function does that.

_get_unique_directory()

std::string turi::_get_unique_directory	(	const std::string &	file,
		size_t	line
	)

Make a unique directory name.

add_metadata()

void turi::add_metadata	(	CoreML::Specification::Model &	model_spec,
		const std::map< std::string, flexible_type > &	context
	)

Add short description, metadata to the model.

Parameters

[in]	model	CoreML specification model
[in]	context	Dictionary of context passed from python.

atomic_increment() [1/2]

template<typename T , typename U = int>

static T turi::atomic_increment ( T &  value, const U &  increment = 1, typename std::enable_if< std::is_integral< T >::value &&std::is_integral< U >::value >::type *  = 0  )

inlinestatic

Atomically increments the value, and returns the value of the atomic prior to this operation. Equivalent to

old_value = value; value += increment; return value;

Definition at line 292 of file atomic_ops.hpp.

atomic_increment() [2/2]

template<typename T , typename U = int>

static T turi::atomic_increment ( volatile T &  value, const U &  increment = 1, typename std::enable_if< std::is_integral< T >::value &&std::is_integral< U >::value >::type *  = 0  )

inlinestatic

Atomically increments the value, and returns the value of the atomic prior to this operation. Equivalent to

old_value = value; value += increment; return value;

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

Definition at line 307 of file atomic_ops.hpp.

atomic_set_max() [1/2]

template<typename T >

T turi::atomic_set_max	(	T &	max_value,
		T	new_value
	)

Atomically sets the max, returning the value of the atomic prior to setting the max value, or the existing value if nothing changed. Atomic equivalent to:

old_max_value = max_value; max_value = std::max(max_value, new_value); return old_max_value;

Definition at line 176 of file atomic_ops.hpp.

atomic_set_max() [2/2]

template<typename T >

T turi::atomic_set_max	(	volatile T &	max_value,
		T	new_value
	)

Atomically sets the max, returning the value of the atomic prior to this operation, or the existing value if nothing changed. Atomic equivalent to:

old_max_value = max_value; max_value = std::max(max_value, new_value); return old_max_value;

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

Definition at line 207 of file atomic_ops.hpp.

atomic_set_min() [1/2]

template<typename T >

T turi::atomic_set_min	(	T &	min_value,
		T	new_value
	)

Atomically sets the min, returning the value of the atomic prior to this operation. Atomic equivalent to:

old_min_value = min_value; min_value = std::min(min_value, new_value); return old_min_value;

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

Definition at line 236 of file atomic_ops.hpp.

atomic_set_min() [2/2]

template<typename T >

T turi::atomic_set_min	(	volatile T &	min_value,
		T	new_value
	)

Atomically sets the min, returning the value of the atomic prior to this operation. Atomic equivalent to:

old_min_value = min_value; min_value = std::min(min_value, new_value); return old_min_value;

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.

Definition at line 265 of file atomic_ops.hpp.

bit_mask() [1/2]

template<typename T >

static T turi::bit_mask ( unsigned int  n_bits, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns a bitwise mask of the first n_bits customised to type T. For example, bit_bask<uint16_t>(5) & (000101011101011101 b) returns 11101 b.

Template Parameters

T	Type of the mask to be created.

Parameters

n_bits

Index of the bit to flip.

Definition at line 125 of file bitops.hpp.

bit_mask() [2/2]

template<typename T >

static T turi::bit_mask ( size_t  index_begin, unsigned int  index_end, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns a bitwise mask of a segment of bits, [index_begin, index_end).

Template Parameters

T	Type of the mask to be created.

Parameters

index_begin	Start of interval.
index_end	End of interval.

Definition at line 140 of file bitops.hpp.

bit_off()

template<typename T >

static bool turi::bit_off ( T  x, unsigned int  bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns true if a bit is off. Other bits are ignored.

Parameters

x	Unsigned integer to test.
bit	Index of the bit to test.

Definition at line 75 of file bitops.hpp.

bit_on()

template<typename T >

static bool turi::bit_on ( T  x, unsigned int  bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns true if a bit is on. Other bits are ignored.

Parameters

x	Unsigned integer to test.
bit	Index of the bit to test.

Definition at line 63 of file bitops.hpp.

bitwise_log2_ceil()

template<typename T >

static unsigned int turi::bitwise_log2_ceil ( const T &  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns the rounded up version of the bitwise log base two of the number. For example, 00010000 returns 4, and 00010001 returns 5. If v is zero, zero is returned.

Template Parameters

T	Type of v.

Parameters

v	Unsigned integer value.

Definition at line 356 of file bitops.hpp.

bitwise_log2_floor()

template<typename T >

static unsigned int turi::bitwise_log2_floor ( const T &  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns the rounded down version of the bitwise log base two of the number. For example, 00010000 returns 4, 00011111 returns 4, and 00100000 returns 5. If v is zero, zero is returned.

Template Parameters

T	Type of v.

Parameters

v	Unsigned integer value.

Definition at line 369 of file bitops.hpp.

bitwise_pow2_mod()

template<typename T >

static T turi::bitwise_pow2_mod ( const T &  v, unsigned  pow2_idx, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns the modulus of v rounded to the pow2_idx bit. It is the same as v % (2 ** pow2_idx). For example, bitwise_pow2_mod(10, 3) = 10 % 8 = 2.

Template Parameters

T	Type of v.

Parameters

v	Unsigned integer value.
pow2_idx	Power of 2 with which to take the mod.

Definition at line 383 of file bitops.hpp.

boost_parse_image()

void turi::boost_parse_image	(	std::string	filename,
		size_t &	width,
		size_t &	height,
		size_t &	channels,
		Format &	format,
		size_t &	image_data_size,
		std::string	format_string
	)

Parse the image information, set width, height and channels using boost_gil.

boost_read_image()

void turi::boost_read_image	(	std::string	filename,
		char **	out_data,
		size_t &	width,
		size_t &	height,
		size_t &	channels,
		Format &	format,
		size_t &	image_data_size,
		std::string	format_string
	)

Read the content of jpeg image and return the pixel array into out_data Only works with local files.

capi_server_initializer()

std::shared_ptr<turi::unity_server_initializer> turi::capi_server_initializer

(

)

The server initializer function.

This function creates the server initializer for the C-API, which is what determines which models and functions are registered to the unity sever, which then determines what models are packaged in the framework and available from the C-API.

In some cases, it may be desirable to have a custom server initializer, for example if only a subset of the models are needed. In this case, define the macro CAPI_DISABLE_DEFAULT_SERVER_INITIALIZER and create a custom implementation of that function to be compiled in. In addition, the CMakeLists.txt

configure_global_environment()

void EXPORT turi::configure_global_environment

(

std::string

argv0

)

Configures the system global environment. This should be the first thing (or close to the first thing) called on program startup.

create_unity_global_singleton()

void turi::create_unity_global_singleton	(	toolkit_function_registry *	_toolkit_functions,
		toolkit_class_registry *	_classes
	)

Creates the unity_global singleton, passing in the arguments into the unity_global constructor

decode_image_inplace()

void turi::decode_image_inplace

(

image_type &

image

)

Makes an image raw.

encode_image_inplace()

void turi::encode_image_inplace

(

image_type &

image

)

Makes an image png if raw.

escape_string()

void turi::escape_string	(	const std::string &	val,
		char	escape_char,
		bool	use_escape_char,
		char	quote_char,
		bool	use_quote_char,
		bool	double_quote,
		std::string &	output,
		size_t &	output_len
	)

Escapes a string from val into output. The length of the output string is in returned in output_len. Note that output.length() may be greater than the output_len.

Parameters

val	The string to escape
escape_char	The escape character to use (recommended '\')
use_escape_char	If true, escape character is used. Note that if this is false, the resultant string may not always be parseable.
quote_char	The quote character to use. (recommended '"')
use_quote_char	If the output string should be quoted
double_quote	If double quotes are converted to single quotes.

export_activity_classifier_model()

std::shared_ptr<coreml::MLModelWrapper> turi::export_activity_classifier_model	(	const neural_net::model_spec &	nn_spec,
		size_t	prediction_window,
		const flex_list &	features,
		size_t	lstm_hidden_layer_size,
		const flex_list &	class_labels,
		const flex_string &	target
	)

Wraps a trained activity classifier model_spec as a complete MLModel.

export_drawing_classifier_model()

std::shared_ptr<coreml::MLModelWrapper> turi::export_drawing_classifier_model	(	const neural_net::model_spec &	nn_spec,
		const flex_list &	features,
		const flex_list &	class_labels,
		const flex_string &	target
	)

Wraps a trained drawing classifier model_spec as a complete MLModel.

export_linear_regression_as_model_asset() [1/2]

std::shared_ptr<coreml::MLModelWrapper> turi::export_linear_regression_as_model_asset	(	const std::shared_ptr< ml_metadata > &	metadata,
		const Eigen::Matrix< double, Eigen::Dynamic, 1 > &	coefs,
		const std::map< std::string, flexible_type > &	context
	)

Export as model asset

export_linear_regression_as_model_asset() [2/2]

void EXPORT turi::export_linear_regression_as_model_asset	(	const std::string &	filename,
		const std::shared_ptr< ml_metadata > &	metadata,
		const Eigen::Matrix< double, Eigen::Dynamic, 1 > &	coefs,
		const std::map< std::string, flexible_type > &	context
	)

Simplified versions of the above, with exported symbol visibility so that extensions can link to them.

export_linear_svm_as_model_asset()

std::shared_ptr<coreml::MLModelWrapper> turi::export_linear_svm_as_model_asset	(	const std::shared_ptr< ml_metadata > &	metadata,
		const Eigen::Matrix< double, Eigen::Dynamic, 1 > &	coefs,
		const std::map< std::string, flexible_type > &	context
	)

Export linear SVM as model asset.

export_logistic_model_as_model_asset()

std::shared_ptr<coreml::MLModelWrapper> turi::export_logistic_model_as_model_asset	(	const std::shared_ptr< ml_metadata > &	metadata,
		const Eigen::Matrix< double, Eigen::Dynamic, 1 > &	coefs,
		const std::map< std::string, flexible_type > &	context
	)

Export logistic regression as model asset.

export_object_detector_model()

std::shared_ptr<coreml::MLModelWrapper> turi::export_object_detector_model	(	neural_net::pipeline_spec	pipeline,
		size_t	num_classes,
		size_t	num_predictions,
		flex_list	class_labels,
		float	confidence_threshold,
		float	iou_threshold,
		bool	include_non_maximum_suppression,
		bool	use_nms_layer,
		bool	use_most_confident_class
	)

Wraps a trained object detector model_spec as a complete MLModel.

Parameters

nn_spec

The NeuralNetwork to wrap, which must accept an input "image" with shape (3, image_height, image_width) and values in [0,1], output "confidence" with shape (num_predictions, num_classes), and "coordinates" with shape (num_predictions, 4).

export_style_transfer_model()

std::shared_ptr<coreml::MLModelWrapper> turi::export_style_transfer_model	(	const neural_net::model_spec &	nn_spec,
		size_t	image_width,
		size_t	image_height,
		bool	include_flexible_shape,
		std::string	content_feature,
		std::string	style_feature,
		size_t	num_styles
	)

Wraps a trained style transfer model_spec as a complete MLModel.

find_slice_boundary_indices()

template<typename T >

GL_HOT_INLINE std::pair<size_t, size_t> turi::find_slice_boundary_indices	(	const std::vector< std::pair< size_t, T > > &	v,
		size_t	item_index_lb,
		size_t	item_index_ub
	)

Given a sorted sparse vector of (index, value) pairs, efficiently find and return a pair of indices (i1, i2) in the vector such that (item_index_lb <= v[i].first < item_index_ub) for all i = (i1, ..., i2 - 1), and false otherwise.

Definition at line 26 of file utilities.hpp.

first_n_bits_on()

template<typename T >

static bool turi::first_n_bits_on ( const T &  v, unsigned int  top_bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns true if the first n bits of v are on.

Template Parameters

T	Type of v.

Parameters

v	Unsigned integer value.

Definition at line 394 of file bitops.hpp.

flex_type_enum_from_name()

flex_type_enum turi::flex_type_enum_from_name ( const std::string &  name )

inline

Given the printable name of a flexible_type type, returns its corresponding flex_type_enum enumeration value. Reverse of flex_type_enum_to_name.

Definition at line 750 of file flexible_type_base_types.hpp.

flex_type_enum_to_name()

const char* turi::flex_type_enum_to_name ( flex_type_enum  en )

inline

Returns a printable name of a flex_type_enum. Reverse of flex_type_enum_from_name

Definition at line 719 of file flexible_type_base_types.hpp.

flex_type_has_binary_op()

bool turi::flex_type_has_binary_op ( flex_type_enum  left, flex_type_enum  right, char  op  )

inline

Check if numeric operators between these types are supported. if a and b are flexible_types and flex_type_has_binary_op(a.get_type(), b.get_type(), op) is true, this means. that a op= b and a op b is false. op is any of '+','-','*','/','','<','>','='

'<' is the less than operator, and '>' is the greater than operator. '=' is the equality comparison operator. Suceeding on '<' or '>' implies that the less than or equal operator (<=) and the greater than or equal operator (>=) will also succeed

For instance given flexible_type a, b

if (flex_type_has_binary_op(a, b, '+')) {
  a += b // will succeed
  a + b  // will succeed
} else {
  a += b // will fail
  a + b  // will fail
}

Definition at line 393 of file flexible_type_base_types.hpp.

flex_type_is_convertible()

bool turi::flex_type_is_convertible ( flex_type_enum  from, flex_type_enum  to  )

inline

Check if one flexible type is convertable to the other.

Definition at line 353 of file flexible_type_base_types.hpp.

flexible_type_to_json()

JSONNode turi::flexible_type_to_json ( const flexible_type &  val, std::string  name  )

inline

Helper utility for converting from flexible_type to json. TODO: Fill in details

Definition at line 16 of file json_util.hpp.

flexmap_to_varmap()

std::map<std::string, variant_type> turi::flexmap_to_varmap ( const std::map< std::string, flexible_type > &  map )

inline

Cast each flexible type to variant type.

Definition at line 46 of file toolkit_util.hpp.

flip_bit()

template<typename T >

static void turi::flip_bit ( T &  x, unsigned int  bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Flips a bit.

Parameters

x	Reference to unsigned integer to change.
bit	Index of the bit to flip.

Definition at line 111 of file bitops.hpp.

get_common_type()

flex_type_enum turi::get_common_type ( const std::set< flex_type_enum > &  types )

inline

Given a set of types, choose a common type that all types in the set can be converted to and preserves the most data. Not designed to be passed a set with UNDEFINED in it.

Definition at line 826 of file flexible_type_base_types.hpp.

get_curl_error_string()

std::string turi::get_curl_error_string

(

int

status

)

Returns the curl error string for a curl error code returned by download_url.

get_my_pid()

size_t turi::get_my_pid

(

)

Get the PID of my current process.

get_unity_global_singleton()

std::shared_ptr<unity_global> turi::get_unity_global_singleton

(

)

Gets a pointer to the unity global singleton

get_variant_which_name()

std::string turi::get_variant_which_name ( int  i )

inline

Given variant.which() gets the name of the type inside it.

Definition at line 69 of file variant.hpp.

index_first_on_bit()

template<typename T >

static unsigned int turi::index_first_on_bit ( const T &  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Returns the index of the first on bit in v. For example, 010111011000 gives 3. Returns bitsizeof(T) if v is zero.

Template Parameters

T	type of v.

Parameters

v	Unsigned integer.

Definition at line 260 of file bitops.hpp.

index_last_on_bit()

template<typename T >

static unsigned int turi::index_last_on_bit ( const T &  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Index of the last on bit. For example, 01101100 is 6. Returns bitsizeof(T) if v is zero.

Template Parameters

T	Type of v.

Parameters

v	Unsigned integer value.

Definition at line 343 of file bitops.hpp.

infer_type_of_list()

flex_type_enum turi::infer_type_of_list

(

const std::vector< flexible_type > &

vec

)

Utility function to infer the most general type of an in memory vector of flexible_types.

is_power_of_2()

template<typename T >

static bool turi::is_power_of_2 ( const T &  x, _ENABLE_IF_UINT(T)    )

inlinestatic

Tests if x is a power of 2 (i.e. if just one bit is on).

Parameters

x	Unsigned integer to test.

Definition at line 52 of file bitops.hpp.

iterate_through_sparse_item_array()

template<typename T , typename RowProcessFunction >

void turi::iterate_through_sparse_item_array	(	const std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > > &	data,
		RowProcessFunction &&	process_row
	)

Iterates through a sparse vector sarray efficiently, calling a prescribed function on each row and on each element.

Parameters:

data – an sarray of vectors of (index, value) pairs. This vector must be sorted by index.

preprocess_row – the function called on every row. It has the signature:

 preprocess_row(size_t thread_idx, size_t row_idx,
                std::vector<std::pair<size_t, T> >& row)

In this case, row_idx is the row currently being processed and row is the current row. Any modifications to row are discarded after this.

Definition at line 279 of file utilities.hpp.

iterate_through_sparse_item_array_by_slice()

template<typename T , typename SliceInitFunction , typename RowProcessFunction , typename ElementProcessFunction , typename SliceFinalizeFunction >

void turi::iterate_through_sparse_item_array_by_slice	(	const std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > > &	data,
		const std::vector< size_t > &	slice_delimiters,
		SliceInitFunction &&	init_slice,
		RowProcessFunction &&	preprocess_row,
		ElementProcessFunction &&	process_element,
		SliceFinalizeFunction &&	finalize_slice
	)

Iterates through a sparse vector sarray by column slices, with possible optimizations for other functionality. Each row in data is assumed to be a sorted vector of (index, value) pairs, and this function does multiple passes through the data, with each pass handling a contiguous slice of the indices in each row. These slices are determined by the slice_delimiters vector.

Parameters:

data – an sarray of vectors of (index, value) pairs. This vector must be sorted by index.

slice_delimiters – A vector of length (num_slices - 1) that give the boundaries of the slices of indices. For example, [0, 5, 10, 12] would have 3 slices, with (0,5), (5, 10), and (10, 12) being the slices processed.

init_slice – called at the beginning of each slice, before the data is processed. init_slice has the signature

init_slice(size_t slice_idx, size_t item_idx_start, size_t item_idx_end)

where slice_idx is the slice counter (0, 1, ...), and item_idx_start and item_idx_end are taken from slice_boundaries[slice_idx] and slice_boundaries[slice_idx - 1].

preprocess_row – the function called on every row. It has the signature:

 preprocess_row(size_t thread_idx, size_t row_idx,
                size_t slice_item_idx_start, size_t slice_item_idx_end,
                std::vector<std::pair<size_t, T> >& row)

In this case, row_idx is the row currently being processed, slice_item_idx_start and slice_item_idx_end are taken from the current slice, and row is the current row. The row can be modified, but any resulting modifications must preserve the sorted nature. If the row is empty upon return, then processing all future elements is skipped.

process_element – the function called on every element. It has the signature:

 process_element(size_t thread_idx, size_t row_idx,
                 size_t item_idx_start, size_t item_idx_end,
                 size_t item_idx, const T& value)

In this case, row_idx is the row currently being processed, item_idx_start and item_idx_end are taken from the slice, item_idx is the index of the value in the row, and value is the value.

finalize_slice – the function called at the end of every slice. It has the same signature as init_slice:

 finalize_slice(size_t slice_idx, size_t item_idx_start, size_t item_idx_end)

Definition at line 146 of file utilities.hpp.

log1me()

static GL_HOT_INLINE_FLATTEN double turi::log1me ( double  x )

inlinestatic

Numerically stable version of log(1 - exp(x) )

Definition at line 56 of file logit_math.hpp.

log1men()

static GL_HOT_INLINE_FLATTEN double turi::log1men ( double  x )

inlinestatic

Numerically stable version of log(1 - exp(-x) )

Definition at line 64 of file logit_math.hpp.

log1pe()

static GL_HOT_INLINE_FLATTEN double turi::log1pe ( double  x )

inlinestatic

Numerically stable version of log(1 + exp(x) )

Definition at line 42 of file logit_math.hpp.

log1pe_deriviative()

static GL_HOT_INLINE_FLATTEN double turi::log1pe_deriviative ( double  x )

inlinestatic

Numerically stable version of d/dx (log(1 + exp(x) )) = 1 / (1 + exp(-x)) = sigmoid(x).

Definition at line 80 of file logit_math.hpp.

log1pen()

static GL_HOT_INLINE_FLATTEN double turi::log1pen ( double  x )

inlinestatic

Numerically stable version of log(1 + exp(-x) )

Definition at line 49 of file logit_math.hpp.

log1pen_deriviative()

static GL_HOT_INLINE_FLATTEN double turi::log1pen_deriviative ( double  x )

inlinestatic

Numerically stable version of d/dx (log(1 + exp(-x) )) = -1 / (1 + exp(x)).

Definition at line 88 of file logit_math.hpp.

logem1()

static GL_HOT_INLINE_FLATTEN double turi::logem1 ( double  x )

inlinestatic

Numerically stable version of log(exp(x) - 1)

Definition at line 72 of file logit_math.hpp.

make_random_sframe_and_ml_data()

std::pair<sframe, ml_data> turi::make_random_sframe_and_ml_data	(	size_t	n_rows,
		std::string	column_types,
		bool	create_target_column = false,
		bool	target_column_categorical = false
	)

Creates a random SFrame for testing purposes. The column_type_info gives the types of the column.

Parameters

[in]	n_rows	The number of observations to run the timing on.
[in]	column_type_info	A string with each character denoting one type of column. The legend is as follows:

n: numeric column. b: categorical column with 2 categories. z: categorical column with 5 categories. Z: categorical column with 10 categories. c: categorical column with 100 categories. C: categorical column with 1000000 categories. s: categorical column with short string keys and 1000 categories. S: categorical column with short string keys and 100000 categories. v: numeric vector with 10 elements. V: numeric vector with 1000 elements. u: categorical set with up to 10 elements. U: categorical set with up to 1000 elements. d: dictionary with 10 entries. D: dictionary with 100 entries.

Parameters

[in]

create_target_column

If true, then create a random target column as well.

Returns

A pair of sframe, with the raw data, and an ml_data object made from that sframe.

ml_testing_equals()

static bool turi::ml_testing_equals ( const flexible_type &  v1, const flexible_type &  v2  )

inlinestatic

Better equality testing stuff. Handles out-of-order on the categorical_vector, which is assumed by ml_data.

Definition at line 57 of file testing_utils.hpp.

n_leading_ones()

template<typename T >

static unsigned int turi::n_leading_ones ( T  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Counts the number of leading ones in v. For example, 11011000 has 2 leading ones. Returns bitsizeof(T) if v is (~0).

Template Parameters

T	type of v.

Parameters

v	Unsigned integer value.

Definition at line 331 of file bitops.hpp.

n_leading_zeros()

template<typename T >

static unsigned int turi::n_leading_zeros ( T  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Counts the number of leading zeros in v. For example, 01011000 has 1 leading zero. Returns bitsizeof(T) if v is zero.

Template Parameters

T	type of v.

Parameters

v	Unsigned integer value.

Definition at line 287 of file bitops.hpp.

n_trailing_ones()

template<typename T >

static unsigned int turi::n_trailing_ones ( const T &  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Counts the number of trailing ones in v. For example, 010111010111 has 3 trailing ones. Returns bitsizeof(T) if v is (~0).

Template Parameters

T	type of v.

Parameters

v	Unsigned integer.

Definition at line 248 of file bitops.hpp.

n_trailing_zeros()

template<typename T >

static unsigned int turi::n_trailing_zeros ( T  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Counts the number of trailing zeros in v. For example, 010111011000 has 3 trailing zeros. Returns bitsizeof(T) if v is zero.

Template Parameters

T	type of v.

Parameters

v	Unsigned integer.

Definition at line 206 of file bitops.hpp.

num_bits_on()

template<typename T >

static unsigned int turi::num_bits_on ( T  v, _ENABLE_IF_UINT(T)    )

inlinestatic

Counts the number of bits on in x.

Parameters

v	Unsigned integer.

Definition at line 154 of file bitops.hpp.

operator<<() [1/2]

std::ostream& turi::operator<<	(	std::ostream &	out,
		const gl_sarray &	other
	)

Provides printing of the gl_sarray.

operator<<() [2/2]

std::ostream& turi::operator<<	(	std::ostream &	out,
		const gl_sframe &	other
	)

Provides printing of the gl_sframe.

parse_jpeg()

void turi::parse_jpeg	(	const char *	data,
		size_t	length,
		size_t &	width,
		size_t &	height,
		size_t &	channels
	)

Parse the image information, set width, height and channels using libjpeg.

parse_png()

void turi::parse_png	(	const char *	data,
		size_t	length,
		size_t &	width,
		size_t &	height,
		size_t &	channels
	)

Parse the image information, set width, height and channels using libpng.

print_training_device()

void turi::print_training_device

(

std::vector< std::string >

gpu_names

)

This prints the verbose statements based on GPU names it gets from the respective compute contexts.

read_image()

image_type turi::read_image	(	const std::string &	url,
		const std::string &	format_hint
	)

Read the content from url and return an image type object. Throws an exception if failing to decode as format_hint.

read_raw_image()

void turi::read_raw_image	(	const std::string &	url,
		char **	data,
		size_t &	length,
		size_t &	width,
		size_t &	height,
		size_t &	channels,
		Format &	format,
		const std::string &	format_hint
	)

Read raw image data from URL, store data in data and length. Store image information such as width, height, channels, and format.

run_as_native()

template<typename F , typename ... Args>

std::result_of<F(Args...)>::type turi::run_as_native	(	F	f,
		Args...	args
	)

Takes a function call and runs it in a native stack space. Used to get by some libjvm oddities when using coroutines.

Returns an exception if an exception was thrown while executing the inner task.

Definition at line 156 of file execute_task_in_native_thread.hpp.

sarray_compact()

template<typename T >

void turi::sarray_compact	(	sarray< T > &	column,
		size_t	segment_threshold
	)

Inplace compacts an SArray. Fast compact is tried first and if the number of segments do not fall below the target, a slow compaction is performed.

Definition at line 124 of file sframe_compact_impl.hpp.

sarray_fast_compact()

template<typename T >

bool turi::sarray_fast_compact

(

sarray< T > &

column

)

sarray_fast_compact looks for runs of small segments (comprising of less than FAST_COMPACT_BLOCKS_IN_SMALL_SEGMENT block), and rebuilds them into bigger segments. Returns true if any compaction was performed.

Definition at line 49 of file sframe_compact_impl.hpp.

set_bit_off()

template<typename T >

static void turi::set_bit_off ( T &  x, unsigned int  bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Sets a bit to be off.

Parameters

x	Reference to unsigned integer to change.
bit	Index of the bit to set off.

Definition at line 87 of file bitops.hpp.

set_bit_on()

template<typename T >

static void turi::set_bit_on ( T &  x, unsigned int  bit, _ENABLE_IF_UINT(T)    )

inlinestatic

Sets a bit to be on.

Parameters

x	Reference to unsigned integer to change.
bit	Index of the bit to set on.

Definition at line 99 of file bitops.hpp.

setup_pipeline_from_mldata()

void turi::setup_pipeline_from_mldata	(	CoreML::Pipeline &	pipeline,
		std::shared_ptr< ml_metadata >	metadata
	)

Initiates a pipeline from an MLData metadata object so that it takes the input in the form of the input from the mldata, then outputs it as a final vector named vectorized_features that can then be used by other algorithms. The pipeline is returned. The input variables of the pipeline are the same as those given in the ml_data metadata object. The classifier or regressor needs to be added to the pipeline, along with the appropriate output variables.

sframe_compact()

void turi::sframe_compact	(	sframe &	sf,
		size_t	segment_threshold
	)

Inplace compacts an SFrame. Fast compact is tried first and if the number of segments do not fall below the target, a slow compaction is performed.

sframe_fast_compact()

bool turi::sframe_fast_compact

(

const sframe &

sf

)

sframe_fast_compact looks for runs of small segments (comprising of less than FAST_COMPACT_BLOCKS_IN_SMALL_SEGMENT block), and rebuilds them into bigger segments. Returns true if any compaction was performed.

sigmoid()

static GL_HOT_INLINE_FLATTEN double turi::sigmoid ( double  x )

inlinestatic

Numerically stable version of 1 / (1 + exp(-x));

If x < 0, then this is equal to

exp(- abs(x)) / (1 + exp(-abs(x))) = 1 / (1 + exp(x))

If x >=0, then this is equal to 1 / (1 + exp(-x)).

This separation into the positive and negative case is so that the code never uses the result of exp(x) where x is large and positive, which could easily result in overflow.

Definition at line 31 of file logit_math.hpp.

stop_log_rotation()

void turi::stop_log_rotation

(

)

Stops log rotation.

No-op if log rotation was not started.

Not safe for concurrent use.

to_flat_dict() [1/2]

flex_dict turi::to_flat_dict	(	const flexible_type &	dict_or_list,
		const flex_string &	separator,
		const flex_string &	undefined_string,
		std::function< flexible_type(const flexible_type &)>	nonnumeric_value_handler
	)

Flattens list or dictionary types to a non-nested dictionary of (string key : numeric value) pairs. Each nested key is a concatenation of the keys in the separation with sep_char separating them. For example, if sep_char = ".", then

{"a" : {"b" : 1}, "c" : 2}

becomes

{"a.b" : 1, "c" : 2}.

• List and vector elements are handled by converting the index of the appropriate element to a string.
• String values are handled by treating them as a single {"string_value" : 1} pair.
• FLEX_UNDEFINED values are handled by replacing them with the string contents of undefined_string.
• image and datetime types are handled by calling image_value_handler and datetime_value_handler. These functions must either throw an exception, which propegates up, return any other flexible type (e.g. dict, list, vector, etc.), or return FLEX_UNDEFINED, in which case that value is ignored.

to_flat_dict() [2/2]

flex_dict turi::to_flat_dict	(	const flexible_type &	input,
		const flex_string &	separator,
		const flex_string &	undefined_string,
		const std::string &	image_policy = "error",
		const std::string &	datetime_policy = "error"
	)

Identical to to_flat_dict, except that image_policy and datetime_policy determine the handling of image and datetime types rather than a custom function. Currently, the only possible value for this is "error".

to_sarray_of_flat_dictionaries()

gl_sarray turi::to_sarray_of_flat_dictionaries	(	gl_sarray	input,
		const flex_string &	sep,
		const flex_string &	undefined_string,
		const std::string &	image_policy = "error",
		const std::string &	datetime_policy = "error"
	)

Applies to_flat_dict to all elements in an sarray, returning the transformed sarray.

to_variant()

template<typename T >

variant_type turi::to_variant ( const T &  f )

inline

Converts an arbitrary value to a variant. T can be alot of possibilities. See the supported type list for details.

Definition at line 308 of file variant.hpp.

translate_row_to_ml_data_entry() [1/3]

std::vector<ml_data_entry> turi::translate_row_to_ml_data_entry	(	const std::shared_ptr< ml_metadata > &	metadata,
		const std::vector< ml_data_entry_global_index > &	row
	)

Translation from one row type to another

translate_row_to_ml_data_entry() [2/3]

std::vector<ml_data_entry> turi::translate_row_to_ml_data_entry	(	const std::shared_ptr< ml_metadata > &	metadata,
		const DenseVector &	row
	)

Translation routines.

translate_row_to_ml_data_entry() [3/3]

std::vector<ml_data_entry> turi::translate_row_to_ml_data_entry	(	const std::shared_ptr< ml_metadata > &	metadata,
		const SparseVector &	v
	)

Translates the original sparse row format to the ml_data_entry vector.

translate_row_to_original() [1/3]

std::vector<flexible_type> turi::translate_row_to_original	(	const std::shared_ptr< ml_metadata > &	metadata,
		const DenseVector &	v
	)

Translates the original sparse row format to the original flexible types.

translate_row_to_original() [2/3]

std::vector<flexible_type> turi::translate_row_to_original	(	const std::shared_ptr< ml_metadata > &	metadata,
		const SparseVector &	v
	)

Translates the original sparse row format to the original flexible types.

translate_row_to_original() [3/3]

std::vector<flexible_type> turi::translate_row_to_original	(	const std::shared_ptr< ml_metadata > &	metadata,
		const std::vector< ml_data_entry_global_index > &	row
	)

Translate a vector of global indices to the next

transpose_sparse_sarray()

template<typename T >

std::shared_ptr<sarray<std::vector<std::pair<size_t, T> > > > turi::transpose_sparse_sarray	(	std::shared_ptr< sarray< std::vector< std::pair< size_t, T > > > >	data,
		const std::vector< size_t > &	item_counts,
		size_t	max_memory_usage
	)

Transpose a sparse sarray of sorted vectors of entry pairs. Does it effeciently and close to within memory bounds.

The input data is a sarray of vectors of (column_index, value) pairs. This is then transposed to an similar sarray of vectors of (row_index, value) pairs, where each row consists of the matching values of column_index in the original sarray. The resulting vectors are sorted.

To do this efficiently, the number of elements for each column_index are required ahead of time. This is passed in as item_counts, which should be of size equal to the column dimension.

max_memory_usage gives the maximum memory allowed for the computation. The larger max_memory_usage is, the fewer passes through the data are required.

Definition at line 340 of file utilities.hpp.

unescape_string() [1/4]

void turi::unescape_string	(	std::string &	cal,
		bool	use_escape_char,
		char	escape_char,
		char	quote_char,
		bool	double_quote
	)

Unescapes a string inplace

unescape_string() [2/4]

void turi::unescape_string	(	std::string &	cal,
		char	escape_char,
		char	quote_char,
		bool	double_quote
	)

Unescapes a string inplace

unescape_string() [3/4]

size_t turi::unescape_string	(	char *	cal,
		size_t	length,
		bool	use_escape_char,
		char	escape_char,
		char	quote_char,
		bool	double_quote
	)

Unescapes a string inplace, returning the new length

unescape_string() [4/4]

size_t turi::unescape_string	(	char *	cal,
		size_t	length,
		char	escape_char,
		char	quote_char,
		bool	double_quote
	)

Unescapes a string inplace, returning the new length

variant_deep_load() [1/2]

void turi::variant_deep_load	(	variant_type &	v,
		iarchive &	iarc
	)

Deserialize the variant type, allocate new resources for the pointer types.

variant_deep_load() [2/2]

template<typename T >

void turi::variant_deep_load	(	T &	v,
		iarchive &	iarc
	)

Overload of above for types castable to and from variant_type

Definition at line 40 of file variant_deep_serialize.hpp.

variant_deep_save() [1/2]

void turi::variant_deep_save	(	const variant_type &	v,
		oarchive &	oarc
	)

Serialize the variant type, deep copying the pointer types.

variant_deep_save() [2/2]

template<typename T >

void turi::variant_deep_save	(	const T &	v,
		oarchive &	oarc
	)

Overload of above for types castable to and from variant_type

Definition at line 22 of file variant_deep_serialize.hpp.

variant_get_ref() [1/2]

template<typename T >

static T& turi::variant_get_ref ( variant_type &  v )

inlinestatic

Gets a reference to a content of a variant. Throws if variant contains an inappropriate type.

Definition at line 131 of file variant.hpp.

variant_get_ref() [2/2]

template<typename T >

static const T& turi::variant_get_ref ( const variant_type &  v )

inlinestatic

Gets a const reference to the content of a variant. Throws if variant contains an inappropriate type.

Definition at line 144 of file variant.hpp.

variant_get_value()

template<typename T >

std::decay<T>::type turi::variant_get_value ( const variant_type &  v )

inline

Reads an arbitrary type from a variant. T can be alot of possibilities. See the supported type list for details.

Definition at line 320 of file variant.hpp.

variant_set_value()

template<typename T >

void turi::variant_set_value ( variant_type &  v, const T &  f  )

inline

Stores an arbitrary value in a variant

Definition at line 297 of file variant.hpp.

varmap_to_flexmap()

std::map<std::string, flexible_type> turi::varmap_to_flexmap ( const variant_map_type &  map )

inline

Extract all flexible_type values from the varmap into a std::map<std::String, flexible_type>. All other value types will be ignored.

Definition at line 33 of file toolkit_util.hpp.

write_image()

void turi::write_image	(	const std::string &	filename,
		char *	data,
		size_t	width,
		size_t	height,
		size_t	channels,
		Format	format
	)

Write the pixels into file as jpeg Only works with local files.

Variable Documentation

DEFAULT_NUM_GRAPH_LAMBDA_WORKERS

size_t turi::DEFAULT_NUM_GRAPH_LAMBDA_WORKERS

Number of graph lambda workers

DEFAULT_NUM_PYLAMBDA_WORKERS

size_t turi::DEFAULT_NUM_PYLAMBDA_WORKERS

Number of pylambda workers

DIR_ARCHIVE_INI_FILE

const char* turi::DIR_ARCHIVE_INI_FILE

This file is the human readable INI file in the directory containing information about the archive.

DIR_ARCHIVE_OBJECTS_BIN

const char* turi::DIR_ARCHIVE_OBJECTS_BIN

This file is the binary archive used to hold serializable object.

FLEX_UNDEFINED

flexible_type turi::FLEX_UNDEFINED = flex_undefined()

static

A global static variable to be used when an undefined value is needed. i.e.

flexible_type f = FLEX_UNDEFINED;

Definition at line 1405 of file flexible_type.hpp.