Classes
class	execution_node

class	generic_aggregator

struct	less_than_full_function

struct	less_than_partial_function

struct	materialize_options

struct	node_info

class	operator_impl< planner_node_type::APPEND_NODE >

class	operator_impl< planner_node_type::BINARY_TRANSFORM_NODE >

struct	operator_impl< planner_node_type::CONSTANT_NODE >

class	operator_impl< planner_node_type::GENERALIZED_TRANSFORM_NODE >

struct	operator_impl< planner_node_type::GENERALIZED_UNION_PROJECT_NODE >

class	operator_impl< planner_node_type::IDENTITY_NODE >

class	operator_impl< planner_node_type::LAMBDA_TRANSFORM_NODE >

class	operator_impl< planner_node_type::LOGICAL_FILTER_NODE >

struct	operator_impl< planner_node_type::PROJECT_NODE >

struct	operator_impl< planner_node_type::RANGE_NODE >

struct	operator_impl< planner_node_type::REDUCE_NODE >

struct	operator_impl< planner_node_type::SFRAME_SOURCE_NODE >

class	operator_impl< planner_node_type::TERNARY_OPERATOR >

class	operator_impl< planner_node_type::TRANSFORM_NODE >

struct	operator_impl< planner_node_type::UNION_NODE >

class	opt_append_on_source

class	opt_eliminate_empty_append

class	opt_eliminate_identity_project

class	opt_merge_all_same_sarrays

class	opt_merge_projects

class	opt_project_append_exchange

class	opt_project_on_source

class	opt_split_contractive_expansive_projection

class	opt_transform

class	opt_union_on_source

class	opt_union_project_exchange

class	opt_union_project_merge

class	optimization_engine

struct	optimization_transform_registry

class	planner

struct	planner_node

class	query_context

class	query_operator

struct	query_operator_attributes

class	subplan_executor

Typedefs
typedef std::shared_ptr< planner_node >	pnode_ptr
	A handy typedef.

Enumerations
enum	planner_node_type : int

Functions
sframe	permute_sframe (sframe &values_sframe, std::shared_ptr< sarray< flexible_type > > forward_map)

std::shared_ptr< sframe >	ec_sort (std::shared_ptr< planner_node > sframe_planner_node, const std::vector< std::string > column_names, const std::vector< size_t > &key_column_indices, const std::vector< bool > &sort_orders)

std::shared_ptr< sframe >	groupby_aggregate (const std::shared_ptr< planner_node > &source, const std::vector< std::string > &source_column_names, const std::vector< std::string > &keys, const std::vector< std::string > &output_column_names, const std::vector< std::pair< std::vector< std::string >, std::shared_ptr< group_aggregate_value >>> &groups)

std::shared_ptr< sframe >	sort (std::shared_ptr< planner_node > sframe_planner_node, const std::vector< std::string > column_names, const std::vector< size_t > &sort_column_indices, const std::vector< bool > &sort_orders)

std::shared_ptr< sframe >	sort_and_merge (const std::shared_ptr< sarray< std::pair< flex_list, std::string >>> &partition_array, const std::vector< bool > &partition_sorted, const std::vector< size_t > &partition_sizes, const std::vector< bool > &sort_orders, const std::vector< size_t > &permute_order, const std::vector< std::string > &column_names, const std::vector< flex_type_enum > &column_types)

std::vector< flex_type_enum >	infer_planner_node_type (std::shared_ptr< planner_node > pnode)

int64_t	infer_planner_node_length (std::shared_ptr< planner_node > pnode)

size_t	infer_planner_node_num_output_columns (std::shared_ptr< planner_node > pnode)

size_t	infer_planner_node_num_dependency_nodes (std::shared_ptr< planner_node > pnode)

std::shared_ptr< query_operator >	planner_node_to_operator (std::shared_ptr< planner_node > pnode)

std::string	planner_node_type_to_name (planner_node_type type)

planner_node_type	planner_node_name_to_type (const std::string &name)

query_operator_attributes	planner_node_type_to_attributes (planner_node_type type)

std::pair< bool, bool >	prove_equal_length (const std::shared_ptr< planner_node > &a, const std::shared_ptr< planner_node > &b)

bool	consumes_inputs_at_same_rates (const query_operator_attributes &attr)

bool	is_linear_transform (const query_operator_attributes &attr)

bool	is_sublinear_transform (const query_operator_attributes &attr)

bool	is_source_node (const query_operator_attributes &attr)

bool	is_parallel_slicable (const std::shared_ptr< planner_node > &n)

bool	is_linear_graph (const std::shared_ptr< planner_node > &n)

std::vector< size_t >	get_parallel_slicable_codes (const std::shared_ptr< planner_node > &n)

std::string	planner_node_repr (const std::shared_ptr< planner_node > &node)

pnode_ptr	make_segmented_graph (pnode_ptr n, size_t split_idx, size_t n_splits, std::map< pnode_ptr, pnode_ptr > &memo)

pnode_ptr	make_sliced_graph (pnode_ptr n, size_t begin_index, size_t end_index, std::map< pnode_ptr, pnode_ptr > &memo)

std::vector< size_t >	get_stages_to_run (const materialize_options &exec_params)

void	populate_transforms (optimization_transform_registry *otr)

template<typename ResultType , typename ReduceFunctionType , typename AggregateFunctionType >
ResultType	reduce (std::shared_ptr< planner_node > input, ReduceFunctionType reduce_fn, AggregateFunctionType aggregate_fn, ResultType init=ResultType())

Variables
recursive_mutex	global_query_lock

Detailed Description

SFrame Lazy Evaluation and Execution

Enumeration Type Documentation

◆ planner_node_type

enum turi::query_eval::planner_node_type : int

strong

An enumeration of all operator types.

Definition at line 24 of file operator_properties.hpp.

Function Documentation

◆ consumes_inputs_at_same_rates()

bool turi::query_eval::consumes_inputs_at_same_rates ( const query_operator_attributes & attr )

This operator consumes all inputs at the same rate, and there is exactly one row for every input row.

◆ get_parallel_slicable_codes()

std::vector<size_t> turi::query_eval::get_parallel_slicable_codes ( const std::shared_ptr< planner_node > & n )

Returns a set of integers giving the different parallel slicable units for the inputs of a particular node. If

◆ get_stages_to_run()

std::vector<size_t> turi::query_eval::get_stages_to_run ( const materialize_options & exec_params )

Deterime which stages are run, given exec_params.

◆ infer_planner_node_length()

int64_t turi::query_eval::infer_planner_node_length ( std::shared_ptr< planner_node > pnode )

Infers the length of the output of a planner node by backtracking its dependencies.

Returns -1 if the length cannot be computed without an actual execution.

◆ infer_planner_node_num_dependency_nodes()

size_t turi::query_eval::infer_planner_node_num_dependency_nodes ( std::shared_ptr< planner_node > pnode )

Returns the number of nodes in this planning graph, including pnode.

◆ infer_planner_node_num_output_columns()

size_t turi::query_eval::infer_planner_node_num_output_columns ( std::shared_ptr< planner_node > pnode )

Infers the number of columns present in the output.

◆ infer_planner_node_type()

std::vector<flex_type_enum> turi::query_eval::infer_planner_node_type ( std::shared_ptr< planner_node > pnode )

Infers the type schema of a planner node by backtracking its dependencies.

◆ is_linear_graph()

bool turi::query_eval::is_linear_graph ( const std::shared_ptr< planner_node > & n )

Returns true if the graph contains only linear transformations.

◆ is_linear_transform()

bool turi::query_eval::is_linear_transform ( const query_operator_attributes & attr )

A collection of flags used in actually doing the query optimization.

◆ is_parallel_slicable()

bool turi::query_eval::is_parallel_slicable ( const std::shared_ptr< planner_node > & n )

Returns true if the output of this node can be parallel sliceable by the sources on this block, and false otherwise.

◆ is_source_node()

bool turi::query_eval::is_source_node ( const query_operator_attributes & attr )

This operator is a source node.

◆ is_sublinear_transform()

bool turi::query_eval::is_sublinear_transform ( const query_operator_attributes & attr )

This operator consumes all inputs at the same rate, but reduces the rows in the output.

◆ make_segmented_graph()

pnode_ptr turi::query_eval::make_segmented_graph	(	pnode_ptr	n,
		size_t	split_idx,
		size_t	n_splits,
		std::map< pnode_ptr, pnode_ptr > &	memo
	)

Turns a node graph into the proper segmented part.

◆ make_sliced_graph()

pnode_ptr turi::query_eval::make_sliced_graph	(	pnode_ptr	n,
		size_t	begin_index,
		size_t	end_index,
		std::map< pnode_ptr, pnode_ptr > &	memo
	)

Slice the node graph input with begin and end.

Note:

only allows forward slice, i.e begin_index <= end_index
allows recursive slice, for example:

n1 = make_sliced_graph(n0, 5, 10) // n1 contains row 5 to 9 of n0

n2 = make_sliced_graph(n1, 1, 2) // n2 contains row 1 of n1, which is row 6 of n0

final slice range cannot exeeds the original graph

n1 = make_sliced_graph(n0, 0, n0.size() +1) // throws error

◆ planner_node_name_to_type()

planner_node_type turi::query_eval::planner_node_name_to_type ( const std::string & name )

Get the type of the node from the name.

◆ planner_node_repr()

std::string turi::query_eval::planner_node_repr ( const std::shared_ptr< planner_node > & node )

Representation of the node as a string.

◆ planner_node_to_operator()

std::shared_ptr<query_operator> turi::query_eval::planner_node_to_operator ( std::shared_ptr< planner_node > pnode )

Transforms a planner node into the operator.

◆ planner_node_type_to_attributes()

query_operator_attributes turi::query_eval::planner_node_type_to_attributes ( planner_node_type type )

Get the attribute struct from the type.

◆ planner_node_type_to_name()

std::string turi::query_eval::planner_node_type_to_name ( planner_node_type type )

Get the name of the node from the type.

◆ populate_transforms()

void turi::query_eval::populate_transforms ( optimization_transform_registry * otr )

Populate the transform registry with all transforms.

◆ prove_equal_length()

std::pair<bool, bool> turi::query_eval::prove_equal_length	(	const std::shared_ptr< planner_node > &	a,
		const std::shared_ptr< planner_node > &	b
	)

Attempts to prove that the two inputs have equal length. Returns a pair [can_prove, is_equal_length]. If can_prove is false, we were unable to confirm that the two inputs have equal or non-equal length; the value of is_equal_length is then meaningless but it will be set to false.

If can_prove is true, and is_equal_length is true, we ensure that the two have equal length. If is_equal_length is false, we ensure that the two do not have equal length.

Classes

Typedefs

Enumerations

Functions

Variables