#include <core/storage/sframe_data/sarray_v2_block_writer.hpp>

Public Member Functions
void	init (std::string group_index_file, size_t num_segments, size_t num_columns)

void	open_segment (size_t segment_id, std::string filename)

void	set_options (const std::string &option, int64_t value)

size_t	write_block (size_t segment_id, size_t column_id, char *data, block_info block)

size_t	write_typed_block (size_t segment_id, size_t column_id, const std::vector< flexible_type > &data, block_info block)

template<typename T >
size_t	write_block (size_t segment_id, size_t column_id, const std::vector< T > &data, block_info block)

void	close_segment (size_t segment_id)

group_index_file_information &	get_index_info ()

void	write_index_file ()

Detailed Description

Provides the file writing implementation for the v2 block format. See the sarray_v2_block_manager for details on the format.

Basic usage is:

block_writer writer;
writer.init("index", #segments, #columns);
for i = 0 to  #segments:
  writer.open_segment(i, filename)
// now you perform repeat calls to the following functions
// to write blocks to the segments/columns
writer.write_block(...)
writer.write_typed_block(...)
// close all writes
for i = 0 to  #segments:
  writer.close_segment(i)
// output the array group index file
writer.write_index_file()

Definition at line 59 of file sarray_v2_block_writer.hpp.

Member Function Documentation

◆ close_segment()

void turi::v2_block_impl::block_writer::close_segment ( size_t segment_id )

Closes the segment file

◆ get_index_info()

group_index_file_information& turi::v2_block_impl::block_writer::get_index_info ( )

Gets a modifiable reference to the index information for the data written. If the segment is not closed yet, partial results will be provided.

◆ init()

void turi::v2_block_impl::block_writer::init	(	std::string	group_index_file,
		size_t	num_segments,
		size_t	num_columns
	)

Opens a block writer with a target index file, the number of segments to write, and the number of columns to write.

◆ open_segment()

void turi::v2_block_impl::block_writer::open_segment	(	size_t	segment_id,
		std::string	filename
	)

Opens a segment, using a given file name.

◆ set_options()

void turi::v2_block_impl::block_writer::set_options	(	const std::string &	option,
		int64_t	value
	)

Sets write options. The only option available now is "disable_padding". If set to non-zero, disables 4K padding of blocks.

◆ write_block() [1/2]

size_t turi::v2_block_impl::block_writer::write_block	(	size_t	segment_id,
		size_t	column_id,
		char *	data,
		block_info	block
	)

Writes a block of data into a segment.

Parameters

segmentid	The segment to write to
data	Pointer to the data to write
block_info	Metadata about the block.

The only fields in block_info which must be filled is block_size and num_elem. Returns the actual number of bytes written.

◆ write_block() [2/2]

template<typename T >

size_t turi::v2_block_impl::block_writer::write_block	(	size_t	segment_id,
		size_t	column_id,
		const std::vector< T > &	data,
		block_info	block
	)

inline

Writes a block of arbitrary contents. Direct serialization is used.

Definition at line 118 of file sarray_v2_block_writer.hpp.

◆ write_index_file()

void turi::v2_block_impl::block_writer::write_index_file ( )

Writes the index file

◆ write_typed_block()

size_t turi::v2_block_impl::block_writer::write_typed_block	(	size_t	segment_id,
		size_t	column_id,
		const std::vector< flexible_type > &	data,
		block_info	block
	)