gl_sarray.hpp

/* Copyright © 2017 Apple Inc. All rights reserved.
 *
 * Use of this source code is governed by a BSD-3-clause license that can
 * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 */
#ifndef TURI_UNITY_GL_SARRAY_HPP
#define TURI_UNITY_GL_SARRAY_HPP
#include <cmath>
#include <memory>
#include <cstddef>
#include <string>
#include <iostream>
#include <core/storage/sframe_data/sframe_rows.hpp>
#include <core/storage/sframe_data/group_aggregate_value.hpp>
#include <core/data/flexible_type/flexible_type.hpp>
#include <visualization/server/plot.hpp>

namespace turi {
/**************************************************************************/
/*                                                                        */
/*                          Forward Declarations                          */
/*                                                                        */
/**************************************************************************/
class unity_sarray;
class unity_sarray_base;
class gl_sframe;
class gl_sarray_range;

template <typename T>
class sarray;

template <typename T>
class sarray_reader;

template <typename T>
class sarray_reader_buffer;

/**
 * \ingroup group_glsdk
 * An immutable, homogeneously typed array object backed by persistent storage.
 *
 * The gl_sarray is a contiguous column of a single type with missing value
 * support, and works with disk to support the holding of data that is much
 * larger than the machine's main memory. Runtime typing of the gl_sarray is
 * managed through the \ref flexible_type, which is an efficient runtime typed
 * value. The types supported by the flexible_type are listed in \ref
 * flex_type_enum.
 *
 *
 * ### Construction
 *
 * Abstractly the gl_sarray provides an interface to read and write \ref
 * flexible_type values where all values have the same type at runtime (for
 * instance flex_type_enum::INTEGER). The special type \ref
 * flex_type_enum::UNDEFINED (or the value \ref FLEX_UNDEFINED )
 * is used to denote a missing value and can be used in combination with any
 * types.
 *
 * For instance:
 *
 * \code
 * // creates an array of 5 integers
 * gl_sarray g({1,2,3,4,5});
 *
 * // creates an array of 5 doubles
 * gl_sarray g({1.0,2.0,3.0,4.0,5.0});
 *
 * // creates an array of 4 doubles with one missing value
 * gl_sarray g({1.0,2.0,3.0,FLEX_UNDEFINED,5.0});
 * \endcode
 *
 * While the gl_sarray is conceptually immutable, all that really means is that
 * element-wise modifications are not permitted. However, full SArray assignments
 * are permitted.
 *
 * \code
 * gl_sarray g({1,2,3,4,5});
 * gl_sarray s = g + 1;
 * // s is {2,3,4,5,6}
 * \endcode
 *
 * ### Usage
 *
 * The gl_sarray API is designed to very closely mimic the Python SArray API
 * and supports much of the Python-like capabilities, but in C++.
 *
 * For instance, vector and operations:
 * \code
 * gl_sarray s{1,2,3,4,5};
 * gl_sarray y{2.0,3.0,2.5,1.5,2.5};
 * auto t = (s + 10) / y;
 * \endcode
 *
 * Logical filters:
 * \code
 * gl_sarray s{1,2,3,4,5};
 * gl_sarray selector{0,0,1,1,1}
 * auto t = s[selector];
 * // t is [3,4,5]
 *
 *
 * gl_sarray s{1,2,3,4,5};
 * auto t = s[s < 3];
 * // t is [1,2]
 * \endcode
 *
 * Python Range slicing:
 * \code
 * gl_sarray s{1,2,3,4,5};
 * auto t = s[{0,3}];
 * auto u = s[{-3,-1}];
 *
 * // t is [1,2]
 * // u is [3,4]
 * \endcode
 *
 * And many others.
 *
 * The gl_sarray can be read \b inefficiently using operator[]
 * \code
 * gl_sarray s{1,2,3,4,5};
 * int val = s[2];
 * // val == 3
 * \endcode
 *
 * Or iterated efficiently using the \ref range_iterator
 * \code
 * for (const auto& i: sa.range_iterator() {
 *   ...
 * }
 * \endcode
 *
 *
 * The range_iterator materializes the SFrame if not already materialized, but
 * \ref materialize_to_callback can be used to read the SFrame without
 * materialization.
 *
 * The gl_sarray can constructed in a variety of means:
 *   - If the data to be written is already in memory, it can be created
 *     using the
 *     \ref gl_sarray::gl_sarray(const std::vector<flexible_type>& values, flex_type_enum dtype) "gl_sarray constructor"
 *   - Otherwise, the \ref gl_sarray_writer can be used which provides a simple
 *     write interface.
 *
 * ### Python Binding
 *
 * When used as an input argument in an SDK function, it permits a Python SArray
 * to be passed as an argument. When used in an output argument, it will return
 * a Python SArray.
 *
 * For instance:
 * \code
 * //
 * // Compiled as example.so
 * //
 * gl_sarray add_one_to_array(gl_sarray data) {
 *   return s + 1;
 * }
 * BEGIN_FUNCTION_REGISTRATION
 * REGISTER_FUNCTION(add_one_to_array, "data");
 * END_FUNCTION_REGISTRATION
 * \endcode
 *
 * Will allow this to be done in Python:
 * \code{.py}
 * import turicreate as gl
 * import example
 * sa = SArray([1,2,3,4,5])
 * ret = example.add_one_to_array(sa)
 * # ret is now [2,3,4,5,6]
 * \endcode
 *
 * ### Details
 *
 * The gl_sarray is internally a reference object. i.e. in the code below,
 * both a and b will point to the same underlying sarray. However since
 * gl_sarray's are immutable, this does not introduce any interface quirks.
 * \code
 * gl_sarray a{1,2,3};
 * gl_sarray b = a;
 * \endcode
 *
 * The gl_sarray is also lazy evaluated behind the scenes to minimize disk
 * access. Thus regardless of the size of the SArray or the complexity of the
 * lambda operation, this operation will run quickly.
 * \code
 * b = (sa.apply(some_complicated_function) + 5) / 2;
 * \endcode
 *
 * This may have the unfortunate effect of hiding errors until materialization
 * is forced to occur. i.e. it might be some time much later in your code
 * that errors in some_complicated_function will trigger.
 *
 * However, not all operations are lazy and certain operations will force
 * materialization, and that is a constant target for optimization.
 *
 * If you want to force materialization yourself, use \ref materialize()
 */
class gl_sarray {
 public:
  /// Constructs an empty SArray
  gl_sarray();

  /// Copy Constructor
  gl_sarray(const gl_sarray&);

  /// Move Constructor
  gl_sarray(gl_sarray&&);

  /// Copy Assignment
  gl_sarray& operator=(const gl_sarray&);

  /// Move Assignment
  gl_sarray& operator=(gl_sarray&&);

  /**
   * Constructs a gl_sarray from a binary SArray saved previously with
   * \ref save().
   *
   * \see save()
   */
  explicit gl_sarray(const std::string& directory);

  /**
   * Constructs an gl_sarray from a in memory vector of values.
   * \code
   * std::vector<flexible_type> values{1,2,3,4,5};
   *
   * // auto infers data type
   * gl_sarray sa(values);
   *
   * // resultant array is of floating point type.
   * // Automatic type casting is performed internally.
   * gl_sarray sa(values, flex_type_enum::FLOAT);
   * \endcode
   *
   */
  gl_sarray(const std::vector<flexible_type>& values,
            flex_type_enum dtype = flex_type_enum::UNDEFINED);

  virtual ~gl_sarray();

  void construct_from_vector(const std::vector<flexible_type>& values,
            flex_type_enum dtype = flex_type_enum::UNDEFINED);

  /**
   * Constructs a gl_sarray from an initializer list of values.
   *
   * Type is automatically determined.
   * \code
   * // creates an array of 5 integers
   * gl_sarray g({1,2,3,4,5});
   *
   * // creates an array of 5 doubles
   * gl_sarray g({1.0,2.0,3.0,4.0,5.0});
   *
   * // non-contiguous type. Most general type is selected.
   * // This will result in an array of strings.
   * gl_sarray g({1,2.0,"3"});
   * \endcode
   */
  gl_sarray(const std::initializer_list<flexible_type>& values);

/**************************************************************************/
/*                                                                        */
/*                          Static Constructors                           */
/*                                                                        */
/**************************************************************************/

  /**
   * Returns a gl_sarray of size with a constant value.
   *
   * \param value The value to fill the array
   * \param size The size of the array
   *
   * \code
   * // Construct an SArray consisting of 10 zeroes:
   * gl_sarray zeros = gl_sarray::from_const(0, 10);
   * \endcode
   */
  static gl_sarray from_const(const flexible_type& value, size_t size);

  /**
   * Returns a gl_sarray of a sequence of integer values.
   * \param start The starting value
   * \param end One past the last value
   * \param reverse If the values are in reverse
   *
   * \code
   * // returns a sequence of values from 0 to 99
   * gl_sarray::from_sequence(0, 100);
   * // returns a sequence of values from 99 to 0
   * gl_sarray::from_sequence(0, 100, true);
   * \endcode
   */
  static gl_sarray from_sequence(size_t start, size_t end, bool reverse=false);

  /**
   * Constructs an SArray from a json record files.
   *
   * A json record file contains an array of dictionaries.
   * Resultant SArray is of dictionary type.
   */
  static gl_sarray read_json(const std::string& url);

  /**************************************************************************/
  /*                                                                        */
  /*                        Implicit Type Converters                        */
  /*                                                                        */
  /**************************************************************************/
  /// \cond TURI_INTERNAL
  /**
   * \internal
   * Implicit conversion from backend unity_sarray objects.
   */
  gl_sarray(std::shared_ptr<unity_sarray> sarray);
  /**
   * \internal
   * Implicit conversion from backend unity_sarray_base objects.
   */
  gl_sarray(std::shared_ptr<unity_sarray_base> sarray);
  /**
   * \internal
   * Implicit conversion from backend sarray objects.
   */
  gl_sarray(std::shared_ptr<sarray<flexible_type> > sarray);
  /**
   * \internal
   * Implicit conversion to backend sarray objects.
   */
  operator std::shared_ptr<unity_sarray>() const;
  /**
   * \internal
   * Implicit conversion to backend sarray objects.
   */
  operator std::shared_ptr<unity_sarray_base>() const;

  /**
   * \internal
   * Conversion to materialized backend sarray object.
   */
  std::shared_ptr<sarray<flexible_type> > materialize_to_sarray() const;

  /// \endcond

  /**************************************************************************/
  /*                                                                        */
  /*                           Operator Overloads                           */
  /*                                                                        */
  /**************************************************************************/

  /**
   * \name Numeric operator overloads.
   *
   * Most operators are overloaded and will perform element-wise operations
   * on the entire array.
   *
   * For instance:
   * \code
   * gl_sarray a{1,2,3,4,5};
   * // an array of 5 exclamation marks
   * gl_sarray b = gl_sarray::from_const("!", 5);
   *
   * auto ret = (2 * sa - 1).astype(flex_type_enum::STRING) + b;
   * // results in ret being the array ["1!", "3!", "5!", "7!", "9!"];
   * \endcode
   *
   * Comparison operators will return a gl_sarray of binary integers.
   * \code
   * gl_sarray a{1,2,3,4,5};
   * auto ret = a > 3;
   * // ret will be an integer array containing [0,0,0,1,1]
   * \endcode
   *
   * Logical and bitwise operators are equivalent: & and && mean the same thing
   * and | and || and provide logical element-wise "and" and "or"s.
   *
   * \code
   * gl_sarray a{1,2,3,4,5};
   * auto ret = a > 1 && a <= 4;
   * // ret will be an integer array containing [0,1,1,1,0]
   * \endcode
   *
   * These are useful for the logical filter operation:
   * \code
   * gl_sarray a{1,2,3,4,5};
   * gl_sarray b = a.astype(flex_type_enum::STRING);
   * auto ret = b[a > 1 && a <= 4];
   * // ret will be an string array containing ["2","3","4"]
   * \endcode
   *
   * The logical and bitwise operators can be used with non-integral arrays
   * in which case all empty values evaluate to False. i.e. for string,
   * list, and dictionary SArrays, empty values are interpreted as false.
   *
   * For instance:
   * \code
   * gl_sarray a{"1","","2"}; // sarray of strings
   * gl_sarray b{1,1,0}; // sarray of integers
   * auto ret = a && b; // ret is now {1, 0, 0}
   * \endcode
   */
  ///@{
  gl_sarray operator+(const gl_sarray& other) const;
  gl_sarray operator-(const gl_sarray& other) const;
  gl_sarray operator*(const gl_sarray& other) const;
  gl_sarray operator/(const gl_sarray& other) const;
  gl_sarray operator<(const gl_sarray& other) const;
  gl_sarray operator>(const gl_sarray& other) const;
  gl_sarray operator<=(const gl_sarray& other) const;
  gl_sarray operator>=(const gl_sarray& other) const;
  gl_sarray operator==(const gl_sarray& other) const;

  gl_sarray operator+(const flexible_type& other) const;
  gl_sarray operator-(const flexible_type& other) const;
  gl_sarray operator*(const flexible_type& other) const;
  gl_sarray operator/(const flexible_type& other) const;
  gl_sarray operator<(const flexible_type& other) const;
  gl_sarray operator>(const flexible_type& other) const;
  gl_sarray operator<=(const flexible_type& other) const;
  gl_sarray operator>=(const flexible_type& other) const;
  gl_sarray operator==(const flexible_type& other) const;

  gl_sarray operator+=(const gl_sarray& other);
  gl_sarray operator-=(const gl_sarray& other);
  gl_sarray operator*=(const gl_sarray& other);
  gl_sarray operator/=(const gl_sarray& other);

  gl_sarray operator+=(const flexible_type& other);
  gl_sarray operator-=(const flexible_type& other);
  gl_sarray operator*=(const flexible_type& other);
  gl_sarray operator/=(const flexible_type& other);
  gl_sarray operator&&(const gl_sarray& other) const;
  gl_sarray operator||(const gl_sarray& other) const;
  gl_sarray operator&(const gl_sarray& other) const;
  gl_sarray operator|(const gl_sarray& other) const;
  ///@}

  /**
   * Performs an element-wise substring search of "item". The current array
   * must contains strings and item must be a string. Produces a 1 for each
   * row if item is a substring of the row and 0 otherwise.
   */
  gl_sarray contains(const flexible_type& other) const;

  /**
   * Returns the value at a particular array index; generally inefficient.
   *
   * This returns the value of the array at a particular index. Will raise
   * an exception if the index is out of bounds. This operation is generally
   * inefficient: the range_iterator() is prefered.
   */
  flexible_type operator[](int64_t i) const;

  /**
   * Performs a logical filter.
   *
   * This function performs a logical filter: i.e. it subselects all the
   * elements in this array where the corresponding value in the other array
   * evaluates to true.
   * \code
   * gl_sarray a{1,2,3,4,5};
   * auto ret = a[a > 1 && a <= 4];
   * // ret is now the array [2,3,4]
   * \endcode
   */
  gl_sarray operator[](const gl_sarray& slice) const;

  /**
   * Performs a slice Python style.
   *
   * \param slice A list of 2 or 3 values. If 2 values, this is interpreted as
   * {start, end} indices, with an implicit value of step = 1.
   * If 3 values, this is interpreted as {start, step, end}.
   * Values at the positions [start, start+step, start+2*start, ...] are returned
   * until end (exclusive) is reached. Negative start and end values are
   * interpreted as offsets from the end of the array.
   *
   * Given a gl_sarray
   * \code
   * gl_sarray a{1,2,3,4,5,6,7,8,9,10};
   * \endcode
   *
   * Slicing a consecutive range:
   * \code
   * auto ret = a[{1,4}];  // start at index 1, end at index 4
   * // ret is the array [2,3,4]
   * \endcode
   *
   * Slicing a range with a step:
   * \code
   * auto ret = a[{1,2,8}];  // start at index 1, end at index 8 with step size 2
   * // ret is the array [2,4,6,8]
   * \endcode
   *
   * Using negative indexing:
   * \code
   * auto ret = a[{-3,-1}];  // start at end - 3, end at index end - 1
   * // ret is the array [8,9]
   * \endcode
   */
  gl_sarray operator[](const std::initializer_list<int64_t>& slice) const;

  /**************************************************************************/
  /*                                                                        */
  /*                              Make Friends                              */
  /*                                                                        */
  /**************************************************************************/
  friend gl_sarray operator+(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator-(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator*(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator/(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator<(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator>(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator<=(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator>=(const flexible_type& opnd, const gl_sarray& opnd2);
  friend gl_sarray operator==(const flexible_type& opnd, const gl_sarray& opnd2);


  /**************************************************************************/
  /*                                                                        */
  /*                               Iterators                                */
  /*                                                                        */
  /**************************************************************************/
  friend class gl_sarray_range;


  /**
   * Calls a callback function passing each row of the SArray.
   *
   * This does not materialize the array if not necessary.
   * The callback may be called in parallel in which case the argument provides
   * a thread number. The function should return false, but may return
   * true at anytime to quit the iteration process. It may also throw exceptions
   * which will be forwarded to the caller of this function.
   *
   * Each call to the callback passes:
   *  - a thread id,
   *  - a shared_ptr to an sframe_rows object
   *
   * The sframe_rows object looks like a vector<vector<flexible_type>>.
   * i.e. to look at all the rows, you need to write:
   *
   * \code
   * sa.materalize_to_callback([&](size_t, const std::shared_ptr<sframe_rows>& rows) {
   *   for(const auto& row: *rows) {
   *      // each row looks like an std::vector<flexible_type>
   *      // and can be casted to to a vector<flexible_type> if necessary
   *
   *      // But this this is an sarray, the element you want is always in
   *      // row[0]
   *   }
   * });
   * \endcode
   *
   * \param callback The callback to call
   * \param nthreads Number of threads. If not specified, #cpus is used
   */
  void materialize_to_callback(
      std::function<bool(size_t, const std::shared_ptr<sframe_rows>&)> callback,
      size_t nthreads = (size_t)(-1));


  /**
   * Returns a one pass range object with begin() and end() iterators.
   *
   * This will materialize the array.
   *
   * See \ref materialize_to_callback for a lazy version.
   *
   * \param start The starting index of the range
   * \param end The ending index of the range
   *
   * \code
   * // create a sequence of 1,000 integer values
   * gl_sarray sa = gl_sarray::from_sequence(0,1000);
   *
   * // get a range over the entire array
   * auto ra = sa.range_iterator();
   * auto iter = ra.begin();
   * while (iter != ra.end()) {
   *   std::cout << *iter;
   *   ++iter;
   * }
   * \endcode
   *
   * Or more compactly with C++11 syntax:
   * \code
   * for(const auto& val: sa.range_iterator()) {
   *   std::cout << val << "\n";
   * }
   * \endcode
   *
   * The range returned only supports one pass. The outcome of a second call to
   * begin() is undefined after any iterator is advanced.
   *
   * \see gl_sarray_range
   */
  gl_sarray_range range_iterator(size_t start=0, size_t end=(size_t)(-1)) const;

  /**************************************************************************/
  /*                                                                        */
  /*                          All other functions                           */
  /*                                                                        */
  /**************************************************************************/

  /**
   *
   * Saves the gl_sarray to file.
   *
   * When format is "binary" (default), the saved SArray will be in a directory
   * named with the `targetfile` parameter. When format is "text" or "csv",
   * it is saved as a single human readable text file.
   *
   * \param filename  A local path or a remote URL.  If format is 'text', it
   * will be saved as a text file. If format is 'binary', a directory will be
   * created at the location which will contain the SArray.
   *
   * \param format Either "binary", "text", "csv". Defaults to "binary". optional.
   *     Format in which to save the SFrame. Binary saved SArrays can be
   *     loaded much faster and without any format conversion losses.
   *     'text' and 'csv' are synonymous: Each SArray row will be written
   *     as a single line in an output text file. If not
   *     given, will try to infer the format from filename given. If file
   *     name ends with 'csv', 'txt' or '.csv.gz', then save as 'csv' format,
   *     otherwise save as 'binary' format.
   */
  void save(const std::string& directory, const std::string& format="binary") const;

  /**
   * The size of the SArray.
   */
  size_t size() const;

  /**
   * True if size() == 0.
   */
  bool empty() const;

  /**
   * Returns data type of the gl_sarray.
   *
   * \code
   * gl_sarray sa{1,2,3,4,5};
   * flex_type_enum dtype = sa.dtype(); // dtype is flex_type_enum::INTEGER
   *
   * gl_sarray sa{"1","2","3","4","5"};
   * flex_type_enum dtype = sa.dtype(); // dtype is flex_type_enum::STRING
   * \endcode
   */
  flex_type_enum dtype() const;

  /**
   *  For a gl_sarray that is lazily evaluated, force persist this sarray to disk,
   *  committing all lazy evaluated operations.
   *
   *  \see is_materialized
   */
  void materialize() const;

  /**
   * Returns whether or not the sarray has been materialized.
   *
   * \see materialize
   */
  bool is_materialized() const;

  /**
   * Returns an gl_sarray which contains the first n rows of this gl_sarray.
   *
   * \param n  The number of rows to fetch.
   * \code
   * gl_sarray sa({0,1,2,3,4,5,6,7,8,9});
   * auto ret = sa.head(5); // an array of values [0,1,2,3,4]
   * \endcode
   */
  gl_sarray head(size_t n) const;

  /**
   * Returns an gl_sarray which contains the last n rows of this gl_sarray.
   *
   * \param n  The number of rows to fetch.
   * \code
   * gl_sarray sa({0,1,2,3,4,5,6,7,8,9});
   * auto ret = sa.tail(5); // an array of values [5,6,7,8,9]
   * \endcode
   */
  gl_sarray tail(size_t n) const;
  /**
   *
   * Count words in the gl_sarray.
   *
   * \param to_lower Optional. If True, all words are converted to lower case
   * before counting.
   *
   * Return an gl_sarray of dictionary type where each
   * element contains the word count for each word that appeared in the
   * corresponding input element. The words are split on all whitespace and
   * punctuation characters. Only works if this SArray is of string type.
   * Parameters:
   *
   * \code
   * sa = turicreate.SArray(["The quick brown fox jumps.",
   *                     "Word word WORD, word!!!word"])
   * auto ret = count_words(sa)
   * // output array is of type flex_type_enum::DICT and contains
   * [{'quick': 1, 'brown': 1, 'jumps': 1, 'fox': 1, 'the': 1}, {'word': 5}]
   * \endcode
   *
   * \see count_ngrams
   */
  gl_sarray count_words(bool to_lower=true, turi::flex_list delimiters={"\r", "\v", "\n", "\f", "\t", " "}) const;


  /**
   * Return an SArray of dict type where each element contains the count for
   * each of the n-grams that appear in the corresponding input element. The
   * n-grams can be specified to be either character n-grams or word n-grams. The
   * input SArray must contain strings.  Parameters:
   *
   * \param n Optional. The number of words in each n-gram. An n value of 1
   * returns word counts. Defaults to 2.
   *
   * \param method Optional. Either "word" or "character". If “word”, the
   * function performs a count of word n-grams. If “character”, does a character
   * n-gram count. Defaults to "word".
   *
   * \param to_lower Optional. If true, all words are converted to lower case
   * before counting. Defaults to true.
   *
   * \param ignore_space Optional. If method is “character”, indicates if
   * spaces between words are counted as part of the n-gram. For instance, with
   * the input SArray element of “fun games”, if this parameter is set to False
   * one tri-gram would be ‘n g’. If ignore_space is set to True, there would be
   * no such tri-gram (there would still be ‘nga’). This parameter has no effect
   * if the method is set to “word”. Defaults to true.
   *
   * \code
   *  gl_sarray sa({"I like big dogs. I LIKE BIG DOGS."});
   *  gl_sarray ret = count_ngrams(sa, 3);
   *  // returns gl_sarray of dictionary type containing
   *  // [{'big dogs i': 1, 'like big dogs': 2, 'dogs i like': 1, 'i like big': 2}]
   * \endcode
   * \code
   *  gl_sarray sa(["Fun. Is. Fun"]);
   *  gl_sarray ret = count_ngrams(sa, 3, "character")
   *  // returns gl_sarray of dictionary type containing
   *  [{'fun': 2, 'nis': 1, 'sfu': 1, 'isf': 1, 'uni': 1}]
   * \endcode
   *
   mp* \see count_words
   */
  gl_sarray count_ngrams(size_t n=2, std::string method="word",
                         bool to_lower=true, bool ignore_space=true) const;

  /**
   * Filter an SArray of dictionary type by the given keys. By default, all
   * keys that are in the provided list in "keys" are \b excluded from the
   * returned SArray.
   *
   *  \param keys A collection of keys to trim down the elements in the
   *  SArray.
   *
   *  \param exclude Optional If True, all keys that are in the input key list
   *  are removed. If False, only keys that are in the input key list are
   *  retained. Defaults to true.
   *
   * \code
   *  gl_sarray sa({flex_dict{{"this",1}, {"is",1}, {"dog",2}},
   *                flex_dict{{"this", 2}, {"are",2}, {"cat", 1}} });
   *  gl_Sarray ret = sa.dict_trim_by_keys({"this", "is", "and", "are"});
   *  // returns an SArray of dictionary type  containing
   *  // [{'dog': 2}, {'cat': 1}]
   * \endcode
   *
   * \see dict_trim_by_values
   */
  gl_sarray dict_trim_by_keys(const std::vector<flexible_type>& keys,
                              bool exclude=true) const;

  /**
   * Filter dictionary values to a given range (inclusive). Trimming is only
   * performed on values which can be compared to the bound values. Fails on
   * SArrays whose data type is not ``dict``.
   *
   * \param lower Optional. The lowest dictionary value that would be retained
   * in the result. If FLEX_UNDEFINED , lower bound is not applied. Defaults to
   * FLEX_UNDEFINED.
   *
   * \param upper Optional. The highest dictionary value that would be retained
   * in the result.  If FLEX_UNDEFINED, upper bound is not applied. Defaults to
   * FLEX_UNDEFINED.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"this",1},{"is",5},{"dog",7}},
   *                      flex_dict{{"this", 2},{"are",1},{"cat", 5}} });
   * std::cout << sa.dict_trim_by_values(2,5);
   * std::cout << sa.dict_trim_by_values(upper=5);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: dict
   * Rows: 2
   * [{'is': 5}, {'this': 2, 'cat': 5}]
   *
   * dtype: dict
   * Rows: 2
   * [{'this': 1, 'is': 5}, {'this': 2, 'are': 1, 'cat': 5}]
   * \endcode
   *
   * \see dict_trim_by_keys
   */
  gl_sarray dict_trim_by_values(const flexible_type& lower = FLEX_UNDEFINED,
                                const flexible_type& upper = FLEX_UNDEFINED) const;
  /**
   * Create an gl_sarray that contains all the keys from each dictionary
   * element as a list. Fails on gl_sarray objects whose data type is not "dict".
   *
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"this",1},{ "is",5},{ "dog",7}},
   *                      flex_dict{{"this", 2},{ "are", 1},{ "cat", 5}}});
   * std::cout << sa.dict_keys();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: list
   * Rows: 2
   * [['this', 'is', 'dog'], ['this', 'are', 'cat']]
   * \endcode
   *
   * \see dict_values
   */
  gl_sarray dict_keys() const;

  /**
   * Create an \ref gl_sarray that contains all the values from each dictionary
   * element as a list. Fails on \ref gl_sarray objects whose data type is not
   * "dict".
   *
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"this",1},{"is",5},{"dog",7}},
   *                      flex_dict{{"this", 2},{"are", 1},{"cat", 5}}});
   * std::cout << sa.dict_values();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: list
   * Rows: 2
   * [[1, 5, 7], [2, 1, 5]]
   * \endcode
   *
   * \see dict_keys
   */
  gl_sarray dict_values() const;

  /**
   * Create a boolean \ref gl_sarray by checking the keys of an \ref gl_sarray
   * of dictionaries. An element of the output \ref gl_sarray is True if the
   * corresponding input element's dictionary has any of the given keys.  Fails
   * on \ref gl_sarray objects whose data type is not "dict".
   *
   * \param keys A list of key values to check each dictionary against.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"this",1},{ "is",5},{ "dog",7}},
   *                      flex_dict{{"animal",1}},
   *                      flex_dict{{"this", 2},{ "are", 1},{ "cat", 5}}});
   * std::cout << sa.dict_has_any_keys({"is", "this", "are"});
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [1, 1, 0]
   * \endcode
   *
   * \see dict_has_all_keys
   */
  gl_sarray dict_has_any_keys(const std::vector<flexible_type>& keys) const;

  /**
   * Create a boolean \ref gl_sarray by checking the keys of an \ref gl_sarray
   * of dictionaries. An element of the output \ref gl_sarray is True if the
   * corresponding input element's dictionary has all of the given keys.  Fails
   * on \ref gl_sarray objects whose data type is not "dict".
   *
   * \param keys A list of key values to check each dictionary against.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"this",1},{"is",5},{"dog",7}},
   *                      flex_dict{{"this", 2},{"are", 1},{"cat", 5}}});
   * std::cout << sa.dict_has_all_keys({"is", "this"});
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 2
   * [1, 0]
   * \endcode
   *
   * \see dict_has_any_keys
   */
  gl_sarray dict_has_all_keys(const std::vector<flexible_type>& keys) const;


  /**
   * Transform each element of the \ref gl_sarray by a given function. The
   * result \ref gl_sarray is of type "dtype". "fn" should be a function that
   * returns exactly one value which can be cast into the type specified by
   * "dtype".
   *
   * \param fn The function to transform each element. Must return exactly one
   *     value which can be cast into the type specified by "dtype".
   *
   * \param dtype The data type of the new \ref gl_sarray.
   *
   * \param skip_undefined Optional. If true, will not apply "fn" to
   * any undefined values. Defaults to true.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1,2,3});
   * std::cout << sa.apply([](const flexible_type& x) { return x*1; },
   *                       flex_type_enum::INTEGER);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [2, 4, 6]
   * \endcode
   *
   * \see gl_sframe::apply
   */
  gl_sarray apply(std::function<flexible_type(const flexible_type&)> fn,
                  flex_type_enum dtype,
                  bool skip_undefined=true) const;

  /**
   * Filter this \ref gl_sarray by a function.  Returns a new \ref gl_sarray
   * filtered by this \ref gl_sarray.  If "fn" evaluates an element to true,
   * this element is copied to the new \ref gl_sarray. If not, it isn't. Throws
   * an exception if the return type of "fn" is not castable to a boolean
   * value.
   *
   * \param fn Function that filters the \ref gl_sarray. Must evaluate to bool
   * or int.
   *
   * \param skip_undefined Optional. If true, will not apply fn to any
   * undefined values.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1,2,3});
   * std::cout <<  sa.filter([](flexible_type x){ return x < 3; });
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 2
   * [1, 2]
   * \endcode
   *
   * This function is equivalent to the combination of a logical_filter and
   * an apply.
   * \code
   * res = sa[sa.apply(fn)];
   * \endcode
   */
  gl_sarray filter(std::function<bool(const flexible_type&)> fn,
                  bool skip_undefined=true) const;

  /**
   * Create an \ref gl_sarray which contains a subsample of the current
   * \ref gl_sarray.
   *
   * \param fraction The fraction of the rows to fetch. Must be between 0 and 1.
   *
   * Example:
   * \code
   * auto  sa = gl_sarray::from_sequence(0, 10);
   * std::cout <<  sa.sample(.3);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [2, 6, 9]
   * \endcode
   */
  gl_sarray sample(double fraction) const;

  /**
   * Create an \ref gl_sarray which contains a subsample of the current
   * \ref gl_sarray.
   *
   * \param fraction The fraction of the rows to fetch. Must be between 0 and 1.
   *
   * \param seed The random seed for the random number generator.
   * Deterministic output is obtained if this is set to a constant.
   *
   * Example:
   * \code
   * auto  sa = gl_sarray::from_sequence(0, 10);
   * std::cout <<  sa.sample(.3, 12345);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [1,3,6,9]
   * \endcode
   */
  gl_sarray sample(double fraction, size_t seed, bool exact=false) const;

  /**
   * Returns an SArray with a hash of each element. seed can be used to change
   * the hash function to allow this method to be used for random number generation.
   *
   * \param seed Defaults to 0. Can be changed to different values to get different hash results.
   *
   * Example:
   * \code
   * sa = gl_sarray::from_sequence(0,10);
   * std::cout << sa.hash(123)
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 10
   * [-2176393851141330893, 7600995152976636137, -5571280844667425574, -4385410391720336496, -4446257658862464208, -7571182417602171808, 3644372782970789199, 3084542717492096231, 4758268028978242780, -6520852338875851008]
   * \endcode
   */
  gl_sarray hash(size_t seed = 0) const;

  /**
   * Return true if every element of the \ref gl_sarray evaluates to true. For
   * numeric \ref gl_sarray objects zeros and missing values ("None") evaluate
   * to false, while all non-zero, non-missing values evaluate to true. For
   * string, list, and dictionary \ref gl_sarray objects, empty values (zero
   * length strings, lists or dictionaries) or missing values ("None") evaluate
   * to false. All other values evaluate to true.  Returns true on an empty
   * \ref gl_sarray.
   *
   * Example:
   * \code
   * std::cout <<  gl_sarray({1, FLEX_UNDEFINED}).all();
   * std::cout <<  gl_sarray({1, 0}).all();
   * std::cout <<  gl_sarray({1, 2}).all();
   * std::cout <<  gl_sarray({"hello", "world"}).all();
   * std::cout <<  gl_sarray({"hello", ""}).all();
   * std::cout <<  gl_sarray({}).all();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * 0
   * 0
   * 1
   * 1
   * 0
   * 1
   * \endcode
   *
   * \see any
   */



  bool all() const;

  /**
   * Return true if any element of the \ref gl_sarray evaluates to true. For
   * numeric \ref gl_sarray objects any non-zero value evaluates to true. For
   * string, list, and dictionary \ref gl_sarray objects, any element of
   * non-zero length evaluates to true.  Returns false on an empty \ref
   * gl_sarray.
   *
   * Example:
   * \code
   * std::cout <<  gl_sarray({1, FLEX_UNDEFINED}).any();
   * std::cout <<  gl_sarray({1, 0}).any();
   * std::cout <<  gl_sarray({0, 0}).any();
   * std::cout <<  gl_sarray({"hello", "world"}).any();
   * std::cout <<  gl_sarray({"hello", ""}).any();
   * std::cout <<  gl_sarray({"", ""}).any();
   * std::cout <<  gl_sarray({}).any();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * 1
   * 1
   * 0
   * 1
   * 1
   * 0
   * 0
   * \endcode
   *
   * \see all
   */
  bool any() const;

  /**
   * Get maximum numeric value in \ref gl_sarray.  Returns FLEX_UNDEFINED on an
   * empty \ref gl_sarray. Raises an exception if called on an \ref gl_sarray
   * with non-numeric type.
   *
   * Example:
   * \code
   * std::cout <<  gl_sarray({14, 62, 83, 72, 77, 96, 5, 25, 69, 66}).max();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * 96
   * \endcode
   *
   * \see min
   */
  flexible_type max() const;

  /**
   * Get minimum numeric value in \ref gl_sarray.  Returns FLEX_UNDEFINED on an
   * empty \ref gl_sarray. Raises an exception if called on an \ref gl_sarray
   * with non-numeric type.
   *
   * Example:
   * \code
   * std::cout <<  gl_sarray({14, 62, 83, 72, 77, 96, 5, 25, 69, 66}).min();
   * \endcode
   *
   *
   * \see max
   */
  flexible_type min() const;

  /**
   * Sum of all values in this \ref gl_sarray.
   *
   * Raises an exception if called on an \ref gl_sarray of strings, lists, or
   * dictionaries. If the \ref gl_sarray contains numeric arrays (flex_vec)
   * and all the arrays are the same length, the sum over all the arrays will
   * be returned. Returns FLEX_UNDEFINED on an empty \ref gl_sarray. For large
   * values, this may overflow without warning.
   */
  flexible_type sum() const;

  /**
   * Mean of all the values in the \ref gl_sarray, or mean image.  Returns
   * FLEX_UNDEFINED on an empty \ref gl_sarray. Raises an exception if called
   * on an \ref gl_sarray with non-numeric type or non-Image type.
   */
  flexible_type mean() const;

  /**
   * Standard deviation of all the values in the \ref gl_sarray.
   * Returns FLEX_UNDEFINED on an empty \ref gl_sarray. Raises an exception if
   * called on an \ref gl_sarray with non-numeric type.
   */
  flexible_type std() const;

  /**
   * Number of non-zero elements in the \ref gl_sarray.
   */
  size_t nnz() const;

  /**
   * Number of missing elements in the \ref gl_sarray.
   */
  size_t num_missing() const;


  /**
   * Create a new \ref gl_sarray with all the values cast to str. The string
   * format is specified by the 'str_format' parameter.
   *
   * \param str_format The format to output the string. Default format is
   * "%Y-%m-%dT%H:%M:%S%ZP". See the strftime specification for details on
   * the format string.
   *
   * Example:
   * \code
   *
   *  boost::posix_time::ptime t(boost::gregorian::date(2011, 1, 1));
   *  boost::posix_time::ptime epoch(boost::gregorian::date(1970,1,1));
   *  auto x = (t - epoch).total_seconds();
   *
   *  auto sa = gl_sarray({flex_date_time(x)});
   *  std::cout <<  sa.datetime_to_str("%e %b %Y");
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: str
   * Rows: 1
   * [" 1 Jan 2011"]
   * \endcode
   *
   * \see str_to_datetime
   */
  gl_sarray datetime_to_str(const std::string& str_format="%Y-%m-%dT%H:%M:%S%ZP") const;

  /**
   * Create a new \ref gl_sarray with all the values cast to datetime. The
   * string format is specified by the 'str_format' parameter.
   *
   * \param str_format The format to parse the string. Default format is
   * "%Y-%m-%dT%H:%M:%S%ZP". See the strptime specification for details on
   * the format string.
   *
   * Example:
   *
   * \code
   * auto  sa = gl_sarray({"20-Oct-2011 09:30:10 GMT-05:30"});
   * std::cout <<  sa.str_to_datetime("%d-%b-%Y %H:%M:%S %ZP");
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: datetime
   * Rows: 1
   * [20111020T093010]
   * \endcode
   *
   * \see datetime_to_str
   */
  gl_sarray str_to_datetime(const std::string& str_format="%Y-%m-%dT%H:%M:%S%ZP") const;


  /**
   * Create a new \ref gl_sarray with all the values cast to
   * \ref turi::image_type of uniform size.
   *
   * \param width int The width of the new images.
   *
   * \param height int The height of the new images.
   *
   * \param channels int. Number of channels of the new images.
   *
   * \param undefined_on_failure optional. defaults to true. If true,
   * return FLEX_UNDEFINED type instead of Image type on failure.  If false,
   * raises error upon failure.
   *
   * \param allow_rounding optional. Default to false. If true, rounds
   * non-integer values when converting to Image type.  If false, raises error
   * upon rounding.
   */
  gl_sarray pixel_array_to_image(size_t width, size_t height, size_t channels=3,
                                 bool undefined_on_failure=true) const;

  /**
   * Create a new \ref gl_sarray with all values cast to the given type. Throws
   * an exception if the types are not castable to the given type.
   *
   * \param dtype The type to cast the elements to in \ref gl_sarray
   *
   * \param undefined_on_failure: Optional. Defaults to True. If set to true,
   * runtime cast failures will be emitted as missing values rather than
   * failing.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({'1','2','3','4'});
   * std::cout <<  sa.astype(flex_type_enum::INTEGER);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 4
   * [1, 2, 3, 4]
   * \endcode
   *
   * Given an SArray of strings that look like dicts, convert to a dictionary
   * type:
   * \code
   * auto sa = gl_sarray({'flex_dict{{1:2 3,4}}', 'flex_dict{{a:b c,d}}'});
   * std::cout <<  sa.astype(flex_type_enum::DICT);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: dict
   * Rows: 2
   * [{1: 2, 3: 4}, {'a': 'b', 'c': 'd'}]
   * \endcode
   *
   */
  gl_sarray astype(flex_type_enum dtype, bool undefined_on_failure=true) const;

  /**
   * Create a new \ref gl_sarray with each value clipped to be within the given
   * bounds.  In this case, "clipped" means that values below the lower bound
   * will be set to the lower bound value. Values above the upper bound will be
   * set to the upper bound value. This function can operate on \ref gl_sarray
   * objects of numeric type as well as array type, in which case each
   * individual element in each array is clipped. By default "lower" and
   * "upper" are set to "float('nan')" which indicates the respective bound
   * should be ignored. The method fails if invoked on an \ref gl_sarray of
   * non-numeric type.
   * \param lower Optional. The lower bound used to clip.
   *              Ignored if equal to FLEX_UNDEFINED (the default).
   *
   * \param upper Optional. The upper bound used to clip.
   *              Ignored if equal to FLEX_UNDEFINED (the default).
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1,2,3});
   * std::cout <<  sa.clip(2,2);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [2, 2, 2]
   * \endcode
   *
   * \see clip_lower
   * \see clip_upper
   */
  gl_sarray clip(flexible_type lower=FLEX_UNDEFINED,
                 flexible_type upper=FLEX_UNDEFINED) const;

  /**
   * Create new \ref gl_sarray with all values clipped to the given lower
   * bound. This function can operate on numeric arrays, as well as vector
   * arrays, in which case each individual element in each vector is clipped.
   * Throws an exception if the \ref gl_sarray is empty or the types are
   * non-numeric.
   *
   * \param threshold The lower bound used to clip values.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1,2,3});
   * std::cout << sa.clip_lower(2);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [2, 2, 3]
   * \endcode
   *
   * \see clip
   * \see clip_upper
   */
  gl_sarray clip_lower(flexible_type threshold) const;

  /**
   * Create new \ref gl_sarray with all values clipped to the given upper
   * bound. This function can operate on numeric arrays, as well as vector
   * arrays, in which case each individual element in each vector is clipped.
   *
   * \param threshold The upper bound used to clip values.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1,2,3});
   * std::cout << sa.clip_upper(2);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [1, 2, 2]
   * \endcode
   *
   * \see clip
   * \see clip_lower
   */
  gl_sarray clip_upper(flexible_type threshold) const;

  /**
   * Create new \ref gl_sarray containing only the non-missing values of the
   * \ref gl_sarray.  A missing value shows up in an \ref gl_sarray as
   * 'FLEX_UNDEFINED'.  This will also drop NAN values.
   */
  gl_sarray dropna() const;

  /**
   * Create new \ref gl_sarray with all missing values (FLEX_UNDEFINED or NaN)
   * filled in with the given value.  The size of the new \ref gl_sarray will
   * be the same as the original \ref gl_sarray. If the given value is not the
   * same type as the values in the \ref gl_sarray, "fillna" will attempt to
   * convert the value to the original \ref gl_sarray's type. If this fails, an
   * error will be raised.
   *
   * \param value The value used to replace all missing values
   */
  gl_sarray fillna(flexible_type value) const;

  /**
   * Create an \ref gl_sarray indicating which elements are in the top k.
   * Entries are '1' if the corresponding element in the current \ref gl_sarray is a
   * part of the top k elements, and '0' if that corresponding element is
   * not. Order is descending by default.
   *
   * \param topk Optional. Defaults to 10. The number of elements to determine
   * if 'top'
   *
   * \param reverse Optional. Defaults to false. If true, return the topk
   * elements in ascending order
   */
  gl_sarray topk_index(size_t topk=10, bool reverse=false) const;

  /**
   * Append an \ref gl_sarray to the current \ref gl_sarray. Returns a new
   * \ref gl_sarray with the rows from both \ref gl_sarray objects. Both
   * \ref gl_sarray objects must be of the same type.
   *
   * \param other Another \ref gl_sarray whose rows are appended to current \ref gl_sarray.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({1, 2, 3});
   * auto sa2 = gl_sarray({4, 5, 6});
   * std::cout << sa.append(sa2);
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 6
   * [1, 2, 3, 4, 5, 6]
   * \endcode
   *
   * \see \ref gl_sframe.append
   */
  gl_sarray append(const gl_sarray& other) const;

  /**
   * Get all unique values in the current \ref gl_sarray.  Raises an error
   * if the \ref gl_sarray is of dictionary type. Will not necessarily preserve
   * the order of the given \ref gl_sarray in the new \ref gl_sarray.
   *
   * \see gl_sframe::unique
   */
  gl_sarray unique() const;

  /**
   * Length of each element in the current \ref gl_sarray.  Only works on \ref
   * gl_sarray objects of dict, array, or list type. If a given element is a
   * missing value, then the output elements is also a missing value.  This
   * function is equivalent to the following:
   *
   * sa_item_len =  sa.apply([](const flexible_type& x) {
   *                  return flexible_type(x.get_type() == flex_type_enum::UNDEFINED ? 0 : x.size();)
   *                });
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"is_restaurant", 1}, {"is_electronics", 0}},
   *                     flex_dict{{"is_restaurant", 1}, {"is_retail", 1}, {"is_electronics", 0}},
   *                     flex_dict{{"is_restaurant", 0}, {"is_retail", 1}, {"is_electronics", 0}},
   *                     flex_dict{{"is_restaurant", 0}},
   *                     flex_dict{{"is_restaurant", 1}, {"is_electronics", 1}},
   *                     FLEX_UNDEFINED});
   * std::cout << sa.item_length();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 6
   * [2, 3, 3, 1, 2, None]
   * \endcode
   *
   */
  gl_sarray item_length() const;

  /**
   * Splits an \ref gl_sarray of datetime type to multiple columns, return a
   * new \ref gl_sframe that contains expanded columns. A \ref gl_sarray of datetime will be
   * split by default into an \ref gl_sframe of 6 columns, one for each
   * year/month/day/hour/minute/second element.
   *
   * When splitting a \ref gl_sarray of datetime type, new columns are named:
   * prefix.year, prefix.month, etc. The prefix is set by the parameter
   * "column_name_prefix" and defaults to 'X'. If column_name_prefix is
   * FLEX_UNDEFINED or empty, then no prefix is used.
   *
   * If tzone parameter is true, then timezone information is represented
   * as one additional column which is a float shows the offset from
   * GMT(0.0) or from UTC.
   *
   * \param column_name_prefix Optional. If provided, expanded column names
   * would start with the given prefix.  Defaults to "X".
   *
   * \param limit: Optional. Limits the set of datetime elements to expand.
   *    Elements are 'year','month','day','hour','minute', and 'second'.
   *
   * \param tzone: Optional. A boolean parameter that determines whether to
   * show timezone column or not.  Defaults to false.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({"20-Oct-2011", "10-Jan-2012"});
   * auto date_sarray = sa.str_to_datetime("%d-%b-%Y");
   * auto split_sf = date_sarray.split_datetime("", {"day","year"});
   * std::cout << split_sf;
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   *  Columns:
   *      day integer
   *      year  integer
   *  +----------------+----------------+
   *  | day            | year           |
   *  +----------------+----------------+
   *  | 20             | 2011           |
   *  | 10             | 2012           |
   *  +----------------+----------------+
   *  [2 rows x 2 columns]
   * \endcode
   *
   */
  gl_sframe split_datetime(const std::string& column_name_prefix = "X",
                   const std::vector<std::string>& limit = {"year","month","day","hour","minute","second"},
                   bool tzone=false) const;

  /**
   * Convert an \ref gl_sarray of list, array, or dict type to an \ref
   * gl_sframe with multiple columns.
   *
   * "unpack" expands an \ref gl_sarray using the values of each
   * vector/list/dict as elements in a new \ref gl_sframe of multiple columns.
   * For example, an \ref gl_sarray of lists each of length 4 will be expanded
   * into an \ref gl_sframe of 4 columns, one for each list element. An \ref
   * gl_sarray of lists/arrays of varying size will be expand to a number of
   * columns equal to the longest list/array.  An \ref gl_sarray of
   * dictionaries will be expanded into as many columns as there are keys.
   *
   * When unpacking an \ref gl_sarray of list or vector type, new columns are
   * named: "column_name_prefix".0, "column_name_prefix".1, etc. If unpacking a
   * column of dict type, unpacked columns are named "column_name_prefix".key1,
   * "column_name_prefix".key2, etc.
   *
   * When unpacking an \ref gl_sarray of list or dictionary types, missing
   * values in the original element remain as missing values in the resultant
   * columns.  If the "na_value" parameter is specified, all values equal to
   * this given value are also replaced with missing values. In an \ref
   * gl_sarray of vector type, NaN is interpreted as a missing value.
   *
   * \ref gl_sframe::pack_columns() is the reverse effect of unpack.
   *
   * \param column_name_prefix Optional. If provided, unpacked column
   * names would start with the given prefix. Defaults to "X". If the empty
   * string is used, no prefix is used.
   *
   * \param column_types Optional. Column types for the unpacked columns. If
   * not provided, column types are automatically inferred from first 100 rows.
   * Defaults to FLEX_UNDEFINED.
   *
   * \param na_value Optional. Convert all values that are equal to "na_value"
   * to missing value if specified.
   *
   * \param limit  optional limits in the set of list/vector/dict keys to unpack.
   *     For list/vector gl_sarrays, "limit" must contain integer indices.
   *     For dict gl_sarrays, "limit" must contain dictionary keys.
   *
   * Example:
   * \code
   * auto sa = gl_sarray({flex_dict{{"word", "a"},{"count", 1}},
   *                      flex_dict{{"word", "cat"},{"count", 2}},
   *                      flex_dict{{"word", "is"},{"count", 3}},
   *                      flex_dict{{"word", "coming"},{"count", 4}}});
   * std::cout <<  sa.unpack("");
   * \endcode
   * Produces output:
   * \code{.txt}
   * Columns:
   *     count   int
   *     word    str
   * Rows: 4
   * Data:
   * +-------+--------+
   * | count |  word  |
   * +-------+--------+
   * |   1   |   a    |
   * |   2   |  cat   |
   * |   3   |   is   |
   * |   4   | coming |
   * +-------+--------+
   * [4 rows x 2 columns]
   * \endcode
   *
   * Unpack only the key "word":
   * \code
   * std::cout <<  sa.unpack("X", {}, FLEX_UNDEFINED, {"word"});
   * \endcode
   * Produces output:
   * \code{.txt}
   * Columns:
   *     X.word  str
   * Rows: 4
   * Data:
   * +--------+
   * | X.word |
   * +--------+
   * |   a    |
   * |  cat   |
   * |   is   |
   * | coming |
   * +--------+
   * [4 rows x 1 columns]
   * \endcode
   *
   * Convert all zeros to missing values:
   * \code
   * auto  sa2 = gl_sarray({flex_vec{1, 0, 1},
   *                        flex_vec{1, 1, 1},
   *                        flex_vec{0, 1}});
   * std::cout <<  sa2.unpack("X", {flex_type_enum::INTEGER,
   *                                flex_type_enum::INTEGER,
   *                                flex_type_enum::INTEGER}, 0);
   * \endcode
   * Produces output:
   * \code{.txt}
   * Columns:
   *     X.0     int
   *     X.1     int
   *     X.2     int
   * Rows: 3
   * Data:
   * +------+------+------+
   * | X.0  | X.1  | X.2  |
   * +------+------+------+
   * |  1   | None |  1   |
   * |  1   |  1   |  1   |
   * | None |  1   | None |
   * +------+------+------+
   * [3 rows x 3 columns]
   * \endcode
   */
  gl_sframe unpack(const std::string& column_name_prefix = "X",
                   const std::vector<flex_type_enum>& column_types = std::vector<flex_type_enum>(),
                   const flexible_type& na_value = FLEX_UNDEFINED,
                   const std::vector<flexible_type>& limit = std::vector<flexible_type>()) const;

  /**
   * Sort all values in this \ref gl_sarray.  Sort only works for sarray of
   * type str, int and float, otherwise TypeError will be raised. Creates a
   * new, sorted \ref gl_sarray.
   *
   * \param ascending Optional. Defaults to True. If true, the sarray values
   *     are sorted in ascending order, otherwise, descending order.
   *
   * Example:
   * \code
   * auto sa = SArray({3,2,1});
   * std::cout <<  sa.sort();
   * \endcode
   *
   * Produces output:
   * \code{.txt}
   * dtype: int
   * Rows: 3
   * [1, 2, 3]
   * \endcode
   *
   */
  gl_sarray sort(bool ascending=true) const;


  /**
   *
   *  This returns an SArray with each element sliced accordingly to the
   *  slice specified.
   *
   *  \param start The start position of the slice
   *  \param stop The stop position of the slice
   *  \param step The step size of the slice (default = 1)
   *
   *  \return an SArray with each individual vector/string/list sliced
   *  according to the arguments.
   *
   *  This is conceptually equivalent to the python equivalent of:
   *  \code
   *     g.apply(lambda x: x[start:step:stop])
   *  \endcode
   *
   *  The SArray must be of type list, vector, or string.
   *
   *  For instance:
   *  \code
   *  g = SArray({"abcdef","qwerty"});
   *  std::cout << g.subslice(0, 2);
   *  \endcode
   *
   *  Produces output:
   *  \code{.txt}
   *  dtype: str
   *  Rows: 2
   *  ["ab", "qw"]
   *  \endcode
   *
   *  Negative indeices:
   *  \code
   *  std::cout << g.subslice(3,-1);
   *  \endcode
   *  Produces output:
   *  \code{.txt}
   *  dtype: str
   *  Rows: 2
   *  ["de", "rt"]
   *  \endcode
   *
   *  Arrays:
   *  \code
   *  g = SArray({{1,2,3}, {4,5,6}});
   *  std::cout << g.subslice(0, 1);
   *  \endcode
   *
   *  Produces output:
   *  \code{.txt}
   *  dtype: str
   *  Rows: 2
   *  [[1], [4]]
   *  \endcode
   */
  gl_sarray subslice(flexible_type start = FLEX_UNDEFINED,
                     flexible_type stop = FLEX_UNDEFINED,
                     flexible_type step = FLEX_UNDEFINED) const;

/**
 *
 *  An abstraction to perform cumulative aggregates.
 *    y <- x.cumulative_aggregate(f, w_0)
 *
 *  The abstraction is as follows:
 *    y[i+1], w[i+1] = func(x[i], w[i])
 *  where w[i] is some arbitary state.
 *
 * \param[in] Built in aggregate to use (e.g, sum, min, max etc.)
 * \return SArray
 *
 * \code
 *   sa = SArray([1, 2, 3, 4, 5])
 *   sa.cumulative_aggregate(std::make_shared<groupby_operators::sum>());
 * \endcode
 *
 * produces an SArray that looks like the following:
 *  dtype: int
 *  [1, 3, 6, 10, 15]
 * \endcode
 * \endcode
 *
 */
 gl_sarray cumulative_aggregate(
     std::shared_ptr<group_aggregate_value> aggregator) const;
 gl_sarray builtin_cumulative_aggregate(const std::string& name) const;

  /**
   *
   *  This returns an SArray where each element is a cumulative aggregate of
   *  all its previous elements. Only works in an SArray of numeric type or
   *  numeric-array types.
   *
   * \return an SArray
   *
   * \code
   *   sa = SArray([1, 2, 3, 4, 5])
   *   sa.cumulative_sum()
   * \endcode
   *
   * produces an SArray that looks like the following:
   *  dtype: int
   *  [1, 3, 6, 10, 15]
   * \endcode
   *
   */
  gl_sarray cumulative_sum() const;
  gl_sarray cumulative_min() const;
  gl_sarray cumulative_max() const;
  gl_sarray cumulative_var() const;
  gl_sarray cumulative_std() const;
  gl_sarray cumulative_avg() const;

  /**
   * Apply an aggregate function over a moving window.
   *
   * \param input The input SArray (expects to be materialized)
   * \param fn_name string representation of the aggregation function to use.
   * The mapping is the same string mapping used by the groupby aggregate
   * function.
   * \param window_start The start of the moving window relative to the current
   * value being calculated, inclusive. For example, 2 values behind the current
   * would be -2, and 0 indicates that the start of the window is the current
   * value.
   * \param window_end The end of the moving window relative to the current value
   * being calculated, inclusive. Must be greater than `window_start`. For
   * example, 0 would indicate that the current value is the end of the window,
   * and 2 would indicate that the window ends at 2 data values after the
   * current.
   * \param min_observations The minimum allowed number of non-NULL values in the
   * moving window for the emitted value to be non-NULL. size_t(-1) indicates
   * that all values must be non-NULL.
   *
   * Returns an SArray of the same length as the input, with a type that matches
   * the type output by the aggregation function.
   *
   * Throws an exception if:
   *  - window_end < window_start
   *  - The window size is excessively large (currently hardcoded to UINT_MAX).
   *  - The given function name corresponds to a function that will not operate
   *  on the data type of the input SArray.
   *  - The aggregation function returns more than one non-NULL types.
   *
   *  Example:
   *  \code
   *  gl_sarray a{0,1,2,3,4,5,6,7,8,9};
   *  // Moving window encompasses 3 values behind current and current value.
   *  auto result = a.rolling_apply(std::string("__builtin__avg__"), -3, 0);
   *  \endcode
   *
   *  Produces an SArray with these values:
   *  \code
   *  {NULL,NULL,NULL,1.5,2.5,3.5,4.5,5.5,6.5,7.5}
   *  \endcode
   */
  gl_sarray builtin_rolling_apply(const std::string &fn_name,
                                  ssize_t start,
                                  ssize_t end,
                                  size_t min_observations=size_t(-1)) const;

  /**
   * Show a visualization of the SArray.
   */
  void show(const std::string& path_to_client, const flexible_type& title, const flexible_type& xlabel, const flexible_type& ylabel) const;

  /**
   * Return a visualization of the SArray.
   */
  std::shared_ptr<visualization::Plot> plot(const flexible_type& title, const flexible_type& xlabel, const flexible_type& ylabel) const;

  /**
   * \internal
   * Gets the internal implementation object.
   */
  virtual std::shared_ptr<unity_sarray> get_proxy() const;

 private:
  void instantiate_new();

  void ensure_has_sarray_reader() const;

  std::shared_ptr<unity_sarray> m_sarray;

  mutable std::shared_ptr<sarray_reader<flexible_type> > m_sarray_reader;

}; // gl_sarray



gl_sarray operator+(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator-(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator*(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator/(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator<(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator>(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator<=(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator>=(const flexible_type& opnd, const gl_sarray& opnd2);
gl_sarray operator==(const flexible_type& opnd, const gl_sarray& opnd2);


/**
 * Provides printing of the gl_sarray.
 */
std::ostream& operator<<(std::ostream& out, const gl_sarray& other);







/**
 * \ingroup group_glsdk
 * A range object providing one pass iterators over part or all of a gl_sarray.
 *See \ref gl_sarray::range_iterator for usage examples.
 *
 * \see gl_sarray::range_iterator
 */
class gl_sarray_range {
 public:
  /// content type
  typedef flexible_type type;

  gl_sarray_range(std::shared_ptr<sarray_reader<flexible_type> > m_sarray_reader,
                  size_t start, size_t end);
  gl_sarray_range(const gl_sarray_range&) = default;
  gl_sarray_range(gl_sarray_range&&) = default;
  gl_sarray_range& operator=(const gl_sarray_range&) = default;
  gl_sarray_range& operator=(gl_sarray_range&&) = default;

  /// Iterator type
  struct iterator:
      public boost::iterator_facade<iterator,
                const flexible_type, boost::single_pass_traversal_tag> {
   public:
    iterator() = default;
    iterator(const iterator&) = default;
    iterator(iterator&&) = default;
    iterator& operator=(const iterator&) = default;
    iterator& operator=(iterator&&) = default;

    iterator(gl_sarray_range& range, bool is_start);
   private:
    friend class boost::iterator_core_access;
    void increment();
    void advance(size_t n);
    inline bool equal(const iterator& other) const {
      return m_counter == other.m_counter;
    }
    const type& dereference() const;
    size_t m_counter = 0;
    gl_sarray_range* m_owner = NULL;
  };

  /// const_iterator type
  typedef iterator const_iterator;

  /**
   * Returns an iterator to the start of the range.
   * Once the iterator is advanced, later calls to begin() have undefined
   * behavior.
   *
   * The returned iterator is invalidated once the parent range_iterator is
   * destroyed.
   */
  iterator begin();

  /**
   * Returns an iterator to the end of the range.
   *
   * The returned iterator is invalidated once the parent range_iterator is
   * destroyed.
   */
  iterator end();
 private:
  flexible_type m_current_value;
  std::shared_ptr<sarray_reader_buffer<flexible_type> > m_sarray_reader_buffer;
};


/**
 * Utility function to infer the most general type of an in memory vector of
 * flexible_types.
 */
flex_type_enum infer_type_of_list(const std::vector<flexible_type>& vec);


class gl_sarray_writer_impl;

/**
 * \ingroup group_glsdk
 * Provides the ability to write gl_sarrays.
 * The gl_sarray is internally cut into a collection of segments. Each segment
 * can be written to independently, and the resultant SArray is the effective
 * concatenation of all the segments.
 *
 * \code
 * // Writes an integer SArray of 4 segments.
 * gl_sarray_writer writer(flex_type_enum:INTEGER, 4);
 *
 * // for each segment, write a bunch of 10 values.
 * // segment 0 has 10 0's,
 * // segment 1 has 10 1's,
 * // etc
 * for (size_t seg = 0;seg < 4; ++seg) {
 *   for (size_t i = 0;i < 10; ++i) {
 *     writer.write(i, seg);
 *   }
 * }
 *
 * gl_sarray sa = writer.close();
 * // sa is now an SArray of 40 elements comprising of
 * // the sequence 10 0's, 10 1's, 10 2's, 10 3's
 * \endcode
 *
 * Different segments can be written safely in parallel. It is not safe to
 * write to the same segment simultanously.
 */
class gl_sarray_writer {
 public:
  /**
   * Constructs a writer to write an gl_sarray of a particular type.
   *
   * \param type The content type of the SArray. Everything written to the
   * writer (via \ref write) must be of that type, is implicitly castable to
   * that type, or is a missing value denoted with a FLEX_UNDEFINED value.
   *
   * \param num_segments Optional. The number of segments of the SArray.
   * Adjusting this parameter has little performance impact on the resultant
   * gl_sarray. Modifying this value is only helpful for providing writer
   * parallelism. Defaults to the number of cores on the machine.
   */
  gl_sarray_writer(flex_type_enum type, size_t num_segments = (size_t)(-1));

  /**
   * Writes a single value to a given segment.
   *
   * For instance,
   * \code
   * gl_sarray_writer writer(flex_type_enum:FLOAT, 1);
   * writer.write(1.5, 0); // writes the value 1.5 to segment 0
   * writer.write(1, 0); // writes the value 1.0 to segment 0 (integer can be cast to float)
   * \endcode
   *
   * Strings are the most general type and everything can cast to it. hence:
   * \code
   * gl_sarray_writer writer(flex_type_enum:STRING, 1);
   * writer.write("hello", 0); // writes the value "hello" to segment 0
   * writer.write(1.5, 0); // writes the value "1.5" to segment 0
   * writer.write(1, 0); // writes the value "1" to segment 0
   * \endcode
   *
   * Different segments can be written safely in parallel. It is not safe to
   * write to the same segment simultanously.
   *
   * \param f The value to write. This value should be of the requested type
   * (as set in the constructor), or is castable to the requested type, or is
   * FLEX_UNDEFINED.
   *
   * \param segmentid The segment to write to.
   */
  void write(const flexible_type& f, size_t segmentid);

  /**
   * Writes a range of values to a given segment.
   *
   * Essentially equivalent to:
   * \code
   * while(start != end) write(*start++);
   * \endcode
   *
   * Different segments can be written safely in parallel. It is not safe to
   * write to the same segment simultanously.
   *
   * \param start The start iterator of the range to write.
   *
   * \param end The end iterator of the range to write.
   *
   * \param segmentid The segment to write to.
   */
  template <typename T>
  void write(T begin, T end, size_t segmentid) {
    while (begin != end) {
      write((*begin), segmentid);
      ++begin;
    }
  }

  /**
   * Stops all writes and returns the resultant SArray.
   */
  gl_sarray close();

  /**
   * Returns the number of segments of the SArray; this is the same value
   * provided on construction of the writer.
   */
  size_t num_segments() const;

  ~gl_sarray_writer();

 private:
  std::unique_ptr<gl_sarray_writer_impl> m_writer_impl;

};

} // namespace turi
#endif