arrow 0.15.1

Author: jeroen

README.md

@@ -1,4 +1,4 @@
-# Apache Arrow 0.15.0
+# Apache Arrow 0.15.1
 
 Backports for the R legacy toolchain [lib-4.9.3](lib-4.9.3) built with [rtools-backports](https://github.com/r-windows/rtools-backports/blob/master/mingw-w64-arrow/PKGBUILD).
 
@@ -21,7 +21,7 @@ PKG_LIBS = \
 To test this make sure you install the arrow package from a release tag:
 
 ```r
-remotes::install_github("apache/arrow/[email protected].0")
+remotes::install_github("apache/arrow/[email protected].1")
 ```
 
 To install R package from the arrow master branch you also would need to rebuild the master branch arrow C++ library from source.

include/arrow/array/builder_dict.h

@@ -89,11 +89,7 @@ class ARROW_EXPORT DictionaryMemoTable {
 /// dense array
 ///
 /// Unlike other builders, dictionary builder does not completely
-/// reset the state on Finish calls. The arrays built after the
-/// initial Finish call will reuse the previously created encoding and
-/// build a delta dictionary when new terms occur.
-///
-/// data
+/// reset the state on Finish calls.
 template <typename BuilderType, typename T>
 class DictionaryBuilderBase : public ArrayBuilder {
  public:
@@ -230,44 +226,34 @@ class DictionaryBuilderBase : public ArrayBuilder {
   }
 
   void Reset() override {
+    // Perform a partial reset. Call ResetFull to also reset the accumulated
+    // dictionary values
     ArrayBuilder::Reset();
     indices_builder_.Reset();
+  }
+
+  /// \brief Reset and also clear accumulated dictionary values in memo table
+  void ResetFull() {
+    Reset();
     memo_table_.reset(new internal::DictionaryMemoTable(value_type_));
-    delta_offset_ = 0;
   }
 
   Status Resize(int64_t capacity) override {
     ARROW_RETURN_NOT_OK(CheckCapacity(capacity, capacity_));
     capacity = std::max(capacity, kMinBuilderCapacity);
-
-    if (capacity_ == 0) {
-      // Initialize hash table
-      // XXX should we let the user pass additional size heuristics?
-      delta_offset_ = 0;
-    }
     ARROW_RETURN_NOT_OK(indices_builder_.Resize(capacity));
     capacity_ = indices_builder_.capacity();
     return Status::OK();
   }
 
-  Status FinishInternal(std::shared_ptr<ArrayData>* out) override {
-    // Finalize indices array
-    ARROW_RETURN_NOT_OK(indices_builder_.FinishInternal(out));
-
-    // Generate dictionary array from hash table contents
-    std::shared_ptr<ArrayData> dictionary_data;
-
-    ARROW_RETURN_NOT_OK(
-        memo_table_->GetArrayData(pool_, delta_offset_, &dictionary_data));
-
-    // Set type of array data to the right dictionary type
-    (*out)->type = type();
-    (*out)->dictionary = MakeArray(dictionary_data);
-
-    // Update internals for further uses of this DictionaryBuilder
-    delta_offset_ = memo_table_->size();
-    indices_builder_.Reset();
-
+  /// \brief Return dictionary indices and a delta dictionary since the last
+  /// time that Finish or FinishDelta were called, and reset state of builder
+  /// (except the memo table)
+  Status FinishDelta(std::shared_ptr<Array>* out_indices,
+                     std::shared_ptr<Array>* out_delta) {
+    std::shared_ptr<ArrayData> indices_data;
+    ARROW_RETURN_NOT_OK(FinishWithDictOffset(delta_offset_, &indices_data, out_delta));
+    *out_indices = MakeArray(indices_data);
     return Status::OK();
   }
 
@@ -277,17 +263,45 @@ class DictionaryBuilderBase : public ArrayBuilder {
 
   Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }
 
-  /// is the dictionary builder in the delta building mode
-  bool is_building_delta() { return delta_offset_ > 0; }
-
   std::shared_ptr<DataType> type() const override {
     return ::arrow::dictionary(indices_builder_.type(), value_type_);
   }
 
  protected:
+  Status FinishInternal(std::shared_ptr<ArrayData>* out) override {
+    std::shared_ptr<Array> dictionary;
+    ARROW_RETURN_NOT_OK(FinishWithDictOffset(/*offset=*/0, out, &dictionary));
+
+    // Set type of array data to the right dictionary type
+    (*out)->type = type();
+    (*out)->dictionary = dictionary;
+    return Status::OK();
+  }
+
+  Status FinishWithDictOffset(int64_t dict_offset,
+                              std::shared_ptr<ArrayData>* out_indices,
+                              std::shared_ptr<Array>* out_dictionary) {
+    // Finalize indices array
+    ARROW_RETURN_NOT_OK(indices_builder_.FinishInternal(out_indices));
+
+    // Generate dictionary array from hash table contents
+    std::shared_ptr<ArrayData> dictionary_data;
+    ARROW_RETURN_NOT_OK(memo_table_->GetArrayData(pool_, dict_offset, &dictionary_data));
+
+    *out_dictionary = MakeArray(dictionary_data);
+    delta_offset_ = memo_table_->size();
+
+    // Update internals for further uses of this DictionaryBuilder
+    ArrayBuilder::Reset();
+    return Status::OK();
+  }
+
   std::unique_ptr<DictionaryMemoTable> memo_table_;
 
+  // The size of the dictionary memo at last invocation of Finish, to use in
+  // FinishDelta for computing dictionary deltas
   int32_t delta_offset_;
+
   // Only used for FixedSizeBinaryType
   int32_t byte_width_;
 
@@ -380,6 +394,7 @@ class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuil
                        const uint8_t* valid_bytes = NULLPTR) {
     int64_t null_count_before = this->indices_builder_.null_count();
     ARROW_RETURN_NOT_OK(this->indices_builder_.AppendValues(values, length, valid_bytes));
+    this->capacity_ = this->indices_builder_.capacity();
     this->length_ += length;
     this->null_count_ += this->indices_builder_.null_count() - null_count_before;
     return Status::OK();
@@ -402,6 +417,7 @@ class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder,
                        const uint8_t* valid_bytes = NULLPTR) {
     int64_t null_count_before = this->indices_builder_.null_count();
     ARROW_RETURN_NOT_OK(this->indices_builder_.AppendValues(values, length, valid_bytes));
+    this->capacity_ = this->indices_builder_.capacity();
     this->length_ += length;
     this->null_count_ += this->indices_builder_.null_count() - null_count_before;
     return Status::OK();

include/arrow/io/compressed.h

@@ -62,6 +62,9 @@ class ARROW_EXPORT CompressedOutputStream : public OutputStream {
   Status Tell(int64_t* position) const override;
 
   Status Write(const void* data, int64_t nbytes) override;
+  /// \cond FALSE
+  using Writable::Write;
+  /// \endcond
   Status Flush() override;
 
   /// \brief Return the underlying raw output stream.

include/arrow/io/file.h

@@ -230,6 +230,9 @@ class ARROW_EXPORT MemoryMappedFile : public ReadWriteFileInterface {
 
   /// Write data at the current position in the file. Thread-safe
   Status Write(const void* data, int64_t nbytes) override;
+  /// \cond FALSE
+  using Writable::Write;
+  /// \endcond
 
   /// Set the size of the map to new_size.
   Status Resize(int64_t new_size);

include/arrow/io/memory.h

@@ -104,6 +104,9 @@ class ARROW_EXPORT MockOutputStream : public OutputStream {
   bool closed() const override;
   Status Tell(int64_t* position) const override;
   Status Write(const void* data, int64_t nbytes) override;
+  /// \cond FALSE
+  using Writable::Write;
+  /// \endcond
 
   int64_t GetExtentBytesWritten() const { return extent_bytes_written_; }
 
@@ -124,6 +127,10 @@ class ARROW_EXPORT FixedSizeBufferWriter : public WritableFile {
   Status Seek(int64_t position) override;
   Status Tell(int64_t* position) const override;
   Status Write(const void* data, int64_t nbytes) override;
+  /// \cond FALSE
+  using Writable::Write;
+  /// \endcond
+
   Status WriteAt(int64_t position, const void* data, int64_t nbytes) override;
 
   void set_memcopy_threads(int num_threads);

include/arrow/json/chunked_builder.h

@@ -70,7 +70,7 @@ class ARROW_EXPORT ChunkedArrayBuilder {
 ARROW_EXPORT Status MakeChunkedArrayBuilder(
     const std::shared_ptr<internal::TaskGroup>& task_group, MemoryPool* pool,
     const PromotionGraph* promotion_graph, const std::shared_ptr<DataType>& type,
-    std::unique_ptr<ChunkedArrayBuilder>* out);
+    std::shared_ptr<ChunkedArrayBuilder>* out);
 
 }  // namespace json
 }  // namespace arrow

include/arrow/json/chunker.h

@@ -41,23 +41,48 @@ class ARROW_EXPORT Chunker {
   virtual ~Chunker() = default;
 
   /// \brief Carve up a chunk in a block of data to contain only whole objects
+  ///
+  /// Post-conditions:
+  /// - block == whole + partial
+  /// - `whole` is a valid block of JSON data
+  /// - `partial` doesn't contain an entire JSON object
+  ///
   /// \param[in] block json data to be chunked
   /// \param[out] whole subrange of block containing whole json objects
   /// \param[out] partial subrange of block a partial json object
-  virtual Status Process(const std::shared_ptr<Buffer>& block,
-                         std::shared_ptr<Buffer>* whole,
+  virtual Status Process(std::shared_ptr<Buffer> block, std::shared_ptr<Buffer>* whole,
                          std::shared_ptr<Buffer>* partial) = 0;
 
   /// \brief Carve the completion of a partial object out of a block
+  ///
+  /// Post-conditions:
+  /// - block == completion + rest
+  /// - `partial + completion` is a valid block of JSON data
+  /// - `completion` doesn't contain an entire JSON object
+  ///
   /// \param[in] partial incomplete json object
   /// \param[in] block json data
   /// \param[out] completion subrange of block containing the completion of partial
   /// \param[out] rest subrange of block containing what completion does not cover
-  virtual Status ProcessWithPartial(const std::shared_ptr<Buffer>& partial,
-                                    const std::shared_ptr<Buffer>& block,
+  virtual Status ProcessWithPartial(std::shared_ptr<Buffer> partial,
+                                    std::shared_ptr<Buffer> block,
                                     std::shared_ptr<Buffer>* completion,
                                     std::shared_ptr<Buffer>* rest) = 0;
 
+  /// \brief Like ProcessWithPartial, but for the lastblock of a file
+  ///
+  /// This method allows for a final JSON object without a trailing newline
+  /// (ProcessWithPartial would return an error in that case).
+  ///
+  /// Post-conditions:
+  /// - block == completion + rest
+  /// - `partial + completion` is a valid block of JSON data
+  /// - `completion` doesn't contain an entire JSON object
+  virtual Status ProcessFinal(std::shared_ptr<Buffer> partial,
+                              std::shared_ptr<Buffer> block,
+                              std::shared_ptr<Buffer>* completion,
+                              std::shared_ptr<Buffer>* rest) = 0;
+
   static std::unique_ptr<Chunker> Make(const ParseOptions& options);
 
  protected:

include/arrow/json/options.h

@@ -46,8 +46,7 @@ struct ARROW_EXPORT ParseOptions {
 
   /// Whether objects may be printed across multiple lines (for example pretty-printed)
   ///
-  /// If true, parsing may be slower
-  /// If false, input must end with an empty line
+  /// If true, parsing may be slower.
   bool newlines_in_values = false;
 
   /// How JSON fields outside of explicit_schema (if given) are treated

include/arrow/memory_pool.h

@@ -160,6 +160,17 @@ ARROW_EXPORT MemoryPool* system_memory_pool();
 /// May return NotImplemented if jemalloc is not available.
 ARROW_EXPORT Status jemalloc_memory_pool(MemoryPool** out);
 
+/// \brief Set jemalloc memory page purging behavior for future-created arenas
+/// to the indicated number of milliseconds. See dirty_decay_ms and
+/// muzzy_decay_ms options in jemalloc for a description of what these do. The
+/// default is configured to 1000 (1 second) which releases memory more
+/// aggressively to the operating system than the jemalloc default of 10
+/// seconds. If you set the value to 0, dirty / muzzy pages will be released
+/// immediately rather than with a time decay, but this may reduce application
+/// performance.
+ARROW_EXPORT
+Status jemalloc_set_decay_ms(int ms);
+
 /// Return a process-wide memory pool based on mimalloc.
 ///
 /// May return NotImplemented if mimalloc is not available.

include/arrow/util/config.h

@@ -17,8 +17,8 @@
 
 #define ARROW_VERSION_MAJOR 0
 #define ARROW_VERSION_MINOR 15
-#define ARROW_VERSION_PATCH 0
+#define ARROW_VERSION_PATCH 1
 #define ARROW_VERSION ((ARROW_VERSION_MAJOR * 1000) + ARROW_VERSION_MINOR) * 1000 + ARROW_VERSION_PATCH
 
-/* #undef DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY */
+#define DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY
 /* #undef GRPCPP_PP_INCLUDE */