arrow 0.17.0

Author: jeroen

README.md

@@ -1,4 +1,4 @@
-# Apache Arrow 0.16.0
+# Apache Arrow 0.17.0
 
 Backports for the R legacy toolchain [lib-4.9.3](lib-4.9.3) built with [rtools-backports](https://github.com/r-windows/rtools-backports/blob/master/mingw-w64-arrow/PKGBUILD).
 
@@ -13,13 +13,13 @@ CXX_STD = CXX11
 PKG_LIBS = \
 	-L$(ARROW_LIBS) \
 	-lparquet -larrow_dataset -larrow \
-	-lthrift -lsnappy -lboost_regex-mt-s -lboost_system-mt-s -lz -lzstd -llz4 -lcrypto -lcrypt32
+	-lthrift -lsnappy -lz -lzstd -llz4 -lcrypto -lcrypt32
 ```
 
 To test this make sure you install the arrow package from a release tag:
 
 ```r
-remotes::install_github("apache/arrow/r@apache-arrow-0.16.0")
+remotes::install_github("apache/arrow/r@apache-arrow-0.17.0")
 ```
 
 To install R package from the arrow master branch you also would need to rebuild the master branch arrow C++ library from source.

include/arrow/api.h

@@ -17,8 +17,7 @@
 
 // Coarse public API while the library is in development
 
-#ifndef ARROW_API_H
-#define ARROW_API_H
+#pragma once
 
 #include "arrow/array.h"                    // IYWU pragma: export
 #include "arrow/array/concatenate.h"        // IYWU pragma: export
@@ -41,5 +40,3 @@
 
 /// \brief Top-level namespace for Apache Arrow C++ API
 namespace arrow {}
-
-#endif  // ARROW_API_H

include/arrow/array.h

@@ -92,7 +92,7 @@ class ARROW_EXPORT AdaptiveIntBuilderBase : public ArrayBuilder {
 
 class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveUIntBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+  explicit AdaptiveUIntBuilder(MemoryPool* pool = default_memory_pool());
 
   using ArrayBuilder::Advance;
   using internal::AdaptiveIntBuilderBase::Reset;
@@ -126,7 +126,7 @@ class ARROW_EXPORT AdaptiveUIntBuilder : public internal::AdaptiveIntBuilderBase
 
 class ARROW_EXPORT AdaptiveIntBuilder : public internal::AdaptiveIntBuilderBase {
  public:
-  explicit AdaptiveIntBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+  explicit AdaptiveIntBuilder(MemoryPool* pool = default_memory_pool());
 
   using ArrayBuilder::Advance;
   using internal::AdaptiveIntBuilderBase::Reset;

include/arrow/array/builder_adaptive.h

@@ -190,13 +190,16 @@ class ARROW_EXPORT ArrayBuilder {
     return Status::OK();
   }
 
-  static Status CheckCapacity(int64_t new_capacity, int64_t old_capacity) {
-    if (new_capacity < 0) {
-      return Status::Invalid("Resize capacity must be positive");
+  // Check the requested capacity for validity
+  Status CheckCapacity(int64_t new_capacity) {
+    if (ARROW_PREDICT_FALSE(new_capacity < 0)) {
+      return Status::Invalid(
+          "Resize capacity must be positive (requested: ", new_capacity, ")");
     }
 
-    if (new_capacity < old_capacity) {
-      return Status::Invalid("Resize cannot downsize");
+    if (ARROW_PREDICT_FALSE(new_capacity < length_)) {
+      return Status::Invalid("Resize cannot downsize (requested: ", new_capacity,
+                             ", current length: ", length_, ")");
     }
 
     return Status::OK();

include/arrow/array/builder_base.h

@@ -46,7 +46,7 @@ class BaseBinaryBuilder : public ArrayBuilder {
   using TypeClass = TYPE;
   using offset_type = typename TypeClass::offset_type;
 
-  explicit BaseBinaryBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
+  explicit BaseBinaryBuilder(MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool), offsets_builder_(pool), value_data_builder_(pool) {}
 
   BaseBinaryBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool)
@@ -235,7 +235,7 @@ class BaseBinaryBuilder : public ArrayBuilder {
       return Status::CapacityError("BinaryBuilder cannot reserve space for more than ",
                                    memory_limit(), " child elements, got ", capacity);
     }
-    ARROW_RETURN_NOT_OK(CheckCapacity(capacity, capacity_));
+    ARROW_RETURN_NOT_OK(CheckCapacity(capacity));
 
     // One more than requested for offsets
     ARROW_RETURN_NOT_OK(offsets_builder_.Resize(capacity + 1));
@@ -400,7 +400,7 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
   using TypeClass = FixedSizeBinaryType;
 
   explicit FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
-                                  MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+                                  MemoryPool* pool = default_memory_pool());
 
   Status Append(const uint8_t* value) {
     ARROW_RETURN_NOT_OK(Reserve(1));
@@ -515,10 +515,10 @@ namespace internal {
 class ARROW_EXPORT ChunkedBinaryBuilder {
  public:
   explicit ChunkedBinaryBuilder(int32_t max_chunk_value_length,
-                                MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+                                MemoryPool* pool = default_memory_pool());
 
   ChunkedBinaryBuilder(int32_t max_chunk_value_length, int32_t max_chunk_length,
-                       MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+                       MemoryPool* pool = default_memory_pool());
 
   virtual ~ChunkedBinaryBuilder() = default;
 

include/arrow/array/builder_binary.h

@@ -31,7 +31,7 @@ class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
   using TypeClass = Decimal128Type;
 
   explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
-                             MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);
+                             MemoryPool* pool = default_memory_pool());
 
   using FixedSizeBinaryBuilder::Append;
   using FixedSizeBinaryBuilder::AppendValues;

include/arrow/array/builder_decimal.h

@@ -55,25 +55,24 @@ struct DictionaryScalar<FixedSizeBinaryType> {
 
 class ARROW_EXPORT DictionaryMemoTable {
  public:
-  explicit DictionaryMemoTable(const std::shared_ptr<DataType>& type);
-  explicit DictionaryMemoTable(const std::shared_ptr<Array>& dictionary);
+  DictionaryMemoTable(MemoryPool* pool, const std::shared_ptr<DataType>& type);
+  DictionaryMemoTable(MemoryPool* pool, const std::shared_ptr<Array>& dictionary);
   ~DictionaryMemoTable();
 
-  int32_t GetOrInsert(const bool& value);
-  int32_t GetOrInsert(const int8_t& value);
-  int32_t GetOrInsert(const int16_t& value);
-  int32_t GetOrInsert(const int32_t& value);
-  int32_t GetOrInsert(const int64_t& value);
-  int32_t GetOrInsert(const uint8_t& value);
-  int32_t GetOrInsert(const uint16_t& value);
-  int32_t GetOrInsert(const uint32_t& value);
-  int32_t GetOrInsert(const uint64_t& value);
-  int32_t GetOrInsert(const float& value);
-  int32_t GetOrInsert(const double& value);
-  int32_t GetOrInsert(const util::string_view& value);
-
-  Status GetArrayData(MemoryPool* pool, int64_t start_offset,
-                      std::shared_ptr<ArrayData>* out);
+  Status GetOrInsert(bool value, int32_t* out);
+  Status GetOrInsert(int8_t value, int32_t* out);
+  Status GetOrInsert(int16_t value, int32_t* out);
+  Status GetOrInsert(int32_t value, int32_t* out);
+  Status GetOrInsert(int64_t value, int32_t* out);
+  Status GetOrInsert(uint8_t value, int32_t* out);
+  Status GetOrInsert(uint16_t value, int32_t* out);
+  Status GetOrInsert(uint32_t value, int32_t* out);
+  Status GetOrInsert(uint64_t value, int32_t* out);
+  Status GetOrInsert(float value, int32_t* out);
+  Status GetOrInsert(double value, int32_t* out);
+  Status GetOrInsert(util::string_view value, int32_t* out);
+
+  Status GetArrayData(int64_t start_offset, std::shared_ptr<ArrayData>* out);
 
   /// \brief Insert new memo values
   Status InsertValues(const Array& values);
@@ -103,7 +102,7 @@ class DictionaryBuilderBase : public ArrayBuilder {
                             value_type,
                         MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool),
-        memo_table_(new internal::DictionaryMemoTable(value_type)),
+        memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
         delta_offset_(0),
         byte_width_(-1),
         indices_builder_(pool),
@@ -114,7 +113,7 @@ class DictionaryBuilderBase : public ArrayBuilder {
       enable_if_fixed_size_binary<T1, const std::shared_ptr<DataType>&> value_type,
       MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool),
-        memo_table_(new internal::DictionaryMemoTable(value_type)),
+        memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
         delta_offset_(0),
         byte_width_(static_cast<const T1&>(*value_type).byte_width()),
         indices_builder_(pool),
@@ -125,10 +124,11 @@ class DictionaryBuilderBase : public ArrayBuilder {
       enable_if_parameter_free<T1, MemoryPool*> pool = default_memory_pool())
       : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}
 
+  // This constructor doesn't check for errors. Use InsertMemoValues instead.
   DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
                         MemoryPool* pool = default_memory_pool())
       : ArrayBuilder(pool),
-        memo_table_(new internal::DictionaryMemoTable(dictionary)),
+        memo_table_(new internal::DictionaryMemoTable(pool, dictionary)),
         delta_offset_(0),
         byte_width_(-1),
         indices_builder_(pool),
@@ -143,7 +143,8 @@ class DictionaryBuilderBase : public ArrayBuilder {
   Status Append(const Scalar& value) {
     ARROW_RETURN_NOT_OK(Reserve(1));
 
-    auto memo_index = memo_table_->GetOrInsert(value);
+    int32_t memo_index;
+    ARROW_RETURN_NOT_OK(memo_table_->GetOrInsert(value, &memo_index));
     ARROW_RETURN_NOT_OK(indices_builder_.Append(memo_index));
     length_ += 1;
 
@@ -162,6 +163,24 @@ class DictionaryBuilderBase : public ArrayBuilder {
     return Append(util::string_view(value, byte_width_));
   }
 
+  /// \brief Append a string (only for binary types)
+  template <typename T1 = T>
+  enable_if_binary_like<T1, Status> Append(const uint8_t* value, int32_t length) {
+    return Append(reinterpret_cast<const char*>(value), length);
+  }
+
+  /// \brief Append a string (only for binary types)
+  template <typename T1 = T>
+  enable_if_binary_like<T1, Status> Append(const char* value, int32_t length) {
+    return Append(util::string_view(value, length));
+  }
+
+  /// \brief Append a string (only for string types)
+  template <typename T1 = T>
+  enable_if_string_like<T1, Status> Append(const char* value, int32_t length) {
+    return Append(util::string_view(value, length));
+  }
+
   /// \brief Append a scalar null value
   Status AppendNull() final {
     length_ += 1;
@@ -231,11 +250,11 @@ class DictionaryBuilderBase : public ArrayBuilder {
   /// \brief Reset and also clear accumulated dictionary values in memo table
   void ResetFull() {
     Reset();
-    memo_table_.reset(new internal::DictionaryMemoTable(value_type_));
+    memo_table_.reset(new internal::DictionaryMemoTable(pool_, value_type_));
   }
 
   Status Resize(int64_t capacity) override {
-    ARROW_RETURN_NOT_OK(CheckCapacity(capacity, capacity_));
+    ARROW_RETURN_NOT_OK(CheckCapacity(capacity));
     capacity = std::max(capacity, kMinBuilderCapacity);
     ARROW_RETURN_NOT_OK(indices_builder_.Resize(capacity));
     capacity_ = indices_builder_.capacity();
@@ -282,7 +301,7 @@ class DictionaryBuilderBase : public ArrayBuilder {
 
     // Generate dictionary array from hash table contents
     std::shared_ptr<ArrayData> dictionary_data;
-    ARROW_RETURN_NOT_OK(memo_table_->GetArrayData(pool_, dict_offset, &dictionary_data));
+    ARROW_RETURN_NOT_OK(memo_table_->GetArrayData(dict_offset, &dictionary_data));
 
     *out_dictionary = MakeArray(dictionary_data);
     delta_offset_ = memo_table_->size();
@@ -343,7 +362,7 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
   }
 
   Status Resize(int64_t capacity) override {
-    ARROW_RETURN_NOT_OK(CheckCapacity(capacity, capacity_));
+    ARROW_RETURN_NOT_OK(CheckCapacity(capacity));
     capacity = std::max(capacity, kMinBuilderCapacity);
 
     ARROW_RETURN_NOT_OK(indices_builder_.Resize(capacity));
@@ -421,71 +440,13 @@ class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder,
 };
 
 // ----------------------------------------------------------------------
-// Binary / Unicode builders with slightly expanded APIs
-
-namespace internal {
-
-template <typename T>
-class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
- public:
-  using BASE = DictionaryBuilder<T>;
-  using BASE::Append;
-  using BASE::AppendIndices;
-  using BASE::BASE;
-
-  BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}
-
-  Status Append(const uint8_t* value, int32_t length) {
-    return Append(reinterpret_cast<const char*>(value), length);
-  }
-
-  Status Append(const char* value, int32_t length) {
-    return Append(util::string_view(value, length));
-  }
-};
-
-template <typename T>
-class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
- public:
-  using BASE = Dictionary32Builder<T>;
-  using BASE::Append;
-  using BASE::AppendIndices;
-  using BASE::BASE;
-
-  BinaryDictionary32BuilderImpl()
-      : BinaryDictionary32BuilderImpl(default_memory_pool()) {}
-
-  Status Append(const uint8_t* value, int32_t length) {
-    return Append(reinterpret_cast<const char*>(value), length);
-  }
-
-  Status Append(const char* value, int32_t length) {
-    return Append(util::string_view(value, length));
-  }
-};
-
-}  // namespace internal
-
-class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
-  using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
-  using BASE::BASE;
-};
-
-class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
-  using BASE = BinaryDictionaryBuilderImpl<StringType>;
-  using BASE::BASE;
-};
-
-class BinaryDictionary32Builder
-    : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
-  using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
-  using BASE::BASE;
-};
-
-class StringDictionary32Builder
-    : public internal::BinaryDictionary32BuilderImpl<StringType> {
-  using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
-  using BASE::BASE;
-};
+// Binary / Unicode builders
+// (compatibility aliases; those used to be derived classes with additional
+//  Append() overloads, but they have been folded into DictionaryBuilderBase)
+
+using BinaryDictionaryBuilder = DictionaryBuilder<BinaryType>;
+using StringDictionaryBuilder = DictionaryBuilder<StringType>;
+using BinaryDictionary32Builder = Dictionary32Builder<BinaryType>;
+using StringDictionary32Builder = Dictionary32Builder<StringType>;
 
 }  // namespace arrow

include/arrow/array/builder_dict.h

@@ -54,9 +54,9 @@ class BaseListBuilder : public ArrayBuilder {
       return Status::CapacityError("List array cannot reserve space for more than ",
                                    maximum_elements(), " got ", capacity);
     }
-    ARROW_RETURN_NOT_OK(CheckCapacity(capacity, capacity_));
+    ARROW_RETURN_NOT_OK(CheckCapacity(capacity));
 
-    // one more then requested for offsets
+    // One more than requested for offsets
     ARROW_RETURN_NOT_OK(offsets_builder_.Resize(capacity + 1));
     return ArrayBuilder::Resize(capacity);
   }
@@ -162,7 +162,7 @@ class BaseListBuilder : public ArrayBuilder {
 /// To use this class, you must append values to the child array builder and use
 /// the Append function to delimit each distinct list value (once the values
 /// have been appended to the child array) or use the bulk API to append
-/// a sequence of offests and null values.
+/// a sequence of offsets and null values.
 ///
 /// A note on types.  Per arrow/type.h all types in the c++ implementation are
 /// logical so even though this class always builds list array, this can
@@ -203,7 +203,7 @@ class ARROW_EXPORT LargeListBuilder : public BaseListBuilder<LargeListType> {
 ///
 /// To use this class, you must append values to the key and item array builders
 /// and use the Append function to delimit each distinct map (once the keys and items
-/// have been appended) or use the bulk API to append a sequence of offests and null
+/// have been appended) or use the bulk API to append a sequence of offsets and null
 /// maps.
 ///
 /// Key uniqueness and ordering are not validated.