feat!(datafusion): enable parallel file scanning with eager task bucketing

crates/iceberg/public-api.txt

@@ -86,6 +86,11 @@ pub fn iceberg::arrow::PartitionValueCalculator::partition_type(&self) -> &icebe
 pub fn iceberg::arrow::PartitionValueCalculator::try_new(partition_spec: &iceberg::spec::PartitionSpec, table_schema: &iceberg::spec::Schema) -> iceberg::Result<Self>
 impl core::fmt::Debug for iceberg::arrow::PartitionValueCalculator
 pub fn iceberg::arrow::PartitionValueCalculator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+pub struct iceberg::arrow::PrimitiveLiteralArrayBuilder
+impl iceberg::arrow::PrimitiveLiteralArrayBuilder
+pub fn iceberg::arrow::PrimitiveLiteralArrayBuilder::append_or_null(&mut self, prim_lit: core::option::Option<&iceberg::spec::PrimitiveLiteral>) -> iceberg::Result<bool>
+pub fn iceberg::arrow::PrimitiveLiteralArrayBuilder::finish(self) -> iceberg::Result<arrow_array::array::ArrayRef>
+pub fn iceberg::arrow::PrimitiveLiteralArrayBuilder::try_new(data_type: &arrow_schema::datatype::DataType, capacity: usize) -> iceberg::Result<Self>
 pub struct iceberg::arrow::RecordBatchPartitionSplitter
 impl iceberg::arrow::RecordBatchPartitionSplitter
 pub fn iceberg::arrow::RecordBatchPartitionSplitter::split(&self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<alloc::vec::Vec<(iceberg::spec::PartitionKey, arrow_array::record_batch::RecordBatch)>>
@@ -127,6 +132,7 @@ pub fn iceberg::arrow::arrow_schema_to_schema(schema: &arrow_schema::schema::Sch
 pub fn iceberg::arrow::arrow_schema_to_schema_auto_assign_ids(schema: &arrow_schema::schema::Schema) -> iceberg::Result<iceberg::spec::Schema>
 pub fn iceberg::arrow::arrow_struct_to_literal(struct_array: &arrow_array::array::ArrayRef, ty: &iceberg::spec::StructType) -> iceberg::Result<alloc::vec::Vec<core::option::Option<iceberg::spec::Literal>>>
 pub fn iceberg::arrow::arrow_type_to_type(ty: &arrow_schema::datatype::DataType) -> iceberg::Result<iceberg::spec::Type>
+pub fn iceberg::arrow::create_primitive_array_single_element(data_type: &arrow_schema::datatype::DataType, prim_lit: &core::option::Option<iceberg::spec::PrimitiveLiteral>) -> iceberg::Result<arrow_array::array::ArrayRef>
 pub fn iceberg::arrow::datum_to_arrow_type_with_ree(datum: &iceberg::spec::Datum) -> arrow_schema::datatype::DataType
 pub fn iceberg::arrow::schema_to_arrow_schema(schema: &iceberg::spec::Schema) -> iceberg::Result<arrow_schema::schema::Schema>
 pub fn iceberg::arrow::strip_metadata_from_schema(schema: &arrow_schema::schema::Schema) -> iceberg::Result<arrow_schema::schema::Schema>
@@ -1289,6 +1295,7 @@ pub fn iceberg::scan::TableScan::column_names(&self) -> core::option::Option<&[a
 pub async fn iceberg::scan::TableScan::plan_files(&self) -> iceberg::Result<iceberg::scan::FileScanTaskStream>
 pub fn iceberg::scan::TableScan::snapshot(&self) -> core::option::Option<&iceberg::spec::SnapshotRef>
 pub async fn iceberg::scan::TableScan::to_arrow(&self) -> iceberg::Result<iceberg::scan::ArrowRecordBatchStream>
+pub fn iceberg::scan::TableScan::to_arrow_from_tasks(&self, tasks: iceberg::scan::FileScanTaskStream) -> iceberg::Result<iceberg::scan::ArrowRecordBatchStream>
 impl core::fmt::Debug for iceberg::scan::TableScan
 pub fn iceberg::scan::TableScan::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
 pub struct iceberg::scan::TableScanBuilder<'a>
@@ -3034,6 +3041,7 @@ pub fn iceberg::table::Table::metadata_location_result(&self) -> iceberg::Result
 pub fn iceberg::table::Table::metadata_ref(&self) -> iceberg::spec::TableMetadataRef
 pub fn iceberg::table::Table::reader_builder(&self) -> iceberg::arrow::ArrowReaderBuilder
 pub fn iceberg::table::Table::readonly(&self) -> bool
+pub fn iceberg::table::Table::runtime(&self) -> &iceberg::Runtime
 pub fn iceberg::table::Table::scan(&self) -> iceberg::scan::TableScanBuilder<'_>
 impl core::clone::Clone for iceberg::table::Table
 pub fn iceberg::table::Table::clone(&self) -> iceberg::table::Table
@@ -3094,6 +3102,7 @@ pub mod iceberg::util
 pub mod iceberg::util::snapshot
 pub fn iceberg::util::snapshot::ancestors_between(table_metadata: &iceberg::spec::TableMetadataRef, latest_snapshot_id: i64, oldest_snapshot_id: core::option::Option<i64>) -> impl core::iter::traits::iterator::Iterator<Item = iceberg::spec::SnapshotRef> + core::marker::Send
 pub fn iceberg::util::snapshot::ancestors_of(table_metadata: &iceberg::spec::TableMetadataRef, snapshot_id: i64) -> impl core::iter::traits::iterator::Iterator<Item = iceberg::spec::SnapshotRef> + core::marker::Send
+pub fn iceberg::util::available_parallelism() -> core::num::nonzero::NonZeroUsize
 pub mod iceberg::writer
 pub mod iceberg::writer::base_writer
 pub mod iceberg::writer::base_writer::data_file_writer