Remove all_valid/all_invalid from Validity

encodings/alp/src/alp/compute/mask.rs

@@ -32,7 +32,7 @@ impl MaskKernel for ALP {
         mask: &ArrayRef,
         ctx: &mut ExecutionCtx,
     ) -> VortexResult<Option<ArrayRef>> {
-        let vortex_mask = Validity::Array(mask.not()?).to_mask(array.len());
+        let vortex_mask = Validity::Array(mask.not()?).execute_mask(array.len(), ctx)?;
         let masked_encoded = array.encoded().clone().mask(mask.clone())?;
         let masked_patches = array
             .patches()

encodings/datetime-parts/src/canonical.rs

@@ -147,19 +147,17 @@ mod test {
         ))
         .unwrap();
 
-        assert_eq!(
-            date_times.validity_mask().unwrap(),
-            validity.to_mask(date_times.len())
-        );
-
         let mut ctx = ExecutionCtx::new(VortexSession::empty());
+
+        assert!(date_times.validity()?.mask_eq(&validity, &mut ctx)?);
+
         let primitive_values = decode_to_temporal(&date_times, &mut ctx)?
             .temporal_values()
             .clone()
             .execute::<PrimitiveArray>(&mut ctx)?;
 
         assert_arrays_eq!(primitive_values, milliseconds);
-        assert_eq!(primitive_values.validity(), &validity);
+        assert!(primitive_values.validity().mask_eq(&validity, &mut ctx)?);
         Ok(())
     }
 }

encodings/datetime-parts/src/compress.rs

@@ -77,7 +77,9 @@ impl TryFrom<TemporalArray> for DateTimePartsArray {
 mod tests {
     use rstest::rstest;
     use vortex_array::IntoArray;
+    use vortex_array::LEGACY_SESSION;
     use vortex_array::ToCanonical;
+    use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::PrimitiveArray;
     use vortex_array::arrays::TemporalArray;
     use vortex_array::extension::datetime::TimeUnit;
@@ -110,8 +112,21 @@ mod tests {
             seconds,
             subseconds,
         } = split_temporal(temporal_array).unwrap();
-        assert_eq!(days.to_primitive().validity(), &validity);
-        assert_eq!(seconds.to_primitive().validity(), &Validity::NonNullable);
-        assert_eq!(subseconds.to_primitive().validity(), &Validity::NonNullable);
+
+        let mut ctx = LEGACY_SESSION.create_execution_ctx();
+        assert!(
+            days.to_primitive()
+                .validity()
+                .mask_eq(&validity, &mut ctx)
+                .unwrap()
+        );
+        assert!(matches!(
+            seconds.to_primitive().validity(),
+            Validity::NonNullable
+        ));
+        assert!(matches!(
+            subseconds.to_primitive().validity(),
+            Validity::NonNullable
+        ));
     }
 }

encodings/fastlanes/src/rle/array/mod.rs

@@ -218,7 +218,9 @@ mod tests {
     use vortex_array::ArrayContext;
     use vortex_array::DynArray;
     use vortex_array::IntoArray;
+    use vortex_array::LEGACY_SESSION;
     use vortex_array::ToCanonical;
+    use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::PrimitiveArray;
     use vortex_array::assert_arrays_eq;
     use vortex_array::dtype::DType;
@@ -397,7 +399,12 @@ mod tests {
         let sliced_array = rle_array.slice(1..4).unwrap();
         let validity_mask = sliced_array.validity_mask().unwrap();
 
-        let expected_mask = Validity::from_iter([false, true, false]).to_mask(3);
+        let mut ctx = LEGACY_SESSION.create_execution_ctx();
+        let expected_mask = Validity::from_iter([false, true, false])
+            .execute_mask(3, &mut ctx)
+            .unwrap();
+        assert_eq!(validity_mask.len(), expected_mask.len());
+        assert_eq!(validity_mask, expected_mask);
         assert_eq!(validity_mask.len(), expected_mask.len());
         assert_eq!(validity_mask, expected_mask);
     }

encodings/pco/public-api.lock

@@ -46,7 +46,7 @@ pub fn vortex_pco::Pco::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::
 
 pub fn vortex_pco::Pco::dtype(array: &vortex_pco::PcoArray) -> &vortex_array::dtype::DType
 
-pub fn vortex_pco::Pco::execute(array: &Self::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
+pub fn vortex_pco::Pco::execute(array: &Self::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionStep>
 
 pub fn vortex_pco::Pco::id(_array: &Self::Array) -> vortex_array::vtable::dyn_::ArrayId
 
@@ -74,7 +74,7 @@ pub struct vortex_pco::PcoArray
 
 impl vortex_pco::PcoArray
 
-pub fn vortex_pco::PcoArray::decompress(&self) -> vortex_error::VortexResult<vortex_array::arrays::primitive::array::PrimitiveArray>
+pub fn vortex_pco::PcoArray::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::arrays::primitive::array::PrimitiveArray>
 
 pub fn vortex_pco::PcoArray::from_array(array: vortex_array::array::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult<Self>
 

encodings/pco/src/array.rs

@@ -22,9 +22,11 @@ use vortex_array::DynArray;
 use vortex_array::ExecutionCtx;
 use vortex_array::ExecutionStep;
 use vortex_array::IntoArray;
+use vortex_array::LEGACY_SESSION;
 use vortex_array::Precision;
 use vortex_array::ProstMetadata;
 use vortex_array::ToCanonical;
+use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::Primitive;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::buffer::BufferHandle;
@@ -263,8 +265,8 @@ impl VTable for Pco {
         Ok(())
     }
 
-    fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
-        Ok(ExecutionStep::Done(array.decompress()?.into_array()))
+    fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
+        Ok(ExecutionStep::Done(array.decompress(ctx)?.into_array()))
     }
 
     fn reduce_parent(
@@ -437,14 +439,14 @@ impl PcoArray {
         }
     }
 
-    pub fn decompress(&self) -> VortexResult<PrimitiveArray> {
+    pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult<PrimitiveArray> {
         // To start, we figure out which chunks and pages we need to decompress, and with
         // what value offset into the first such page.
         let number_type = number_type_from_dtype(&self.dtype);
         let values_byte_buffer = match_number_enum!(
             number_type,
             NumberType<T> => {
-              self.decompress_values_typed::<T>()?
+              self.decompress_values_typed::<T>(ctx)?
             }
         );
 
@@ -457,11 +459,14 @@ impl PcoArray {
         ))
     }
 
-    fn decompress_values_typed<T: Number>(&self) -> VortexResult<ByteBuffer> {
+    fn decompress_values_typed<T: Number>(
+        &self,
+        ctx: &mut ExecutionCtx,
+    ) -> VortexResult<ByteBuffer> {
         // To start, we figure out what range of values we need to decompress.
         let slice_value_indices = self
             .unsliced_validity
-            .to_mask(self.unsliced_n_rows)
+            .execute_mask(self.unsliced_n_rows, ctx)?
             .valid_counts_for_indices(&[self.slice_start, self.slice_stop]);
         let slice_value_start = slice_value_indices[0];
         let slice_value_stop = slice_value_indices[1];
@@ -564,7 +569,11 @@ impl ValiditySliceHelper for PcoArray {
 
 impl OperationsVTable<Pco> for Pco {
     fn scalar_at(array: &PcoArray, index: usize) -> VortexResult<Scalar> {
-        array._slice(index, index + 1).decompress()?.scalar_at(0)
+        let mut ctx = LEGACY_SESSION.create_execution_ctx();
+        array
+            ._slice(index, index + 1)
+            .decompress(&mut ctx)?
+            .scalar_at(0)
     }
 }
 

encodings/pco/src/test.rs

@@ -6,6 +6,7 @@ use std::sync::LazyLock;
 
 use vortex_array::ArrayContext;
 use vortex_array::IntoArray;
+use vortex_array::LEGACY_SESSION;
 use vortex_array::ToCanonical;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::BoolArray;
@@ -47,7 +48,8 @@ fn test_compress_decompress() {
     assert!(compressed.pages.len() < array.nbytes() as usize);
 
     // check full decompression works
-    let decompressed = compressed.decompress().unwrap();
+    let mut ctx = LEGACY_SESSION.create_execution_ctx();
+    let decompressed = compressed.decompress(&mut ctx).unwrap();
     assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data));
 
     // check slicing works
@@ -69,7 +71,8 @@ fn test_compress_decompress_small() {
     let expected = array.into_array();
     assert_arrays_eq!(compressed, expected);
 
-    let decompressed = compressed.decompress().unwrap();
+    let mut ctx = LEGACY_SESSION.create_execution_ctx();
+    let decompressed = compressed.decompress(&mut ctx).unwrap();
     assert_arrays_eq!(decompressed, expected);
 }
 
@@ -78,7 +81,8 @@ fn test_empty() {
     let data: Vec<i32> = vec![];
     let array = PrimitiveArray::from_iter(data.clone());
     let compressed = PcoArray::from_primitive(&array, 3, 100).unwrap();
-    let primitive = compressed.decompress().unwrap();
+    let mut ctx = LEGACY_SESSION.create_execution_ctx();
+    let primitive = compressed.decompress(&mut ctx).unwrap();
     assert_arrays_eq!(primitive, PrimitiveArray::from_iter(data));
 }
 
@@ -118,9 +122,16 @@ fn test_validity_and_multiple_chunks_and_pages() {
     assert_nth_scalar!(slice, 0, 100);
     assert_nth_scalar!(slice, 2, 102);
     let primitive = slice.to_primitive();
-    assert_eq!(
-        primitive.validity(),
-        &Validity::Array(BoolArray::from_iter(vec![true, false, true]).into_array())
+
+    let mut ctx = LEGACY_SESSION.create_execution_ctx();
+    assert!(
+        primitive
+            .validity()
+            .mask_eq(
+                &Validity::Array(BoolArray::from_iter(vec![true, false, true]).into_array()),
+                &mut ctx,
+            )
+            .unwrap()
     );
 }
 

encodings/zstd/public-api.lock

@@ -74,7 +74,7 @@ pub struct vortex_zstd::ZstdArray
 
 impl vortex_zstd::ZstdArray
 
-pub fn vortex_zstd::ZstdArray::decompress(&self) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
+pub fn vortex_zstd::ZstdArray::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
 
 pub fn vortex_zstd::ZstdArray::from_array(array: vortex_array::array::ArrayRef, level: i32, values_per_frame: usize) -> vortex_error::VortexResult<Self>
 

encodings/zstd/src/array.rs

@@ -15,9 +15,11 @@ use vortex_array::DynArray;
 use vortex_array::ExecutionCtx;
 use vortex_array::ExecutionStep;
 use vortex_array::IntoArray;
+use vortex_array::LEGACY_SESSION;
 use vortex_array::Precision;
 use vortex_array::ProstMetadata;
 use vortex_array::ToCanonical;
+use vortex_array::VortexSessionExecute;
 use vortex_array::accessor::ArrayAccessor;
 use vortex_array::arrays::ConstantArray;
 use vortex_array::arrays::PrimitiveArray;
@@ -274,7 +276,7 @@ impl VTable for Zstd {
 
     fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionStep> {
         array
-            .decompress()?
+            .decompress(ctx)?
             .execute::<ArrayRef>(ctx)
             .map(ExecutionStep::Done)
     }
@@ -702,14 +704,14 @@ impl ZstdArray {
         }
     }
 
-    pub fn decompress(&self) -> VortexResult<ArrayRef> {
+    pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
         // To start, we figure out which frames we need to decompress, and with
         // what row offset into the first such frame.
         let byte_width = self.byte_width();
         let slice_n_rows = self.slice_stop - self.slice_start;
         let slice_value_indices = self
             .unsliced_validity
-            .to_mask(self.unsliced_n_rows)
+            .execute_mask(self.unsliced_n_rows, ctx)?
             .valid_counts_for_indices(&[self.slice_start, self.slice_stop]);
 
         let slice_value_idx_start = slice_value_indices[0];
@@ -787,14 +789,14 @@ impl ZstdArray {
         //
         // We ensure that the validity of the decompressed array ALWAYS matches the validity
         // implied by the DType.
-        if !self.dtype().is_nullable() && slice_validity != Validity::NonNullable {
+        if !self.dtype().is_nullable() && !matches!(slice_validity, Validity::NonNullable) {
             assert!(
-                slice_validity.all_valid(slice_n_rows)?,
+                matches!(slice_validity, Validity::AllValid),
                 "ZSTD array expects to be non-nullable but there are nulls after decompression"
             );
 
             slice_validity = Validity::NonNullable;
-        } else if self.dtype.is_nullable() && slice_validity == Validity::NonNullable {
+        } else if self.dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) {
             slice_validity = Validity::AllValid;
         }
         //
@@ -817,7 +819,7 @@ impl ZstdArray {
                 Ok(primitive.into_array())
             }
             DType::Binary(_) | DType::Utf8(_) => {
-                match slice_validity.to_mask(slice_n_rows).indices() {
+                match slice_validity.execute_mask(slice_n_rows, ctx)?.indices() {
                     AllOr::All => {
                         // the decompressed buffer is a bunch of interleaved u32 lengths
                         // and strings of those lengths, we need to reconstruct the
@@ -937,6 +939,10 @@ impl ValiditySliceHelper for ZstdArray {
 
 impl OperationsVTable<Zstd> for Zstd {
     fn scalar_at(array: &ZstdArray, index: usize) -> VortexResult<Scalar> {
-        array._slice(index, index + 1).decompress()?.scalar_at(0)
+        let mut ctx = LEGACY_SESSION.create_execution_ctx();
+        array
+            ._slice(index, index + 1)
+            .decompress(&mut ctx)?
+            .scalar_at(0)
     }
 }

encodings/zstd/src/compute/cast.rs

@@ -6,6 +6,7 @@ use vortex_array::IntoArray;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
 use vortex_array::scalar_fn::fns::cast::CastReduce;
+use vortex_array::validity::Validity;
 use vortex_error::VortexResult;
 
 use crate::Zstd;
@@ -45,11 +46,10 @@ impl CastReduce for Zstd {
             }
             (Nullability::Nullable, Nullability::NonNullable) => {
                 // null => non-null works if there are no nulls in the sliced range
-                let sliced_len = array.slice_stop() - array.slice_start();
-                let has_nulls = !array
-                    .unsliced_validity
-                    .slice(array.slice_start()..array.slice_stop())?
-                    .all_valid(sliced_len)?;
+                let has_nulls = !matches!(
+                    array.validity()?,
+                    Validity::AllValid | Validity::NonNullable
+                );
 
                 // We don't attempt to handle casting when there are nulls.
                 if has_nulls {