22# SPDX-License-Identifier: BSD-3-Clause
33
44from pathlib import Path
5+ from typing import Literal
56
67import polars as pl
78import polars .exceptions as plexc
@@ -55,6 +56,7 @@ def _validate_and_collect(
5556 df : pl .DataFrame | pl .LazyFrame ,
5657 * ,
5758 cast : bool = False ,
59+ engine : Literal ["auto" , "in-memory" ] = "auto" ,
5860) -> pl .DataFrame :
5961 # Whether validation is performed eagerly is now determined by the input type: an
6062 # eager data frame raises within `validate`, a lazy frame raises upon collection.
@@ -64,7 +66,7 @@ def _validate_and_collect(
6466 return result
6567 else :
6668 assert isinstance (result , pl .LazyFrame )
67- return result .collect ()
69+ return result .collect (engine = engine )
6870
6971
7072# -------------------------------------- SCHEMA -------------------------------------- #
@@ -146,7 +148,8 @@ def test_invalid_primary_key_with_examples(
146148 ValidationError if eager else plexc .ComputeError ,
147149 match = r"'primary_key' failed for 2 rows; examples: \[\{'a': 1\}\]" ,
148150 ):
149- _validate_and_collect (MySchema , df )
151+ # For lazy validation, Polars' streaming engine can fail fast and may only count the first batch.
152+ _validate_and_collect (MySchema , df , engine = "in-memory" )
150153
151154
152155@pytest .mark .parametrize ("df_type" , [pl .DataFrame , pl .LazyFrame ])
0 commit comments