diff --git a/spectacles/cli.py b/spectacles/cli.py index 35a896b8..d373fff9 100644 --- a/spectacles/cli.py +++ b/spectacles/cli.py @@ -294,6 +294,7 @@ def main(): chunk_size=args.chunk_size, pin_imports=pin_imports, ignore_hidden=args.ignore_hidden, + ignore_measures=args.ignore_measures, ) ) elif args.command == "assert": @@ -597,7 +598,7 @@ def _build_sql_subparser( "--fail-fast", action="store_true", help=( - "Test explore-by-explore instead of dimension-by-dimension. " + "Test explore-by-explore instead of field-by-field. " "This means that validation takes less time but only returns the first " "error identified in each explore. " ), @@ -646,13 +647,18 @@ def _build_sql_subparser( "--chunk-size", type=int, default=500, - help="Limit the size of explore-level queries by this number of dimensions.", + help="Limit the size of explore-level queries by this number of fields.", ) subparser.add_argument( "--ignore-hidden", action="store_true", help=("Exclude hidden fields from validation."), ) + subparser.add_argument( + "--ignore-measures", + action="store_true", + help=("Exclude measure fields from validation."), + ) _build_validator_subparser(subparser_action, subparser) _build_select_subparser(subparser_action, subparser) @@ -925,8 +931,9 @@ async def run_sql( chunk_size, pin_imports, ignore_hidden, + ignore_measures, ) -> None: - """Runs and validates the SQL for each selected LookML dimension.""" + """Runs and validates the SQL for each selected LookML field.""" # Don't trust env to ignore .netrc credentials async_client = httpx.AsyncClient(trust_env=False) try: @@ -946,6 +953,7 @@ async def run_sql( runtime_threshold, chunk_size, ignore_hidden, + ignore_measures, ) finally: await async_client.aclose() @@ -956,7 +964,7 @@ async def run_sql( errors = sorted( results["errors"], - key=lambda x: (x["model"], x["explore"], x["metadata"].get("dimension")), + key=lambda x: (x["model"], x["explore"], x["metadata"].get("field")), ) if errors: @@ -967,13 +975,13 @@ async def run_sql( message=error["message"], sql=error["metadata"]["sql"], log_dir=log_dir, - dimension=error["metadata"].get("dimension"), + field=error["metadata"].get("field"), lookml_url=error["metadata"].get("lookml_url"), ) if fail_fast: logger.info( printer.dim( - "\n\nTo determine the exact dimensions responsible for " + "\n\nTo determine the exact fields responsible for " f"{'this error' if len(errors) == 1 else 'these errors'}, " "you can rerun \nSpectacles without --fail-fast." ) diff --git a/spectacles/client.py b/spectacles/client.py index 14b7bebd..129d7d70 100644 --- a/spectacles/client.py +++ b/spectacles/client.py @@ -635,19 +635,21 @@ async def get_lookml_models(self, fields: Optional[List] = None) -> List[JsonDic return response.json() @backoff.on_exception(backoff.expo, BACKOFF_EXCEPTIONS, max_tries=DEFAULT_MAX_TRIES) - async def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: - """Gets all dimensions for an explore from the LookmlModel endpoint. + async def get_lookml_fields( + self, model: str, explore: str, ignore_measures: bool = False + ) -> List[str]: + """Gets all fields for an explore from the LookmlModel endpoint. Args: model: Name of LookML model to query. explore: Name of LookML explore to query. Returns: - List[str]: Names of all the dimensions in the specified explore. Dimension - names are returned in the format 'explore_name.dimension_name'. + List[str]: Names of all the fields in the specified explore. Field + names are returned in the format 'explore_name.field_name'. """ - logger.debug(f"Getting all dimensions from explore {model}/{explore}") + logger.debug(f"Getting all fields from explore {model}/{explore}") params = {"fields": ["fields"]} url = utils.compose_url( self.api_url, @@ -659,31 +661,36 @@ async def get_lookml_dimensions(self, model: str, explore: str) -> List[str]: response.raise_for_status() except httpx.HTTPStatusError as error: raise LookerApiError( - name="unable-to-get-dimension-lookml", - title="Couldn't retrieve dimensions.", + name="unable-to-get-field-lookml", + title="Couldn't retrieve LookML fields.", status=response.status_code, detail=( - "Unable to retrieve dimension LookML details " + "Unable to retrieve LookML field details " f"for explore '{model}/{explore}'. Please try again." ), response=response, ) from error - return response.json()["fields"]["dimensions"] + fields = response.json()["fields"]["dimensions"] + + if not ignore_measures: + fields += response.json()["fields"]["measures"] + + return fields @backoff.on_exception(backoff.expo, BACKOFF_EXCEPTIONS, max_tries=5) async def create_query( self, model: str, explore: str, - dimensions: List[str], - fields: Optional[List] = None, + fields: List[str], + request_fields: Optional[List] = None, ) -> Dict: - """Creates a Looker async query for one or more specified dimensions. + """Creates a Looker async query for one or more specified fields. - The query created is a SELECT query, selecting all dimensions specified for a + The query created is a SELECT query, selecting all fields specified for a certain model and explore. Looker builds the query using the `sql` field in the - LookML for each dimension. + LookML for each field. If a Timeout exception is received, attempts to retry. @@ -693,21 +700,21 @@ async def create_query( "Creating async query for %s/%s/%s", model, explore, - "*" if len(dimensions) != 1 else dimensions[0], + "*" if len(fields) != 1 else fields[0], ) body = { "model": model, "view": explore, - "fields": dimensions, + "fields": fields, "limit": 0, "filter_expression": "1=2", } params: Dict[str, list] = {} - if fields is None: + if request_fields is None: params["fields"] = [] else: - params["fields"] = fields + params["fields"] = request_fields url = utils.compose_url(self.api_url, path=["queries"], params=params) response = await self.post(url=url, json=body, timeout=TIMEOUT_SEC) @@ -720,7 +727,7 @@ async def create_query( status=response.status_code, detail=( f"Failed to create query for {model}/{explore}/" - f'{"*" if len(dimensions) > 1 else dimensions[0]}. ' + f'{"*" if len(fields) > 1 else fields[0]}. ' "Please try again." ), response=response, @@ -732,7 +739,7 @@ async def create_query( "Query for %s/%s/%s created as query %s", model, explore, - "*" if len(dimensions) != 1 else dimensions[0], + "*" if len(fields) != 1 else fields[0], query_id, ) return result diff --git a/spectacles/exceptions.py b/spectacles/exceptions.py index 5594f2e1..875e3a80 100644 --- a/spectacles/exceptions.py +++ b/spectacles/exceptions.py @@ -120,7 +120,7 @@ def __init__( metadata = { "line_number": line_number, "lookml_url": lookml_url, - "dimension": field_name, + "field": field_name, "file_path": file_path, "severity": severity, } @@ -134,7 +134,7 @@ def __init__( self, model: str, explore: str, - dimension: Optional[str], + field: Optional[str], sql: str, message: str, line_number: Optional[int] = None, @@ -142,7 +142,7 @@ def __init__( lookml_url: Optional[str] = None, ): metadata = { - "dimension": dimension, + "field": field, "sql": sql, "line_number": line_number, "explore_url": explore_url, diff --git a/spectacles/logger.py b/spectacles/logger.py index 9b3de913..ea8b876a 100644 --- a/spectacles/logger.py +++ b/spectacles/logger.py @@ -109,11 +109,11 @@ def delete_color_codes(text: str) -> str: def log_sql_error( - model: str, explore: str, sql: str, log_dir: str, dimension: Optional[str] = None + model: str, explore: str, sql: str, log_dir: str, field: Optional[str] = None ) -> Path: - file_name = ( - model + "__" + explore + ("__" + dimension if dimension else "") - ).replace(".", "_") + file_name = (model + "__" + explore + ("__" + field if field else "")).replace( + ".", "_" + ) file_name += ".sql" file_path = Path(log_dir) / "queries" / file_name diff --git a/spectacles/lookml.py b/spectacles/lookml.py index 9fa5c5e6..69e9bce0 100644 --- a/spectacles/lookml.py +++ b/spectacles/lookml.py @@ -15,7 +15,7 @@ def __repr__(self): return f"{self.__class__.__name__}(name={self.name})" -class Dimension(LookMlObject): +class LookMlField(LookMlObject): def __init__( self, name: str, @@ -23,7 +23,7 @@ def __init__( explore_name: str, type: str, tags: List[str], - sql: str, + sql: Optional[str], is_hidden: bool, url: Optional[str] = None, ): @@ -38,10 +38,9 @@ def __init__( self.queried: bool = False self.errors: List[ValidationError] = [] - if ( - re.search(r"spectacles\s*:\s*ignore", sql, re.IGNORECASE) - or "spectacles: ignore" in tags - ): + if sql and re.search(r"spectacles\s*:\s*ignore", sql, re.IGNORECASE): + self.ignore = True + elif "spectacles: ignore" in tags: self.ignore = True else: self.ignore = False @@ -54,7 +53,7 @@ def __repr__(self): ) def __eq__(self, other): - if not isinstance(other, Dimension): + if not isinstance(other, LookMlField): return NotImplemented return ( @@ -66,7 +65,7 @@ def __eq__(self, other): ) def __lt__(self, other): - if not isinstance(other, Dimension): + if not isinstance(other, LookMlField): return NotImplemented return (self.model_name, self.explore_name, self.name) < ( @@ -82,8 +81,8 @@ def errored(self): @errored.setter def errored(self, value): raise AttributeError( - "Cannot assign to 'errored' property of a Dimension instance. " - "For a dimension to be considered errored, it must have a ValidationError " + "Cannot assign to 'errored' property of a LookMlField instance. " + "For a field to be considered errored, it must have a ValidationError " "in its 'errors' attribute." ) @@ -100,11 +99,11 @@ def from_json(cls, json_dict, model_name, explore_name): class Explore(LookMlObject): def __init__( - self, name: str, model_name: str, dimensions: Optional[List[Dimension]] = None + self, name: str, model_name: str, fields: Optional[List[LookMlField]] = None ): self.name = name self.model_name = model_name - self.dimensions = [] if dimensions is None else dimensions + self.fields = [] if fields is None else fields self.errors: List[ValidationError] = [] self.successes: List[JsonDict] = [] self.skipped = False @@ -117,13 +116,13 @@ def __eq__(self, other): return ( self.name == other.name and self.model_name == other.model_name - and self.dimensions == other.dimensions + and self.fields == other.fields ) @property def queried(self): - if self.dimensions: - return any(dimension.queried for dimension in self.dimensions) + if self.fields: + return any(field.queried for field in self.fields) else: return self._queried @@ -131,18 +130,16 @@ def queried(self): def queried(self, value: bool): if not isinstance(value, bool): raise TypeError("Value for queried must be boolean.") - if self.dimensions: - for dimension in self.dimensions: - dimension.queried = value + if self.fields: + for field in self.fields: + field.queried = value else: self._queried = value @property def errored(self): if self.queried: - return bool(self.errors) or any( - dimension.errored for dimension in self.dimensions - ) + return bool(self.errors) or any(field.errored for field in self.fields) else: return None @@ -151,21 +148,21 @@ def errored(self, value): raise AttributeError( "Cannot assign to 'errored' property of an Explore instance. " "For an explore to be considered errored, it must have a ValidationError " - "in its 'errors' attribute or contain dimensions in an errored state." + "in its 'errors' attribute or contain fields in an errored state." ) - def get_errored_dimensions(self): - for dimension in self.dimensions: - if dimension.errored: - yield dimension + def get_errored_fields(self): + for field in self.fields: + if field.errored: + yield field @classmethod def from_json(cls, json_dict, model_name): name = json_dict["name"] return cls(name, model_name) - def add_dimension(self, dimension: Dimension): - self.dimensions.append(dimension) + def add_field(self, field: LookMlField): + self.fields.append(field) @property def number_of_errors(self): @@ -174,9 +171,7 @@ def number_of_errors(self): errors = len(self.errors) else: errors = sum( - len(dimension.errors) - for dimension in self.dimensions - if dimension.errored + len(field.errors) for field in self.fields if field.errored ) return errors else: @@ -188,7 +183,7 @@ class CompiledSql: model_name: str explore_name: str sql: str - dimension_name: Optional[str] = None + field_name: Optional[str] = None @classmethod def from_explore(cls, explore: Explore, sql: str) -> "CompiledSql": @@ -197,11 +192,11 @@ def from_explore(cls, explore: Explore, sql: str) -> "CompiledSql": ) @classmethod - def from_dimension(cls, dimension: Dimension, sql: str) -> "CompiledSql": + def from_field(cls, field: LookMlField, sql: str) -> "CompiledSql": return CompiledSql( - model_name=dimension.model_name, - explore_name=dimension.explore_name, - dimension_name=dimension.name, + model_name=field.model_name, + explore_name=field.explore_name, + field_name=field.name, sql=sql, ) @@ -315,14 +310,14 @@ def iter_explores(self, errored: bool = False) -> Iterable[Explore]: else: yield explore - def iter_dimensions(self, errored: bool = False) -> Iterable[Dimension]: + def iter_fields(self, errored: bool = False) -> Iterable[LookMlField]: for explore in self.iter_explores(): - for dimension in explore.dimensions: + for field in explore.fields: if errored: - if dimension.errored: - yield dimension + if field.errored: + yield field else: - yield dimension + yield field @property def errored(self): @@ -410,12 +405,12 @@ def get_results( status = "failed" errors.append(explore.errors[0].to_dict()) elif explore.errored: - dimension_errors = [e for d in explore.dimensions for e in d.errors] - # If an explore has explore-level errors but not dimension-level + field_errors = [e for d in explore.fields for e in d.errors] + # If an explore has explore-level errors but not field-level # errors, return those instead. Skip anything marked as ignored. relevant_errors = [ e.to_dict() - for e in (dimension_errors or explore.errors) + for e in (field_errors or explore.errors) if not e.ignore ] if relevant_errors: @@ -448,35 +443,37 @@ def number_of_errors(self): return sum([model.number_of_errors for model in self.models if model.errored]) -async def build_explore_dimensions( +async def build_explore_fields( client: LookerClient, explore: Explore, ignore_hidden_fields: bool = False, + ignore_measures: bool = False, ) -> None: - """Creates Dimension objects for all dimensions in a given explore.""" - dimensions_json = await client.get_lookml_dimensions( - explore.model_name, explore.name + """Creates LookmlField objects for all fields in a given explore.""" + fields_json = await client.get_lookml_fields( + explore.model_name, explore.name, ignore_measures ) - dimensions: List[Dimension] = [] - for dimension_json in dimensions_json: - dimension: Dimension = Dimension.from_json( - dimension_json, explore.model_name, explore.name + fields: List[LookMlField] = [] + for field_json in fields_json: + field: LookMlField = LookMlField.from_json( + field_json, explore.model_name, explore.name ) - if dimension.url is not None: - dimension.url = client.base_url + dimension.url - if not dimension.ignore and not (dimension.is_hidden and ignore_hidden_fields): - dimensions.append(dimension) + if field.url is not None: + field.url = client.base_url + field.url + if not field.ignore and not (field.is_hidden and ignore_hidden_fields): + fields.append(field) - explore.dimensions = dimensions + explore.fields = fields async def build_project( client: LookerClient, name: str, filters: Optional[List[str]] = None, - include_dimensions: bool = False, + include_fields: bool = False, ignore_hidden_fields: bool = False, + ignore_measures: bool = False, include_all_explores: bool = False, ) -> Project: """Creates an object (tree) representation of a LookML project.""" @@ -510,11 +507,13 @@ async def build_project( for explore in model.explores if is_selected(model.name, explore.name, filters) ] - if include_dimensions: + if include_fields: for explore in model.explores: task = asyncio.create_task( - build_explore_dimensions(client, explore, ignore_hidden_fields), - name=f"build_explore_dimensions_{explore.name}", + build_explore_fields( + client, explore, ignore_hidden_fields, ignore_measures + ), + name=f"build_explore_fields_{explore.name}", ) tasks.append(task) diff --git a/spectacles/printer.py b/spectacles/printer.py index f6e54e19..8d3ae1dd 100644 --- a/spectacles/printer.py +++ b/spectacles/printer.py @@ -123,12 +123,12 @@ def print_sql_error( message: str, sql: str, log_dir: str, - dimension: Optional[str] = None, + field: Optional[str] = None, lookml_url: Optional[str] = None, ) -> None: path = model + "/" - if dimension: - path += dimension + if field: + path += field else: path += explore print_header(red(path), LINE_WIDTH + COLOR_CODE_LENGTH) @@ -138,7 +138,7 @@ def print_sql_error( if lookml_url: logger.info("\n" + f"LookML: {lookml_url}") - file_path = log_sql_error(model, explore, sql, log_dir, dimension) + file_path = log_sql_error(model, explore, sql, log_dir, field) logger.info("\n" + f"Test SQL: {file_path}") diff --git a/spectacles/runner.py b/spectacles/runner.py index ec3264be..34460f5c 100644 --- a/spectacles/runner.py +++ b/spectacles/runner.py @@ -307,6 +307,7 @@ async def validate_sql( runtime_threshold: int = DEFAULT_RUNTIME_THRESHOLD, chunk_size: int = DEFAULT_CHUNK_SIZE, ignore_hidden_fields: bool = False, + ignore_measures: bool = False, ) -> JsonDict: if filters is None: filters = ["*/*"] @@ -320,8 +321,9 @@ async def validate_sql( self.client, name=self.project, filters=filters, - include_dimensions=True, + include_fields=True, ignore_hidden_fields=ignore_hidden_fields, + ignore_measures=ignore_measures, ) base_explores: Set[CompiledSql] = set() if incremental: @@ -353,8 +355,9 @@ async def validate_sql( self.client, name=self.project, filters=filters, - include_dimensions=True, + include_fields=True, ignore_hidden_fields=ignore_hidden_fields, + ignore_measures=ignore_measures, ) target_explores: Set[CompiledSql] = set() if incremental: @@ -401,20 +404,20 @@ async def validate_sql( async with self.branch_manager(ref=ref): await validator.search(explores, fail_fast, chunk_size, profile=profile) - # Create dimension tests for the desired ref when explores errored + # Create field tests for the desired ref when explores errored if not fail_fast and incremental: - errored_dimensions = project.iter_dimensions(errored=True) - # For errored dimensions, create dimension tests for the target ref + errored_fields = project.iter_fields(errored=True) + # For errored fields, create field tests for the target ref async with self.branch_manager(ref=target, ephemeral=True): target_ref = self.branch_manager.ref - logger.debug("Compiling SQL for dimensions at the target ref") - compiled_dimensions = await asyncio.gather( + logger.debug("Compiling SQL for fields at the target ref") + compiled_fields = await asyncio.gather( *( - validator.compile_dimension(dimension) - for dimension in project.iter_dimensions(errored=True) + validator.compile_field(field) + for field in project.iter_fields(errored=True) ) ) - target_dimensions = set(compiled_dimensions) + target_fields = set(compiled_fields) # Keep only the errors that don't exist on the target branch logger.debug( @@ -422,16 +425,15 @@ async def validate_sql( f"@ {target or 'production'}" ) - # Namespace SQL with the dimension name, just in case + # Namespace SQL with the field name, just in case target_sql = tuple( - (compiled.dimension_name, compiled.sql) - for compiled in target_dimensions + (compiled.field_name, compiled.sql) for compiled in target_fields ) - for dimension in errored_dimensions: - for error in dimension.errors: + for field in errored_fields: + for error in field.errors: if ( isinstance(error, SqlError) - and (dimension.name, error.metadata["sql"]) in target_sql + and (field.name, error.metadata["sql"]) in target_sql ): error.ignore = True diff --git a/spectacles/validators/sql.py b/spectacles/validators/sql.py index b0d6a2db..0b97a19e 100644 --- a/spectacles/validators/sql.py +++ b/spectacles/validators/sql.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, Iterator, Union, cast import pydantic from spectacles.client import LookerClient -from spectacles.lookml import CompiledSql, Dimension, Explore +from spectacles.lookml import CompiledSql, LookMlField, Explore from spectacles.exceptions import SpectaclesException, SqlError from spectacles.logger import GLOBAL_LOGGER as logger from spectacles.printer import print_header @@ -22,33 +22,35 @@ @dataclass class Query: explore: Explore - dimensions: tuple[Dimension, ...] + fields: tuple[LookMlField, ...] query_id: str | None = None explore_url: str | None = None errored: bool | None = None runtime: float | None = None def __post_init__(self) -> None: - # Confirm that all dimensions are from the Explore associated here - if len(set((d.model_name, d.explore_name) for d in self.dimensions)) > 1: - raise ValueError("All Dimensions must be from the same model and explore") - elif self.dimensions[0].explore_name != self.explore.name: - raise ValueError("Dimension.explore_name must equal Query.explore.name") - elif self.dimensions[0].model_name != self.explore.model_name: - raise ValueError("Dimension.model_name must equal Query.explore.model_name") + # Confirm that all fields are from the Explore associated here + if len(set((d.model_name, d.explore_name) for d in self.fields)) > 1: + raise ValueError("All LookMlFields must be from the same model and explore") + elif self.fields[0].explore_name != self.explore.name: + raise ValueError("LookMlField.explore_name must equal Query.explore.name") + elif self.fields[0].model_name != self.explore.model_name: + raise ValueError( + "LookMlField.model_name must equal Query.explore.model_name" + ) def __repr__(self) -> str: - return f"Query(explore={self.explore.name} n={len(self.dimensions)})" + return f"Query(explore={self.explore.name} n={len(self.fields)})" def divide(self) -> Iterator[Query]: if not self.errored: raise TypeError("Query.errored must be True to divide") - if len(self.dimensions) < 2: - raise ValueError("Query must have at least 2 dimensions to divide") + if len(self.fields) < 2: + raise ValueError("Query must have at least 2 fields to divide") - midpoint = len(self.dimensions) // 2 - yield Query(self.explore, self.dimensions[:midpoint]) - yield Query(self.explore, self.dimensions[midpoint:]) + midpoint = len(self.fields) // 2 + yield Query(self.explore, self.fields[:midpoint]) + yield Query(self.explore, self.fields[midpoint:]) def to_profiler_format(self) -> ProfilerTableRow: if self.runtime is None: @@ -64,7 +66,7 @@ def to_profiler_format(self) -> ProfilerTableRow: ) return ( self.explore.name, - self.dimensions[0].name if len(self.dimensions) == 1 else "*", + self.fields[0].name if len(self.fields) == 1 else "*", self.runtime, self.query_id, self.explore_url, @@ -85,7 +87,7 @@ def print_profile_results(queries: List[Query], runtime_threshold: int) -> None: [query.to_profiler_format() for query in queries_by_runtime], headers=[ "Explore", - "Dimension(s)", + "Field(s)", "Runtime (s)", "Query ID", "Explore From Here", @@ -101,7 +103,7 @@ def print_profile_results(queries: List[Query], runtime_threshold: int) -> None: class SqlValidator: - """Runs and validates the SQL for each selected LookML dimension. + """Runs and validates the SQL for each selected LookML field. Args: client: Looker API client. @@ -129,31 +131,31 @@ def __init__( self._long_running_queries: List[Query] = [] async def compile_explore(self, explore: Explore) -> CompiledSql: - if not explore.dimensions: + if not explore.fields: raise AttributeError( - f"Explore '{explore.name}' is missing dimensions, " + f"Explore '{explore.name}' is missing fields, " "meaning this query won't have fields and will error. " - "Often this happens because you didn't include dimensions " + "Often this happens because you didn't include fields " "when you built the project." ) - dimensions = [dimension.name for dimension in explore.dimensions] - # Create a query that includes all dimensions + fields = [field.name for field in explore.fields] + # Create a query that includes all fields query = await self.client.create_query( - explore.model_name, explore.name, dimensions, fields=["id"] + explore.model_name, explore.name, fields, request_fields=["id"] ) sql = await self.client.run_query(query["id"]) return CompiledSql.from_explore(explore, sql) - async def compile_dimension(self, dimension: Dimension) -> CompiledSql: - # Create a query for the dimension + async def compile_field(self, field: LookMlField) -> CompiledSql: + # Create a query for the field query = await self.client.create_query( - dimension.model_name, - dimension.explore_name, - [dimension.name], - fields=["id"], + field.model_name, + field.explore_name, + [field.name], + request_fields=["id"], ) sql = await self.client.run_query(query["id"]) - return CompiledSql.from_dimension(dimension, sql) + return CompiledSql.from_field(field, sql) async def search( self, @@ -182,17 +184,17 @@ async def search( try: for explore in explores: # Sorting makes it more likely to prune the tree faster in binsearch - dimensions = tuple(sorted(explore.dimensions)) - if len(dimensions) == 0: + fields = tuple(sorted(explore.fields)) + if len(fields) == 0: logger.warning( f"Warning: Explore '{explore.name}' does not have any non-ignored " - "dimensions and will not be validated." + "fields and will not be validated." ) - elif len(dimensions) <= chunk_size: - queries_to_run.put_nowait(Query(explore, dimensions)) + elif len(fields) <= chunk_size: + queries_to_run.put_nowait(Query(explore, fields)) else: - for i in range(0, len(dimensions), chunk_size): - chunk = dimensions[i : i + chunk_size] + for i in range(0, len(fields), chunk_size): + chunk = fields[i : i + chunk_size] query = Query(explore, chunk) queries_to_run.put_nowait(query) @@ -249,10 +251,10 @@ async def _run_query( logger.debug("Waiting to acquire a query slot") await query_slot.acquire() result = await self.client.create_query( - model=query.dimensions[0].model_name, - explore=query.dimensions[0].explore_name, - dimensions=[dimension.name for dimension in query.dimensions], - fields=["id", "share_url"], + model=query.fields[0].model_name, + explore=query.fields[0].explore_name, + fields=[field.name for field in query.fields], + request_fields=["id", "share_url"], ) query.query_id = result["id"] query.explore_url = result["share_url"] @@ -348,7 +350,7 @@ async def _get_query_results( SqlError( model=explore.model_name, explore=explore.name, - dimension=None, + field=None, sql=query_result.sql, message=error.full_message, line_number=line_number, @@ -357,38 +359,38 @@ async def _get_query_results( ) # Make child queries and put them back on the queue - elif len(query.dimensions) > 1: + elif len(query.fields) > 1: for child in query.divide(): await queries_to_run.put(child) - # Assign the error(s) to its dimension - elif len(query.dimensions) == 1: - dimension = query.dimensions[0] - dimension.queried = True + # Assign the error(s) to its fields + elif len(query.fields) == 1: + field = query.fields[0] + field.queried = True for error in query_result.get_valid_errors(): line_number = ( error.sql_error_loc.line if error.sql_error_loc else None ) - dimension.errors.append( + field.errors.append( SqlError( - model=dimension.model_name, - explore=dimension.explore_name, - dimension=dimension.name, + model=field.model_name, + explore=field.explore_name, + field=field.name, sql=query_result.sql, message=error.full_message, line_number=line_number, - lookml_url=dimension.url, + lookml_url=field.url, explore_url=query.explore_url, ) ) else: raise ValueError( - "Query had an unexpected number of dimensions. " - "Queries must have at least one dimension, but " - f"{query!r} had {len(query.dimensions)} dimensions." + "Query had an unexpected number of fields. " + "Queries must have at least one field, but " + f"{query!r} had {len(query.fields)} fields." ) # Indicate there are no more queries or subqueries to run diff --git a/tests/conftest.py b/tests/conftest.py index 54c8b84a..11a58e84 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,13 @@ import pytest from spectacles.exceptions import SqlError -from spectacles.lookml import Project, Model, Explore, Dimension +from spectacles.lookml import Project, Model, Explore, LookMlField from spectacles.types import JsonDict from tests.utils import load_resource @pytest.fixture -def dimension(): - return Dimension( +def field(): + return LookMlField( name="age", model_name="eye_exam", explore_name="users", @@ -37,7 +37,7 @@ def project(): @pytest.fixture def sql_error(): return SqlError( - dimension="users.age", + field="users.age", explore="users", model="eye_exam", sql="SELECT age FROM users WHERE 1=2 LIMIT 1", diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 84c9ea74..338ac794 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -28,11 +28,11 @@ async def test_unsupported_api_version_should_raise_error(): ) -async def test_create_query_with_dimensions_should_return_certain_fields( +async def test_create_query_with_fields_should_return_certain_fields( looker_client: LookerClient, ): query = await looker_client.create_query( - model="eye_exam", explore="users", dimensions=["id", "age"] + model="eye_exam", explore="users", fields=["id", "age"] ) assert set(("id", "share_url")) <= set(query.keys()) assert int(query["limit"]) == 0 @@ -40,11 +40,11 @@ async def test_create_query_with_dimensions_should_return_certain_fields( assert query["model"] == "eye_exam" -async def test_create_query_without_dimensions_should_return_certain_fields( +async def test_create_query_without_fields_should_return_certain_fields( looker_client: LookerClient, ): query = await looker_client.create_query( - model="eye_exam", explore="users", dimensions=[] + model="eye_exam", explore="users", fields=[] ) assert set(("id", "share_url")) <= set(query.keys()) assert int(query["limit"]) == 0 diff --git a/tests/integration/test_lookml.py b/tests/integration/test_lookml.py index 0f681e8d..16f2c47a 100644 --- a/tests/integration/test_lookml.py +++ b/tests/integration/test_lookml.py @@ -1,24 +1,24 @@ import pytest from spectacles.client import LookerClient -from spectacles.lookml import Explore, build_project, build_explore_dimensions +from spectacles.lookml import Explore, build_project, build_explore_fields from spectacles.exceptions import SpectaclesException class TestBuildProject: - async def test_model_explore_dimension_counts_should_match( + async def test_model_explore_field_counts_should_match( self, looker_client: LookerClient ): project = await build_project( looker_client, name="eye_exam", filters=["eye_exam/users"], - include_dimensions=True, + include_fields=True, ) assert len(project.models) == 1 assert len(project.models[0].explores) == 1 - dimensions = project.models[0].explores[0].dimensions - assert len(dimensions) == 6 - assert "users.city" in [dim.name for dim in dimensions] + fields = project.models[0].explores[0].fields + assert len(fields) == 7 + assert "users.city" in [dim.name for dim in fields] assert not project.errored assert project.queried is False @@ -38,17 +38,29 @@ async def test_duplicate_selectors_should_be_deduplicated( ) assert len(project.models) == 1 - async def test_hidden_dimension_should_be_excluded_with_ignore_hidden( + async def test_hidden_field_should_be_excluded_with_ignore_hidden( self, looker_client: LookerClient ): project = await build_project( looker_client, name="eye_exam", filters=["eye_exam/users"], - include_dimensions=True, + include_fields=True, ignore_hidden_fields=True, ) - assert len(project.models[0].explores[0].dimensions) == 5 + assert len(project.models[0].explores[0].fields) == 6 + + async def test_measure_field_should_be_excluded_with_measures_hidden( + self, looker_client: LookerClient + ): + project = await build_project( + looker_client, + name="eye_exam", + filters=["eye_exam/users"], + include_fields=True, + ignore_measures=True, + ) + assert len(project.models[0].explores[0].fields) == 6 class TestBuildUnconfiguredProject: @@ -62,19 +74,29 @@ async def test_project_with_no_configured_models_should_raise_error( await build_project(looker_client, name="eye_exam_unconfigured") -class TestBuildDimensions: - async def test_dimension_count_should_match( +class TestBuildFields: + async def test_field_count_should_match( self, looker_client: LookerClient, explore: Explore ): - await build_explore_dimensions(looker_client, explore) - assert len(explore.dimensions) == 6 + await build_explore_fields(looker_client, explore) + assert len(explore.fields) == 7 - async def test_hidden_dimension_should_be_excluded_with_ignore_hidden( + async def test_hidden_fields_should_be_excluded_with_ignore_hidden( self, looker_client: LookerClient, explore: Explore ): - await build_explore_dimensions( + await build_explore_fields( looker_client, explore, ignore_hidden_fields=True, ) - assert len(explore.dimensions) == 5 + assert len(explore.fields) == 6 + + async def test_measure_fields_should_be_excluded_with_measures_hidden( + self, looker_client: LookerClient, explore: Explore + ): + await build_explore_fields( + looker_client, + explore, + ignore_measures=True, + ) + assert len(explore.fields) == 6 diff --git a/tests/integration/test_sql_validator.py b/tests/integration/test_sql_validator.py index 123c3c6e..29ad34ca 100644 --- a/tests/integration/test_sql_validator.py +++ b/tests/integration/test_sql_validator.py @@ -24,7 +24,7 @@ async def explores( validator.client, name="eye_exam", filters=[f"eye_exam/{explore_name}"], - include_dimensions=True, + include_fields=True, ) explores = tuple(project.iter_explores()) await validator.search(explores, fail_fast=False) @@ -44,6 +44,6 @@ def test_explores_errored_should_be_set_correctly(explores: Tuple[Explore, ...]) assert explore.errored -def test_ignored_dimensions_should_not_be_queried(explores: Tuple[Explore, ...]): +def test_ignored_fields_should_not_be_queried(explores: Tuple[Explore, ...]): for explore in explores: - assert not any(dim.queried for dim in explore.dimensions if dim.ignore is True) + assert not any(dim.queried for dim in explore.fields if dim.ignore is True) diff --git a/tests/resources/response_dimensions.json b/tests/resources/response_fields.json similarity index 100% rename from tests/resources/response_dimensions.json rename to tests/resources/response_fields.json diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 02d22f44..e620c052 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -55,11 +55,11 @@ def client_kwargs(): all_lookml_tests={"project": "project_name"}, run_lookml_test={"project": "project_name"}, get_lookml_models={}, - get_lookml_dimensions={"model": "model_name", "explore": "explore_name"}, + get_lookml_fields={"model": "model_name", "explore": "explore_name"}, create_query={ "model": "model_name", "explore": "explore_name", - "dimensions": ["dimension_a", "dimension_b"], + "fields": ["dimension_a", "dimension_b"], }, create_query_task={"query_id": 13041}, get_query_task_multi_results={"query_task_ids": ["ajsdkgj", "askkwk"]}, diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index f514c454..d4b7ba42 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -3,7 +3,7 @@ def test_logging_failing_explore_sql(tmpdir, sql_error): - sql_error.metadata["dimension"] = None + sql_error.metadata["field"] = None expected_directory = Path(tmpdir) / "queries" expected_directory.mkdir(exist_ok=True) @@ -12,7 +12,7 @@ def test_logging_failing_explore_sql(tmpdir, sql_error): sql_error.explore, sql_error.metadata["sql"], tmpdir, - sql_error.metadata["dimension"], + sql_error.metadata["field"], ) expected_path = expected_directory / "eye_exam__users.sql" @@ -22,7 +22,7 @@ def test_logging_failing_explore_sql(tmpdir, sql_error): assert content == "SELECT age FROM users WHERE 1=2 LIMIT 1" -def test_logging_failing_dimension_sql(tmpdir, sql_error): +def test_logging_failing_field_sql(tmpdir, sql_error): expected_directory = Path(tmpdir) / "queries" expected_directory.mkdir(exist_ok=True) @@ -31,7 +31,7 @@ def test_logging_failing_dimension_sql(tmpdir, sql_error): sql_error.explore, sql_error.metadata["sql"], tmpdir, - sql_error.metadata["dimension"], + sql_error.metadata["field"], ) expected_path = expected_directory / "eye_exam__users__users_age.sql" diff --git a/tests/unit/test_lookml.py b/tests/unit/test_lookml.py index 43ddfa19..b0170710 100644 --- a/tests/unit/test_lookml.py +++ b/tests/unit/test_lookml.py @@ -1,6 +1,6 @@ from copy import deepcopy import pytest -from spectacles.lookml import Model, Explore, Dimension, Project +from spectacles.lookml import Model, Explore, LookMlField, Project from spectacles.exceptions import SqlError from tests.utils import load_resource @@ -19,69 +19,69 @@ def test_explore_from_json(): explore = Explore.from_json(json_dict[0]["explores"][0], model_name) assert explore.name == "test_explore_one" assert explore.model_name == model_name - assert explore.dimensions == [] + assert explore.fields == [] -def test_dimension_from_json(): +def test_field_from_json(): model_name = "eye_exam" explore_name = "users" - json_dict = load_resource("response_dimensions.json") - dimension = Dimension.from_json(json_dict[0], model_name, explore_name) - assert dimension.name == "test_view.dimension_one" - assert dimension.model_name == model_name - assert dimension.explore_name == explore_name - assert dimension.type == "number" - assert dimension.url == "/projects/spectacles/files/test_view.view.lkml?line=340" - assert dimension.sql == "${TABLE}.dimension_one " - assert not dimension.ignore - - -def test_ignored_dimension_with_whitespace(): + json_dict = load_resource("response_fields.json") + field = LookMlField.from_json(json_dict[0], model_name, explore_name) + assert field.name == "test_view.dimension_one" + assert field.model_name == model_name + assert field.explore_name == explore_name + assert field.type == "number" + assert field.url == "/projects/spectacles/files/test_view.view.lkml?line=340" + assert field.sql == "${TABLE}.dimension_one " + assert not field.ignore + + +def test_ignored_field_with_whitespace(): name = "test_view.dimension_one" model_name = "eye_exam" explore_name = "users" - dimension_type = "number" + field_type = "number" tags = [] url = "/projects/spectacles/files/test_view.view.lkml?line=340" sql = " -- spectacles: ignore\n${TABLE}.dimension_one " is_hidden = False - dimension = Dimension( - name, model_name, explore_name, dimension_type, tags, sql, url, is_hidden + field = LookMlField( + name, model_name, explore_name, field_type, tags, sql, url, is_hidden ) - assert dimension.ignore + assert field.ignore -def test_ignored_dimension_with_no_whitespace(): +def test_ignored_field_with_no_whitespace(): name = "test_view.dimension_one" model_name = "eye_exam" explore_name = "users" - dimension_type = "number" + field_type = "number" tags = [] url = "/projects/spectacles/files/test_view.view.lkml?line=340" sql = "--spectacles:ignore\n${TABLE}.dimension_one " is_hidden = False - dimension = Dimension( - name, model_name, explore_name, dimension_type, tags, sql, url, is_hidden + field = LookMlField( + name, model_name, explore_name, field_type, tags, sql, url, is_hidden ) - assert dimension.ignore + assert field.ignore -def test_ignored_dimension_with_tags(): +def test_ignored_field_with_tags(): name = "test_view.dimension_one" model_name = "eye_exam" explore_name = "users" - dimension_type = "number" + field_type = "number" tags = ["spectacles: ignore"] url = "/projects/spectacles/files/test_view.view.lkml?line=340" sql = "${TABLE}.dimension_one " is_hidden = False - dimension = Dimension( - name, model_name, explore_name, dimension_type, tags, sql, url, is_hidden + field = LookMlField( + name, model_name, explore_name, field_type, tags, sql, url, is_hidden ) - assert dimension.ignore + assert field.ignore -@pytest.mark.parametrize("obj_name", ("dimension", "explore", "model", "project")) +@pytest.mark.parametrize("obj_name", ("field", "explore", "model", "project")) def test_comparison_to_mismatched_type_object_should_fail( request: pytest.FixtureRequest, obj_name: str ): @@ -110,26 +110,26 @@ def test_non_bool_errored_should_raise_value_error( lookml_obj.errored = 1 -def test_dimensions_with_different_sql_can_be_equal(dimension: Dimension): - a = dimension +def test_fields_with_different_sql_can_be_equal(field: LookMlField): + a = field b = deepcopy(a) b.sql = "${TABLE}.another_column" assert a == b -def test_dimension_should_not_be_errored_if_not_queried( - dimension: Dimension, sql_error: SqlError +def test_field_should_not_be_errored_if_not_queried( + field: LookMlField, sql_error: SqlError ): - assert dimension.errored is None - dimension.errors = [sql_error] - assert dimension.errored is None - dimension.queried = True - assert dimension.errored is True + assert field.errored is None + field.errors = [sql_error] + assert field.errored is None + field.queried = True + assert field.errored is True -def test_should_not_be_able_to_set_errored_on_dimension(dimension: Dimension): +def test_should_not_be_able_to_set_errored_on_field(field: LookMlField): with pytest.raises(AttributeError): - dimension.errored = True + field.errored = True def test_should_not_be_able_to_set_errored_on_explore(explore: Explore): @@ -137,9 +137,9 @@ def test_should_not_be_able_to_set_errored_on_explore(explore: Explore): explore.errored = True -def test_dimensions_can_be_sorted_by_name(): +def test_fields_can_be_sorted_by_name(): unsorted = [ - Dimension( + LookMlField( name="b", model_name="", explore_name="", @@ -148,7 +148,7 @@ def test_dimensions_can_be_sorted_by_name(): sql="", is_hidden=False, ), - Dimension( + LookMlField( name="a", model_name="", explore_name="", @@ -157,7 +157,7 @@ def test_dimensions_can_be_sorted_by_name(): sql="", is_hidden=False, ), - Dimension( + LookMlField( name="c", model_name="", explore_name="", @@ -169,12 +169,12 @@ def test_dimensions_can_be_sorted_by_name(): ] assert sorted(unsorted) != unsorted - assert [dimension.name for dimension in sorted(unsorted)] == ["a", "b", "c"] + assert [field.name for field in sorted(unsorted)] == ["a", "b", "c"] -def test_dimensions_can_be_sorted_by_explore_name(): +def test_fields_can_be_sorted_by_explore_name(): unsorted = [ - Dimension( + LookMlField( name="", model_name="", explore_name="b", @@ -183,7 +183,7 @@ def test_dimensions_can_be_sorted_by_explore_name(): sql="", is_hidden=False, ), - Dimension( + LookMlField( name="", model_name="", explore_name="c", @@ -192,7 +192,7 @@ def test_dimensions_can_be_sorted_by_explore_name(): sql="", is_hidden=False, ), - Dimension( + LookMlField( name="", model_name="", explore_name="a", @@ -204,14 +204,14 @@ def test_dimensions_can_be_sorted_by_explore_name(): ] assert sorted(unsorted) != unsorted - assert [dimension.explore_name for dimension in sorted(unsorted)] == ["a", "b", "c"] + assert [field.explore_name for field in sorted(unsorted)] == ["a", "b", "c"] def test_parent_queried_behavior_should_depend_on_its_child( - explore: Explore, dimension: Dimension, model, project: Project + explore: Explore, field: LookMlField, model, project: Project ): for parent, child, attr in [ - (explore, dimension, "dimensions"), + (explore, field, "fields"), (model, explore, "explores"), (project, model, "models"), ]: @@ -229,52 +229,52 @@ def test_parent_queried_behavior_should_depend_on_its_child( def test_comparison_to_mismatched_type_object_fails( - dimension: Dimension, explore: Explore, model, project: Project + field: LookMlField, explore: Explore, model, project: Project ): - assert dimension != 1 + assert field != 1 assert explore != 1 assert model != 1 assert project != 1 def test_explore_number_of_errors_batch_with_errors( - dimension: Dimension, explore: Explore, sql_error: SqlError + field: LookMlField, explore: Explore, sql_error: SqlError ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True explore.errors = [sql_error] assert explore.number_of_errors == 1 def test_explore_number_of_errors_batch_with_no_errors( - dimension: Dimension, explore: Explore + field: LookMlField, explore: Explore ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True assert explore.number_of_errors == 0 def test_explore_number_of_errors_single_with_errors( - dimension: Dimension, explore: Explore, sql_error: SqlError + field: LookMlField, explore: Explore, sql_error: SqlError ): - dimension.errors = [sql_error] - dimension.queried = True - explore.dimensions = [dimension, dimension] + field.errors = [sql_error] + field.queried = True + explore.fields = [field, field] assert explore.number_of_errors == 2 def test_explore_number_of_errors_single_with_no_errors( - dimension: Dimension, explore: Explore + field: LookMlField, explore: Explore ): - dimension.queried = True - explore.dimensions = [dimension, dimension] + field.queried = True + explore.fields = [field, field] assert explore.number_of_errors == 0 def test_model_number_of_errors_batch_with_errors( - dimension: Dimension, explore: Explore, model: Model, sql_error: SqlError + field: LookMlField, explore: Explore, model: Model, sql_error: SqlError ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True explore.errors = [sql_error] model.explores = [explore, explore] @@ -282,28 +282,28 @@ def test_model_number_of_errors_batch_with_errors( def test_model_number_of_errors_batch_with_no_errors( - dimension: Dimension, explore: Explore, model: Model + field: LookMlField, explore: Explore, model: Model ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True model.explores = [explore, explore] assert model.number_of_errors == 0 def test_model_number_of_errors_single_with_errors( - dimension: Dimension, explore: Explore, model: Model, sql_error: SqlError + field: LookMlField, explore: Explore, model: Model, sql_error: SqlError ): - dimension.errors = [sql_error] - explore.dimensions = [dimension, dimension] + field.errors = [sql_error] + explore.fields = [field, field] explore.queried = True model.explores = [explore, explore] assert model.number_of_errors == 4 def test_model_number_of_errors_single_with_no_errors( - dimension: Dimension, explore: Explore, model: Model + field: LookMlField, explore: Explore, model: Model ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explore.queried = True model.explores = [explore, explore] assert model.number_of_errors == 0 @@ -327,13 +327,13 @@ def test_model_get_errored_explores_returns_the_correct_explore( def test_project_number_of_errors_batch_with_errors( - dimension: Dimension, + field: LookMlField, explore: Explore, model: Model, project: Project, sql_error: SqlError, ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True explore.errors = [sql_error] model.explores = [explore, explore] @@ -342,9 +342,9 @@ def test_project_number_of_errors_batch_with_errors( def test_project_number_of_errors_batch_with_no_errors( - dimension: Dimension, explore: Explore, model: Model, project: Project + field: LookMlField, explore: Explore, model: Model, project: Project ): - explore.dimensions = [dimension] + explore.fields = [field] explore.queried = True model.explores = [explore, explore] project.models = [model] @@ -352,14 +352,14 @@ def test_project_number_of_errors_batch_with_no_errors( def test_project_number_of_errors_single_with_errors( - dimension: Dimension, + field: LookMlField, explore: Explore, model: Model, project: Project, sql_error: SqlError, ): - dimension.errors = [sql_error] - explore.dimensions = [dimension, dimension] + field.errors = [sql_error] + explore.fields = [field, field] explore.queried = True model.explores = [explore, explore] project.models = [model, model] @@ -367,9 +367,9 @@ def test_project_number_of_errors_single_with_errors( def test_project_number_of_errors_single_with_no_errors( - dimension: Dimension, explore: Explore, model: Model, project: Project + field: LookMlField, explore: Explore, model: Model, project: Project ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explore.queried = True model.explores = [explore, explore] project.models = [model, model] diff --git a/tests/unit/test_printer.py b/tests/unit/test_printer.py index a024a13a..dccd9559 100644 --- a/tests/unit/test_printer.py +++ b/tests/unit/test_printer.py @@ -44,7 +44,7 @@ def test_mark_line_even_number_of_lines(): def test_sql_error_prints_with_relevant_info(mock_log, sql_error, caplog): model = "model_a" explore = "explore_a" - dimension = "view_a.dimension_a" + field = "view_a.dimension_a" message = "A super important error occurred." printer.print_sql_error( @@ -58,7 +58,7 @@ def test_sql_error_prints_with_relevant_info(mock_log, sql_error, caplog): assert model in caplog.text assert explore in caplog.text assert message in caplog.text - assert dimension not in caplog.text + assert field not in caplog.text printer.print_sql_error( model=model, diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 2ab2a0df..acd4ed5b 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1,50 +1,48 @@ import pytest -from spectacles.lookml import Dimension, Explore +from spectacles.lookml import LookMlField, Explore from spectacles.validators.sql import Query from copy import deepcopy -def test_query_dimensions_should_belong_to_own_explore( - explore: Explore, dimension: Dimension +def test_query_fields_should_belong_to_own_explore( + explore: Explore, field: LookMlField ): - # Dimensions come from different explores - wrong_dimension = deepcopy(dimension) - wrong_dimension.explore_name = "not_eye_exam" + # Fields come from different explores + wrong_field = deepcopy(field) + wrong_field.explore_name = "not_eye_exam" with pytest.raises(ValueError): - Query(explore=explore, dimensions=(dimension, wrong_dimension)) + Query(explore=explore, fields=(field, wrong_field)) - # All dimensions come from a different explore + # All fields come from a different explore explore.name = "not_eye_exam" with pytest.raises(ValueError): - Query(explore=explore, dimensions=(dimension, dimension)) + Query(explore=explore, fields=(field, field)) -def test_query_divide_with_different_numbers_of_dimensions( - explore: Explore, dimension: Dimension +def test_query_divide_with_different_numbers_of_fields( + explore: Explore, field: LookMlField ): - query = Query(explore=explore, dimensions=tuple([dimension] * 2), errored=True) - assert sorted([len(child.dimensions) for child in query.divide()]) == [1, 1] + query = Query(explore=explore, fields=tuple([field] * 2), errored=True) + assert sorted([len(child.fields) for child in query.divide()]) == [1, 1] - query = Query(explore=explore, dimensions=tuple([dimension] * 5), errored=True) - assert sorted([len(child.dimensions) for child in query.divide()]) == [2, 3] + query = Query(explore=explore, fields=tuple([field] * 5), errored=True) + assert sorted([len(child.fields) for child in query.divide()]) == [2, 3] - query = Query(explore=explore, dimensions=tuple([dimension] * 8), errored=True) - assert sorted([len(child.dimensions) for child in query.divide()]) == [4, 4] + query = Query(explore=explore, fields=tuple([field] * 8), errored=True) + assert sorted([len(child.fields) for child in query.divide()]) == [4, 4] - query = Query(explore=explore, dimensions=tuple([dimension] * 101), errored=True) - assert sorted([len(child.dimensions) for child in query.divide()]) == [50, 51] + query = Query(explore=explore, fields=tuple([field] * 101), errored=True) + assert sorted([len(child.fields) for child in query.divide()]) == [50, 51] -def test_query_with_one_dimension_should_not_divide( - explore: Explore, dimension: Dimension -): - query = Query(explore=explore, dimensions=(dimension,), errored=True) +def test_query_with_one_field_should_not_divide(explore: Explore, field: LookMlField): + query = Query(explore=explore, fields=(field,), errored=True) with pytest.raises(ValueError): next(query.divide()) -def test_query_should_not_divide_if_not_errored(explore: Explore, dimension: Dimension): - query = Query(explore=explore, dimensions=(dimension, dimension)) +def test_query_should_not_divide_if_not_errored(explore: Explore, field: LookMlField): + query = Query(explore=explore, fields=(field, field)) with pytest.raises(TypeError): next(query.divide()) @@ -56,16 +54,14 @@ def test_query_should_not_divide_if_not_errored(explore: Explore, dimension: Dim assert next(query.divide()) -def test_query_should_convert_to_profiler_format( - explore: Explore, dimension: Dimension -): +def test_query_should_convert_to_profiler_format(explore: Explore, field: LookMlField): explore_url = "https://spectacles.looker.com/x" query_id = "12345" runtime = 10.0 query = Query( explore=explore, - dimensions=(dimension, dimension), + fields=(field, field), runtime=runtime, query_id=query_id, explore_url=explore_url, @@ -80,14 +76,14 @@ def test_query_should_convert_to_profiler_format( query = Query( explore=explore, - dimensions=(dimension,), + fields=(field,), runtime=runtime, query_id=query_id, explore_url=explore_url, ) assert query.to_profiler_format() == ( explore.name, - dimension.name, + field.name, runtime, query_id, explore_url, diff --git a/tests/unit/test_sql_validator.py b/tests/unit/test_sql_validator.py index 87af9c22..6f61ae11 100644 --- a/tests/unit/test_sql_validator.py +++ b/tests/unit/test_sql_validator.py @@ -7,7 +7,7 @@ import httpx import respx from spectacles.validators.sql import Query, SqlValidator -from spectacles.lookml import Explore, Dimension +from spectacles.lookml import Explore, LookMlField from spectacles.exceptions import LookerApiError from spectacles.client import LookerClient @@ -40,11 +40,11 @@ def query_slot() -> asyncio.Semaphore: @pytest.fixture -def query(explore: Explore, dimension: Dimension) -> Query: - return Query(explore, (dimension,), query_id="12345") +def query(explore: Explore, field: LookMlField) -> Query: + return Query(explore, (field,), query_id="12345") -async def test_compile_explore_without_dimensions_should_not_work( +async def test_compile_explore_without_fields_should_not_work( explore: Explore, validator: SqlValidator ): with pytest.raises(AttributeError): @@ -54,12 +54,12 @@ async def test_compile_explore_without_dimensions_should_not_work( async def test_compile_explore_compiles_sql( mocked_api: respx.MockRouter, explore: Explore, - dimension: Dimension, + field: LookMlField, validator: SqlValidator, ): query_id = 12345 sql = "SELECT * FROM users" - explore.dimensions = [dimension] + explore.fields = [field] mocked_api.post("queries", params={"fields": "id"}, name="create_query").respond( 200, json={"id": query_id} ) @@ -70,14 +70,14 @@ async def test_compile_explore_compiles_sql( assert compiled.explore_name == explore.name assert compiled.model_name == explore.model_name assert compiled.sql == sql - assert compiled.dimension_name is None + assert compiled.field_name is None mocked_api["create_query"].calls.assert_called_once() mocked_api["run_query"].calls.assert_called_once() -async def test_compile_dimension_compiles_sql( +async def test_compile_field_compiles_sql( mocked_api: respx.MockRouter, - dimension: Dimension, + field: LookMlField, validator: SqlValidator, ): query_id = 12345 @@ -88,11 +88,11 @@ async def test_compile_dimension_compiles_sql( mocked_api.get(f"queries/{query_id}/run/sql", name="run_query").respond( 200, text=sql ) - compiled = await validator.compile_dimension(dimension) - assert compiled.explore_name == dimension.explore_name - assert compiled.model_name == dimension.model_name + compiled = await validator.compile_field(field) + assert compiled.explore_name == field.explore_name + assert compiled.model_name == field.model_name assert compiled.sql == sql - assert compiled.dimension_name is dimension.name + assert compiled.field_name is field.name mocked_api["create_query"].calls.assert_called_once() mocked_api["run_query"].calls.assert_called_once() @@ -261,8 +261,8 @@ async def test_get_query_results_error_query_is_divided( } }, ) - # Need more than one dimension so the query will be divided - query.dimensions = (query.dimensions[0], query.dimensions[0]) + # Need more than one field so the query will be divided + query.fields = (query.fields[0], query.fields[0]) validator._task_to_query[query_task_id] = query task = asyncio.create_task( @@ -284,7 +284,7 @@ async def test_get_query_results_error_query_is_divided( assert query.errored # If not fail fast, the explore won't be marked as queried because we haven't yet - # queried the individual dimensions + # queried the individual fields if fail_fast: assert query.explore.queried assert query.explore.errored @@ -376,9 +376,9 @@ async def test_search_works_with_passing_query( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explores = (explore,) query_id = 12345 @@ -420,9 +420,9 @@ async def test_search_works_with_error_query( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explores = (explore,) explore_url = "https://spectacles.looker.com/x" @@ -522,7 +522,7 @@ async def test_search_works_with_error_query( if fail_fast: assert explore.errors[0].message == message else: - assert all(d.errors[0].message == message for d in explore.dimensions) + assert all(d.errors[0].message == message for d in explore.fields) @pytest.mark.parametrize("fail_fast", (True, False)) @@ -531,9 +531,9 @@ async def test_search_handles_exceptions_raised_while_running_queries( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explores = (explore,) mocked_api.post( @@ -554,9 +554,9 @@ async def test_search_handles_exceptions_raised_while_getting_query_results( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): - explore.dimensions = [dimension, dimension] + explore.fields = [field, field] explores = (explore,) query_id = 12345 query_task_id = "abcdef12345" @@ -586,10 +586,10 @@ async def test_search_with_chunk_size_should_limit_queries( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): chunk_size = 10 - explore.dimensions = [dimension] * 100 + explore.fields = [field] * 100 explore_url = "https://spectacles.looker.com/x" # Define some factories to make IDs sensible across requests @@ -648,11 +648,11 @@ def get_query_results_factory(request) -> httpx.Response: assert len(body["fields"]) == 10 -async def test_search_with_explore_without_dimensions_warns( +async def test_search_with_explore_without_fields_warns( explore: Explore, validator: SqlValidator, caplog: pytest.LogCaptureFixture ): caplog.set_level(logging.WARNING) - explore.dimensions = [] + explore.fields = [] await validator.search(explores=(explore,), fail_fast=False) assert explore.name in caplog.text @@ -661,10 +661,10 @@ async def test_looker_api_error_with_queries_in_flight_shuts_down_gracefully( mocked_api: respx.MockRouter, validator: SqlValidator, explore: Explore, - dimension: Dimension, + field: LookMlField, ): chunk_size = 10 - explore.dimensions = [dimension] * 1000 + explore.fields = [field] * 1000 explore_url = "https://spectacles.looker.com/x" # Define some factories to make IDs sensible across requests