diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 710f7542..4c55ec6c 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -12,6 +12,10 @@ permissions: concurrency: group: single-acceptance-job-per-repo +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: integration: if: github.event_name == 'pull_request' && github.event.pull_request.draft == false @@ -32,7 +36,7 @@ jobs: python-version: '3.10' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install "hatch==${HATCH_VERSION}" - name: Run integration tests uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2 diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml index 4c8badba..7175bbb9 100644 --- a/.github/workflows/downstreams.yml +++ b/.github/workflows/downstreams.yml @@ -20,6 +20,10 @@ permissions: contents: read pull-requests: write +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: '1.16.5' + jobs: compatibility: strategy: @@ -44,7 +48,8 @@ jobs: - name: Install toolchain run: | - pip install hatch==1.9.4 + pip install "hatch==${HATCH_VERSION}" + - name: Acceptance uses: databrickslabs/sandbox/downstreams@acceptance/v0.4.2 with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0f504ff9..c105ddcb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,6 +14,10 @@ permissions: concurrency: group: single-acceptance-job-per-repo +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: integration: environment: runtime @@ -32,7 +36,7 @@ jobs: python-version: '3.10' - name: Install hatch - run: pip install hatch==1.9.4 + run: pip install "hatch==${HATCH_VERSION}" - name: Run nightly tests uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 10665e63..31527390 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -14,6 +14,10 @@ on: branches: - main +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: ci: strategy: @@ -34,7 +38,7 @@ jobs: - name: Run unit tests run: | - pip install hatch==1.9.4 + pip install "hatch==${HATCH_VERSION}" make test - name: Publish test coverage @@ -55,7 +59,7 @@ jobs: - name: Format all files run: | - pip install hatch==1.9.4 + pip install "hatch==${HATCH_VERSION}" make dev fmt - name: Fail on differences diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 528e0254..2fb3e248 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,10 @@ on: tags: - 'v*' +env: + HATCH_VERBOSE: "2" + HATCH_VERSION: "1.16.5" + jobs: publish: runs-on: @@ -27,7 +31,7 @@ jobs: - name: Build wheels run: | - pip install hatch==1.9.4 + pip install "hatch==${HATCH_VERSION}" hatch build - name: Draft release diff --git a/src/databricks/labs/lsql/lakeview/model.py b/src/databricks/labs/lsql/lakeview/model.py index d2bd8d9d..0070e6d5 100755 --- a/src/databricks/labs/lsql/lakeview/model.py +++ b/src/databricks/labs/lsql/lakeview/model.py @@ -613,7 +613,20 @@ def as_dict(self) -> Json: @classmethod def from_dict(cls, d: Json) -> Dataset: - return cls(display_name=d.get("displayName", None), name=d.get("name", None), query=d.get("query", None)) + # Compatibility: + # - Dashboard APIs previously placed the queries in the "query" attribute as-is, but now it's placed in the + # "queryLines" attribute as an array of strings. + # - We need to load from both previously saved files, as well as the Dashboard APIs. + # - Canonical format is therefore "query". + query: str | None + match d: + case {"query": str() as query, **_kw}: + pass + case {"queryLines": list() as queryLines, **_kw}: + query = "".join(queryLines) + case _: + query = None + return cls(display_name=d.get("displayName", None), name=d.get("name", None), query=query) @dataclass diff --git a/tests/integration/test_dashboards.py b/tests/integration/test_dashboards.py index feea32ed..09b0eab2 100644 --- a/tests/integration/test_dashboards.py +++ b/tests/integration/test_dashboards.py @@ -2,11 +2,13 @@ import datetime as dt import json import logging +import textwrap import webbrowser from pathlib import Path import pytest from databricks.labs.blueprint.entrypoint import is_in_debug +from databricks.sdk import WorkspaceClient from databricks.sdk.core import DatabricksError from databricks.sdk.service.catalog import SchemaInfo from databricks.sdk.service.dashboards import Dashboard as SDKDashboard @@ -143,6 +145,28 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, make_dash ) +def test_dashboards_save_to_folder_saves_sql_files(ws: WorkspaceClient, make_dashboard, tmp_path: Path) -> None: + dashboards = Dashboards(ws) + sdk_dashboard = make_dashboard() + + (tmp_path / "counter.sql").write_text("SELECT 10 AS count", encoding="utf-8") + dashboard_metadata = DashboardMetadata.from_path(tmp_path) + sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id) + + assert sdk_dashboard.path is not None + lakeview_dashboard = dashboards.get_dashboard(sdk_dashboard.path) + save_path = tmp_path / "saved" + dashboards.save_to_folder(lakeview_dashboard, save_path) + + exported_path = save_path / "counter.sql" + exported_query = exported_path.read_text(encoding="utf-8") + # Exporting formats the queries with sqlglot. + expected_query = textwrap.dedent("""\ + SELECT + 10 AS count""") + assert exported_query == expected_query + + def test_dashboard_deploys_dashboard_with_ten_counters(ws, make_dashboard, tmp_path): dashboards = Dashboards(ws) sdk_dashboard = make_dashboard() diff --git a/tests/unit/lakeview/test_model.py b/tests/unit/lakeview/test_model.py index 3dacbc0a..8c8e2622 100644 --- a/tests/unit/lakeview/test_model.py +++ b/tests/unit/lakeview/test_model.py @@ -1,10 +1,35 @@ from databricks.labs.lsql.lakeview.model import ( + Dataset, PaginationSize, TableV1EncodingMap, TableV1Spec, ) +def test_dataset_serialisation_round_trip() -> None: + dataset = Dataset("a_name", display_name="A Name", query="SELECT a FROM name") + serialized_first = dataset.as_dict() + restored = Dataset.from_dict(serialized_first) + assert dataset == restored + serialized_second = restored.as_dict() + assert serialized_first == serialized_second + + +def test_dataset_from_dict_reads_query_lines() -> None: + dataset = Dataset.from_dict({"name": "d", "queryLines": ["SELECT ", "1"]}) + assert dataset.query == "SELECT 1" + + +def test_dataset_from_dict_query_is_none_when_absent() -> None: + dataset = Dataset.from_dict({"name": "d"}) + assert dataset.query is None + + +def test_dataset_from_dict_reads_query() -> None: + dataset = Dataset.from_dict({"name": "d", "query": "SELECT 1"}) + assert dataset.query == "SELECT 1" + + def test_table_v1_spec_adds_invisible_columns_to_dict(): table_encodings = TableV1EncodingMap(None) spec = TableV1Spec(