Fix dashboard serialisation: query is now queryLines on upstream REST API

.github/workflows/acceptance.yml

@@ -12,6 +12,10 @@ permissions:
 concurrency:
   group: single-acceptance-job-per-repo
 
+env:
+  HATCH_VERBOSE: "2"
+  HATCH_VERSION: "1.16.5"
+
 jobs:
   integration:
     if: github.event_name == 'pull_request' && github.event.pull_request.draft == false
@@ -32,7 +36,7 @@ jobs:
           python-version: '3.10'
 
       - name: Install hatch
-        run: pip install hatch==1.9.4
+        run: pip install "hatch==${HATCH_VERSION}"
 
       - name: Run integration tests
         uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2

.github/workflows/downstreams.yml

@@ -20,6 +20,10 @@ permissions:
   contents: read
   pull-requests: write
 
+env:
+  HATCH_VERBOSE: "2"
+  HATCH_VERSION: '1.16.5'
+
 jobs:
   compatibility:
     strategy:
@@ -44,7 +48,8 @@ jobs:
 
       - name: Install toolchain
         run: |
-          pip install hatch==1.9.4
+          pip install "hatch==${HATCH_VERSION}"
+
       - name: Acceptance
         uses: databrickslabs/sandbox/downstreams@acceptance/v0.4.2
         with:

.github/workflows/nightly.yml

@@ -14,6 +14,10 @@ permissions:
 concurrency:
   group: single-acceptance-job-per-repo
 
+env:
+  HATCH_VERBOSE: "2"
+  HATCH_VERSION: "1.16.5"
+
 jobs:
   integration:
     environment: runtime
@@ -32,7 +36,7 @@ jobs:
           python-version: '3.10'
 
       - name: Install hatch
-        run: pip install hatch==1.9.4
+        run: pip install "hatch==${HATCH_VERSION}"
 
       - name: Run nightly tests
         uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2

.github/workflows/push.yml

@@ -14,6 +14,10 @@ on:
     branches:
       - main
 
+env:
+  HATCH_VERBOSE: "2"
+  HATCH_VERSION: "1.16.5"
+
 jobs:
   ci:
     strategy:
@@ -34,7 +38,7 @@ jobs:
 
       - name: Run unit tests
         run: |
-          pip install hatch==1.9.4
+          pip install "hatch==${HATCH_VERSION}"
           make test
 
       - name: Publish test coverage
@@ -55,7 +59,7 @@ jobs:
 
       - name: Format all files
         run: |
-          pip install hatch==1.9.4
+          pip install "hatch==${HATCH_VERSION}"
           make dev fmt
 
       - name: Fail on differences

.github/workflows/release.yml

@@ -5,6 +5,10 @@ on:
     tags:
       - 'v*'
 
+env:
+  HATCH_VERBOSE: "2"
+  HATCH_VERSION: "1.16.5"
+
 jobs:
   publish:
     runs-on:
@@ -27,7 +31,7 @@ jobs:
 
       - name: Build wheels
         run: |
-          pip install hatch==1.9.4
+          pip install "hatch==${HATCH_VERSION}"
           hatch build
       
       - name: Draft release

src/databricks/labs/lsql/lakeview/model.py

@@ -613,7 +613,20 @@ def as_dict(self) -> Json:
 
     @classmethod
     def from_dict(cls, d: Json) -> Dataset:
-        return cls(display_name=d.get("displayName", None), name=d.get("name", None), query=d.get("query", None))
+        # Compatibility:
+        #  - Dashboard APIs previously placed the queries in the "query" attribute as-is, but now it's placed in the
+        #    "queryLines" attribute as an array of strings.
+        #  - We need to load from both previously saved files, as well as the Dashboard APIs.
+        #  - Canonical format is therefore "query".
+        query: str | None
+        match d:
+            case {"query": str() as query, **_kw}:
+                pass
+            case {"queryLines": list() as queryLines, **_kw}:
+                query = "".join(queryLines)
+            case _:
+                query = None
+        return cls(display_name=d.get("displayName", None), name=d.get("name", None), query=query)
 
 
 @dataclass

tests/integration/test_dashboards.py

@@ -2,11 +2,13 @@
 import datetime as dt
 import json
 import logging
+import textwrap
 import webbrowser
 from pathlib import Path
 
 import pytest
 from databricks.labs.blueprint.entrypoint import is_in_debug
+from databricks.sdk import WorkspaceClient
 from databricks.sdk.core import DatabricksError
 from databricks.sdk.service.catalog import SchemaInfo
 from databricks.sdk.service.dashboards import Dashboard as SDKDashboard
@@ -143,6 +145,28 @@ def test_dashboard_deploys_dashboard_the_same_as_created_dashboard(ws, make_dash
     )
 
 
+def test_dashboards_save_to_folder_saves_sql_files(ws: WorkspaceClient, make_dashboard, tmp_path: Path) -> None:
+    dashboards = Dashboards(ws)
+    sdk_dashboard = make_dashboard()
+
+    (tmp_path / "counter.sql").write_text("SELECT 10 AS count", encoding="utf-8")
+    dashboard_metadata = DashboardMetadata.from_path(tmp_path)
+    sdk_dashboard = dashboards.create_dashboard(dashboard_metadata, dashboard_id=sdk_dashboard.dashboard_id)
+
+    assert sdk_dashboard.path is not None
+    lakeview_dashboard = dashboards.get_dashboard(sdk_dashboard.path)
+    save_path = tmp_path / "saved"
+    dashboards.save_to_folder(lakeview_dashboard, save_path)
+
+    exported_path = save_path / "counter.sql"
+    exported_query = exported_path.read_text(encoding="utf-8")
+    # Exporting formats the queries with sqlglot.
+    expected_query = textwrap.dedent("""\
+        SELECT
+          10 AS count""")
+    assert exported_query == expected_query
+
+
 def test_dashboard_deploys_dashboard_with_ten_counters(ws, make_dashboard, tmp_path):
     dashboards = Dashboards(ws)
     sdk_dashboard = make_dashboard()

tests/unit/lakeview/test_model.py

@@ -1,10 +1,35 @@
 from databricks.labs.lsql.lakeview.model import (
+    Dataset,
     PaginationSize,
     TableV1EncodingMap,
     TableV1Spec,
 )
 
 
+def test_dataset_serialisation_round_trip() -> None:
+    dataset = Dataset("a_name", display_name="A Name", query="SELECT a FROM name")
+    serialized_first = dataset.as_dict()
+    restored = Dataset.from_dict(serialized_first)
+    assert dataset == restored
+    serialized_second = restored.as_dict()
+    assert serialized_first == serialized_second
+
+
+def test_dataset_from_dict_reads_query_lines() -> None:
+    dataset = Dataset.from_dict({"name": "d", "queryLines": ["SELECT ", "1"]})
+    assert dataset.query == "SELECT 1"
+
+
+def test_dataset_from_dict_query_is_none_when_absent() -> None:
+    dataset = Dataset.from_dict({"name": "d"})
+    assert dataset.query is None
+
+
+def test_dataset_from_dict_reads_query() -> None:
+    dataset = Dataset.from_dict({"name": "d", "query": "SELECT 1"})
+    assert dataset.query == "SELECT 1"
+
+
 def test_table_v1_spec_adds_invisible_columns_to_dict():
     table_encodings = TableV1EncodingMap(None)
     spec = TableV1Spec(