Update dashboard ES document (#236)

* Update dashboard ES document

* bump deps

* update tests

* update search query

* fix flake8

Author: feng-tao

databuilder/extractor/neo4j_search_data_extractor.py

@@ -69,22 +69,26 @@ class Neo4jSearchDataExtractor(Extractor):
         """
     )
 
-    # todo: 1. change total_read once we have the usage;
-    #  2. add more fields once we have in the graph; 3. change mode to generic once add more support for dashboard
+    # todo: 1. change mode to generic once add more support for dashboard
     DEFAULT_NEO4J_DASHBOARD_CYPHER_QUERY = textwrap.dedent(
         """
-        MATCH (dashboard:Dashboard)
-        OPTIONAL MATCH (dashboard)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
-        OPTIONAL MATCH (dashboard)-[:DESCRIPTION]->(db_descr:Description)
+        MATCH (db:Dashboard)
+        MATCH (db)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
+        MATCH (dbg)-[:DASHBOARD_GROUP_OF]->(cluster:Cluster)
+        OPTIONAL MATCH (db)-[:DESCRIPTION]->(db_descr:Description)
         OPTIONAL MATCH (dbg)-[:DESCRIPTION]->(dbg_descr:Description)
-        {publish_tag_filter}
-        with dashboard, dbg, db_descr, dbg_descr
-        where dashboard.name is not null
-        return dbg.name as dashboard_group, dashboard.name as dashboard_name,
+        OPTIONAL MATCH (db)-[:EXECUTED]->(last_exec:Execution)
+        WHERE split(last_exec.key, '/')[5] = '_last_successful_execution'
+        OPTIONAL MATCH (db)-[read:READ_BY]->(user:User)
+        OPTIONAL MATCH (db)-[:HAS_QUERY]->(query:Query)
+        with db, dbg, db_descr, dbg_descr, cluster, last_exec, query, SUM(read.read_count) AS total_usage
+        return dbg.name as dashboard_group, db.name as dashboard_name, cluster.name as cluster,
         coalesce(db_descr.description, '') as description,
-        coalesce(dbg.description, '') as dashboard_group_description,
-        'mode' as product,
-        1 AS total_usage
+        coalesce(dbg.description, '') as dashboard_group_description, dbg.dashboard_group_url as group_url,
+        db.dashboard_url as url, db.key as uri,
+        'mode' as product, last_exec.timestamp as last_successful_run_timestamp,
+        COLLECT(DISTINCT query.name) as query_names,
+        total_usage
         order by dbg.name
         """
     )

databuilder/models/dashboard_elasticsearch_document.py

@@ -13,14 +13,26 @@ def __init__(self,
                  description,  # type: Union[str, None]
                  total_usage,  # type: int
                  product='',  # type: Optional[str]
+                 cluster='',  # type: Optional[str]
                  dashboard_group_description=None,  # type: Optional[str]
+                 query_names=None,  # type:  Union[List[str], None]
+                 group_url=None,  # type: Optional[str]
+                 url=None,  # type: Optional[str]
+                 uri=None,  # type: Optional[str]
+                 last_successful_run_timestamp=None,  # type: Optional[int]
                  tags=None  # type: list
                  ):
         # type: (...) -> None
         self.dashboard_group = dashboard_group
         self.dashboard_name = dashboard_name
         self.description = description
+        self.cluster = cluster
         self.product = product
+        self.group_url = group_url
+        self.url = url
+        self.uri = uri
+        self.last_successful_run_timestamp = last_successful_run_timestamp
         self.total_usage = total_usage
         self.dashboard_group_description = dashboard_group_description
+        self.query_names = query_names
         self.tags = tags

setup.py

@@ -2,7 +2,7 @@
 from setuptools import setup, find_packages
 
 
-__version__ = '2.5.0'
+__version__ = '2.5.1'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
 with open(requirements_path) as requirements_file:

tests/unit/models/test_dashboard_elasticsearch_document.py

@@ -15,14 +15,26 @@ def test_to_json(self):
                                        dashboard_name='test_dashboard_name',
                                        description='test_description',
                                        product='mode',
+                                       cluster='gold',
                                        dashboard_group_description='work space group',
+                                       query_names=['query1'],
+                                       group_url='mode_group_url',
+                                       url='mode_report_url',
+                                       uri='mode_dashboard://gold.cluster/dashboard_group/dashboard',
+                                       last_successful_run_timestamp=10,
                                        total_usage=10,
                                        tags=['test'])
 
         expected_document_dict = {"dashboard_group": "test_dashboard_group",
                                   "dashboard_name": "test_dashboard_name",
                                   "description": "test_description",
                                   "product": "mode",
+                                  "cluster": "gold",
+                                  "group_url": "mode_group_url",
+                                  "url": "mode_report_url",
+                                  "uri": "mode_dashboard://gold.cluster/dashboard_group/dashboard",
+                                  "query_names": ['query1'],
+                                  "last_successful_run_timestamp": 10,
                                   "dashboard_group_description": "work space group",
                                   "total_usage": 10,
                                   "tags": ["test"]