feat: Automation Node history (baserow#4787)

* WIP - node history

* Add migrations

* Fix test

* Re-generate migrations

* Node history should show result, not payload.

* Ensure nested node history scrolls

* get_previous_positions() may return None

* Fix get_iteration() to use the new iteration_path

* The scrollbar should be lighter, hide the background container.

* Only show expandable if the node history is completed

* Optimize node history endpoint

* Return the node type and label from the backend, so that we don't have to lookup published node which the frontend doesn't know about

* Use history status icons from design spec

* Remove debugging code

* Display workflow run error

* Don't expand history if workflow is in progress

* Find the most specific error

* Remove old HistorySection and unused translations

* Remove old _NodeHistory.vue, replaced by NodeHistory.vue

* Ensure node type label is truncated

* Align node type header icon

* Ensure Node Explorer's content name is truncated

* Add router label to node type, ensure max-width for label

* Make sure header expand icon is centered

* Truncate error preview

* Ensure node result is created when there is an error. This is needed to show the error in the node history.

* Lint fix

* Ensure parent nodes show error indicator (red text) if child has error.

* Clean up styles

* Clean up

* Handle case when router node has error

* Don't render Context or Modal if not final node

* Document success_count and fail_count in the API response

* When workflow times out, ensure child node histories are also updated.

* Remove unused css definitions

* Use computed props

* Add test to check query count for workflow history

* Update test to ensure node history is updated when workflow times out

* use a single timezone.now() instance

* getNodeType should be a computed prop

* getHistoryIconPath should be a computed prop

* Add test for node history in api response

* Add test for success/fail counts

* Add assertion to existing tests to ensure node result is created on workflow error

* Update the workflow when a node is modified

* Add original_workflow to AutomationWorkflowHistory model, migration, and backfill original_workflow

* Delete any published automations that don't have history.

* Create a snapshot of the history via publish flow.

* Add id to automationnodehistory ordering. Update migrations.

* Fix tests

* Ensure sample data is saved from snapshot to the draft workflow.

* Fix test

* Add tests for snapshots

* Remove unnecessary guard.

* Handle list vs flat result data.

* Ensure workflow is updated for snapshot when we CRUD a node.

* Make the edge detection more generic.

* Add new history_clone workflow state choice

* Refactor common logic to new _clone_workflow().

* Simplify async_start_workflow()

* Exclude history clones from being deleted during publishing.

* Ensure active_published only filters for live workflows

* Refactor existing tests + write new ones

* Fix test

* Add real-time for dispatch start/end, and update history in frontend.

* Increase size of the triple-dot link that reveals the Show result button.

* Ensure original_workflow is ZDM compatible

* Squash local migrations

* Simplify

* Add spinner when history is loading.

* Improve clean_up_previously_published_automations() logic

* Use global cache instead of relying on updated_on for cloned workflow.

* Rename history clone to test clone.

* Create clone only for test, not for simulation.

* Update docstring/comments.

* Move clone creation to toggle_test_run()

* Post-migration fixes

* Ensure async_start_workflow() always receives a draf or live workflow. For tests, it should get/create a clone.

* Fix state confusion

* Add filter to prevent deleting newly created test clone.

* Fix test

---------

Co-authored-by: Jeremie Pardou <571533+jrmi@users.noreply.github.com>

Author: paljort

backend/src/baserow/contrib/automation/api/workflows/serializers.py

@@ -2,7 +2,10 @@
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
+from baserow.api.pagination import PageNumberPagination
 from baserow.contrib.automation.models import (
+    AutomationHistory,
+    AutomationNodeHistory,
     AutomationWorkflow,
     AutomationWorkflowHistory,
 )
@@ -103,14 +106,88 @@ class OrderAutomationWorkflowsSerializer(serializers.Serializer):
     )
 
 
-class AutomationWorkflowHistorySerializer(serializers.ModelSerializer):
+class AutomationHistorySerializer(serializers.ModelSerializer):
     class Meta:
-        model = AutomationWorkflowHistory
+        model = AutomationHistory
         fields = (
             "id",
             "started_on",
             "completed_on",
-            "is_test_run",
             "message",
             "status",
         )
+
+
+class AutomationNodeHistorySerializer(AutomationHistorySerializer):
+    parent_node_id = serializers.SerializerMethodField()
+    iteration = serializers.SerializerMethodField()
+    result = serializers.SerializerMethodField()
+    node_type = serializers.SerializerMethodField()
+    node_label = serializers.SerializerMethodField()
+
+    class Meta:
+        model = AutomationNodeHistory
+        fields = AutomationHistorySerializer.Meta.fields + (
+            "workflow_history",
+            "node",
+            "node_type",
+            "node_label",
+            "parent_node_id",
+            "iteration",
+            "result",
+        )
+
+    def _get_first_node_result(self, obj):
+        results = obj.node_results.all()
+        return results[0] if results else None
+
+    @extend_schema_field(OpenApiTypes.STR)
+    def get_node_type(self, obj):
+        return obj.node.get_type().type
+
+    @extend_schema_field(OpenApiTypes.STR)
+    def get_node_label(self, obj):
+        return obj.node.label
+
+    @extend_schema_field(OpenApiTypes.INT)
+    def get_parent_node_id(self, obj):
+        parent_nodes = obj.node.get_parent_nodes()
+        if not parent_nodes:
+            return None
+        return parent_nodes[-1].id
+
+    @extend_schema_field(OpenApiTypes.INT)
+    def get_iteration(self, obj):
+        result = self._get_first_node_result(obj)
+        if result is None:
+            return None
+
+        if result.iteration_path:
+            return int(result.iteration_path.rsplit(".", 1)[-1])
+
+        return 0
+
+    def get_result(self, obj):
+        result = self._get_first_node_result(obj)
+        return result.result if result else {}
+
+
+class AutomationWorkflowHistorySerializer(AutomationHistorySerializer):
+    node_histories = AutomationNodeHistorySerializer(read_only=True, many=True)
+
+    class Meta:
+        model = AutomationWorkflowHistory
+        fields = AutomationHistorySerializer.Meta.fields + (
+            "is_test_run",
+            "event_payload",
+            "simulate_until_node",
+            "node_histories",
+        )
+
+
+class AutomationWorkflowHistoryPagination(PageNumberPagination):
+    def get_paginated_response(self, data, *, success_count: int, fail_count: int):
+        response = super().get_paginated_response(data)
+        response.data["success_count"] = success_count
+        response.data["fail_count"] = fail_count
+        return response

backend/src/baserow/contrib/automation/api/workflows/views.py

@@ -2,19 +2,20 @@
 
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Count, Q
 
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import OpenApiParameter, extend_schema
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
+from rest_framework.serializers import IntegerField
 from rest_framework.status import HTTP_202_ACCEPTED
 from rest_framework.views import APIView
 
 from baserow.api.applications.errors import ERROR_APPLICATION_DOES_NOT_EXIST
 from baserow.api.decorators import map_exceptions, validate_body
 from baserow.api.jobs.errors import ERROR_MAX_JOB_COUNT_EXCEEDED
 from baserow.api.jobs.serializers import JobSerializer
-from baserow.api.pagination import PageNumberPagination
 from baserow.api.schemas import CLIENT_SESSION_ID_SCHEMA_PARAMETER, get_error_schema
 from baserow.api.serializers import get_example_pagination_serializer_class
 from baserow.contrib.automation.api.workflows.errors import (
@@ -23,12 +24,14 @@
     ERROR_AUTOMATION_WORKFLOW_NOTIFICATION_RECIPIENTS_INVALID,
 )
 from baserow.contrib.automation.api.workflows.serializers import (
+    AutomationWorkflowHistoryPagination,
     AutomationWorkflowHistorySerializer,
     AutomationWorkflowSerializer,
     CreateAutomationWorkflowSerializer,
     OrderAutomationWorkflowsSerializer,
     UpdateAutomationWorkflowSerializer,
 )
+from baserow.contrib.automation.history.constants import HistoryStatusChoices
 from baserow.contrib.automation.history.service import AutomationHistoryService
 from baserow.contrib.automation.workflows.actions import (
     CreateAutomationWorkflowActionType,
@@ -231,7 +234,15 @@ class AutomationWorkflowHistoryView(APIView):
         description="Retrieve the history for a workflow.",
         responses={
             200: get_example_pagination_serializer_class(
-                AutomationWorkflowHistorySerializer
+                AutomationWorkflowHistorySerializer,
+                additional_fields={
+                    "success_count": IntegerField(
+                        help_text="The total number of successful workflow runs."
+                    ),
+                    "fail_count": IntegerField(
+                        help_text="The total number of failed workflow runs."
+                    ),
+                },
             ),
             404: get_error_schema(
                 [
@@ -251,16 +262,26 @@ def get(self, request, workflow_id: int):
             request.user, workflow_id
         )
 
-        paginator = PageNumberPagination(
+        counts = queryset.aggregate(
+            success_count=Count("id", filter=Q(status=HistoryStatusChoices.SUCCESS)),
+            fail_count=Count("id", filter=Q(status=HistoryStatusChoices.ERROR)),
+        )
+
+        paginator = AutomationWorkflowHistoryPagination(
             limit_page_size=settings.AUTOMATION_HISTORY_PAGE_SIZE_LIMIT
         )
+
         page = paginator.paginate_queryset(queryset, request, self)
         serializer = AutomationWorkflowHistorySerializer(
             page,
             many=True,
         )
 
-        return paginator.get_paginated_response(serializer.data)
+        return paginator.get_paginated_response(
+            serializer.data,
+            success_count=counts["success_count"],
+            fail_count=counts["fail_count"],
+        )
 
 
 class OrderAutomationWorkflowsView(APIView):

backend/src/baserow/contrib/automation/history/handler.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Dict, List, Optional, Union
 
-from django.db.models import QuerySet
+from django.db.models import Prefetch, QuerySet
 
 from baserow.contrib.automation.history.constants import HistoryStatusChoices
 from baserow.contrib.automation.history.exceptions import (
@@ -31,9 +31,18 @@ def get_workflow_histories(
             base_queryset = AutomationWorkflowHistory.objects.all()
 
         return base_queryset.filter(
-            workflow=workflow,
+            original_workflow=workflow,
             simulate_until_node__isnull=True,
-        ).prefetch_related("workflow__automation__workspace")
+        ).prefetch_related(
+            Prefetch(
+                "node_histories",
+                queryset=AutomationNodeHistory.objects.select_related(
+                    "node", "node__workflow"
+                )
+                .prefetch_related("node_results")
+                .order_by("started_on"),
+            ),
+        )
 
     def get_workflow_history(
         self, history_id: int, base_queryset: Optional[QuerySet] = None
@@ -60,6 +69,7 @@ def get_workflow_history(
 
     def create_workflow_history(
         self,
+        original_workflow: AutomationWorkflow,
         workflow: AutomationWorkflow,
         started_on: datetime,
         is_test_run: bool,
@@ -73,6 +83,7 @@ def create_workflow_history(
 
         return AutomationWorkflowHistory.objects.create(
             workflow=workflow,
+            original_workflow=original_workflow,
             started_on=started_on,
             is_test_run=is_test_run,
             simulate_until_node=simulate_until_node,

backend/src/baserow/contrib/automation/history/models.py

@@ -20,10 +20,18 @@ class Meta:
 
 
 class AutomationWorkflowHistory(AutomationHistory):
-    workflow = models.ForeignKey(
+    original_workflow = models.ForeignKey(
         "automation.AutomationWorkflow",
         on_delete=models.CASCADE,
         related_name="workflow_histories",
+        # TODO ZDM: Make non-nullable after next release and add backfill
+        # migration. See: https://github.com/baserow/baserow/issues/5236
+        null=True,
+    )
+    workflow = models.ForeignKey(
+        "automation.AutomationWorkflow",
+        on_delete=models.CASCADE,
+        related_name="cloned_workflow_histories",
     )
     simulate_until_node = models.ForeignKey(
         "automation.AutomationNode",

backend/src/baserow/contrib/automation/migrations/0028_automationworkflowhistory_original_workflow_and_more.py

@@ -0,0 +1,49 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def backfill_original_workflow(apps, schema_editor):
+    AutomationWorkflowHistory = apps.get_model(
+        "automation", "AutomationWorkflowHistory"
+    )
+    AutomationWorkflowHistory.objects.filter(original_workflow__isnull=True).update(
+        original_workflow_id=models.F("workflow_id")
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("automation", "0027_alter_automationnodehistory_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="automationworkflowhistory",
+            name="original_workflow",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="workflow_histories",
+                to="automation.automationworkflow",
+            ),
+        ),
+        migrations.RunPython(
+            backfill_original_workflow,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.AlterField(
+            model_name="automationworkflowhistory",
+            name="workflow",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="cloned_workflow_histories",
+                to="automation.automationworkflow",
+            ),
+        ),
+        migrations.AlterField(
+            model_name='automationworkflow',
+            name='state',
+            field=models.CharField(choices=[('draft', 'Draft'), ('live', 'Live'), ('paused', 'Paused'), ('disabled', 'Disabled'), ('test_clone', 'Test Clone')], db_default='draft', default='draft', max_length=20),
+        ),
+    ]

backend/src/baserow/contrib/automation/models.py

@@ -1,6 +1,11 @@
 from django.db import models
 
-from baserow.contrib.automation.history.models import AutomationWorkflowHistory
+from baserow.contrib.automation.history.models import (
+    AutomationHistory,
+    AutomationNodeHistory,
+    AutomationNodeResult,
+    AutomationWorkflowHistory,
+)
 from baserow.contrib.automation.workflows.models import (
     AutomationWorkflow,
     DuplicateAutomationWorkflowJob,
@@ -11,7 +16,10 @@
     "Automation",
     "AutomationWorkflow",
     "DuplicateAutomationWorkflowJob",
+    "AutomationHistory",
     "AutomationWorkflowHistory",
+    "AutomationNodeHistory",
+    "AutomationNodeResult",
 ]