Jawad/asana connectors

.github/workflows/deploy.yml

@@ -15,26 +15,38 @@
 # limitations under the License.
 #
 
-name: Deploy Production
+name: Deploy to Production
 
 on:
-  push:
-    tags:
-      - 'v*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version tag to deploy (e.g., v1.0.0)'
+        required: true
+        type: string
+  workflow_run:
+    workflows: ["Build Release Images"]
+    types:
+      - completed
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
-
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      - name: Extract version
-        id: vars
-        run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Get version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=${{ github.event.workflow_run.head_branch }}" >> $GITHUB_OUTPUT
+          fi
 
       - name: SSH Deploy
         uses: appleboy/ssh-action@v1.0.3
         env:
-          DEPLOY_VERSION: ${{ steps.vars.outputs.VERSION }}
+          DEPLOY_VERSION: ${{ steps.version.outputs.VERSION }}
         with:
           host: ${{ secrets.SERVER_HOST }}
           username: ${{ secrets.SERVER_USER }}
@@ -43,5 +55,7 @@ jobs:
           script: |
             cd /opt/devlake
             sed -i "s/^APP_VERSION=.*/APP_VERSION=${DEPLOY_VERSION}/" .env
-            docker pull jawad4khan/devlake:${DEPLOY_VERSION}
-            ./deploy.sh
+            sudo docker pull jawad4khan/devlake:${DEPLOY_VERSION}
+            sudo docker pull jawad4khan/devlake-config-ui:${DEPLOY_VERSION}
+            sudo ./deploy.sh
+

.github/workflows/release.yml

@@ -15,31 +15,97 @@
 # limitations under the License.
 #
 
-name: Build Release Image
+name: Build Release Images
 
 on:
   push:
     tags:
       - 'v*'
 
 jobs:
-  build-image:
+  build-backend:
+    name: Build DevLake Backend
     runs-on: ubuntu-latest
-
     steps:
+      - name: Free Disk Space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          docker system prune -af
+          docker volume prune -f
+
       - uses: actions/checkout@v4
 
       - name: Extract tag
         id: vars
-        run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+        run: |
+          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build and Push Image
+      - name: Build and Push devlake backend image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          push: true
+          tags: jawad4khan/devlake:${{ steps.vars.outputs.TAG }}
+          platforms: linux/amd64
+          cache-from: |
+            apache/devlake:amd64-builder
+            apache/devlake:base
+          build-args: |
+            TAG=${{ steps.vars.outputs.TAG }}
+            SHA=${{ steps.vars.outputs.SHORT_SHA }}
+
+  build-config-ui:
+    name: Build DevLake Config UI
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free Disk Space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          docker system prune -af
+          docker volume prune -f
+
+      - uses: actions/checkout@v4
+
+      - name: Extract tag
+        id: vars
         run: |
-          docker build -t jawad4khan/devlake:${{ steps.vars.outputs.TAG }} ./backend
-          docker push jawad4khan/devlake:${{ steps.vars.outputs.TAG }}
+          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and Push config-ui image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./config-ui
+          push: true
+          tags: jawad4khan/devlake-config-ui:${{ steps.vars.outputs.TAG }}
+          platforms: linux/amd64

AGENTS.md

@@ -130,3 +130,93 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/
 - Migration scripts must be added to `All()` in `register.go`
 - API changes require running `make swag` to update Swagger docs
 - Python plugins require `libgit2` for gitextractor functionality
+
+## Asana Plugin Implementation (2025-02-03)
+
+### Overview
+Implemented a complete Asana plugin (`backend/plugins/asana/`) to collect projects (boards), sections, and tasks from Asana's REST API and map them to DevLake's ticket/board domain model.
+
+### Architecture Decisions
+- **Scope Model**: Asana **Project** = DevLake **Board** (scope)
+- **Authentication**: Personal Access Token (PAT) via Bearer token (`Authorization: Bearer <token>`)
+- **API Base URL**: `https://app.asana.com/api/1.0/` (default endpoint)
+- **Pagination**: Asana uses offset-based pagination via `next_page.offset` in response; implemented sequential fetching with `GetNextPageCustomData`
+
+### Implementation Details
+
+#### Models (`backend/plugins/asana/models/`)
+- **Connection** (`connection.go`): `AsanaConn` (Token, RestConnection), `AsanaConnection` (BaseConnection + AsanaConn)
+- **Scope** (`project.go`): `AsanaProject` implements `ToolLayerScope`; primary key: `(ConnectionId, Gid)`
+- **Scope Config** (`scope_config.go`): `AsanaScopeConfig` embeds `common.ScopeConfig` with `Entities: ["TICKET"]`
+- **Tool Layer**:
+  - `task.go`: `AsanaTask` (Gid, Name, Notes, Completed, DueOn, ProjectGid, SectionGid, AssigneeGid, CreatorGid, etc.)
+  - `section.go`: `AsanaSection` (Gid, Name, ProjectGid)
+  - `user.go`: `AsanaUser` (Gid, Name, Email) - optional, for assignee/creator enrichment
+- **Migration**: `20250203_add_init_tables.go` creates all tables via `migrationhelper.AutoMigrateTables`
+
+#### API Layer (`backend/plugins/asana/api/`)
+- **Connection API** (`connection_api.go`): Test connection via `GET users/me`, CRUD operations, default endpoint handling
+- **Scope API** (`scope_api.go`): PutScopes, GetScopeList, GetScope, PatchScope, DeleteScope (uses `:scopeId` path param, not `:projectId`)
+- **Scope Config API** (`scope_config_api.go`): Full CRUD for scope configs
+- **Blueprint V200** (`blueprint_v200.go`): Maps Asana projects to `ticket.Board` domain entities when scope config includes `DOMAIN_TYPE_TICKET`
+- **Remote API** (`remote_api.go`): Proxy endpoint for direct API access
+- **Init** (`init.go`): Sets up `DsHelper[AsanaConnection, AsanaProject, AsanaScopeConfig]` with default endpoint
+
+#### Tasks (`backend/plugins/asana/tasks/`)
+- **Project**: `project_collector.go` (GET `/projects/{gid}`), `project_extractor.go` (extracts to `_tool_asana_projects`)
+- **Section**: `section_collector.go` (GET `/projects/{gid}/sections`), `section_extractor.go` (extracts to `_tool_asana_sections`)
+- **Task**: 
+  - `task_collector.go`: Collects tasks with offset pagination (limit=100, offset from `next_page.offset`)
+  - `task_extractor.go`: Extracts task data including memberships (project/section), assignee, creator, parent
+  - `task_convertor.go`: Converts `AsanaTask` → `ticket.Issue` + `ticket.BoardIssue` using `didgen` for domain IDs
+- **API Client** (`api_client.go`): Creates `ApiAsyncClient` with Bearer auth, sets default endpoint if missing
+- **Task Data** (`task_data.go`): `AsanaOptions` (ConnectionId, ProjectId, ScopeConfigId), `AsanaTaskData`, `CreateRawDataSubTaskArgs` helper
+
+#### Plugin Entry (`backend/plugins/asana/`)
+- **Main** (`asana.go`): `PluginEntry impl.Asana` for plugin loading
+- **Impl** (`impl/impl.go`): Implements all required interfaces:
+  - `PluginMeta`: Name="asana", Description, RootPkgPath
+  - `PluginTask`: SubTaskMetas (CollectProject, ExtractProject, CollectSection, ExtractSection, CollectTask, ExtractTask, ConvertTask)
+  - `PluginModel`: GetTablesInfo() returns all 6 models
+  - `PluginMigration`: MigrationScripts() from migrationscripts.All()
+  - `PluginApi`: ApiResources() with connections, scopes, scope-configs, test, proxy routes
+  - `PluginSource`: Connection(), Scope(), ScopeConfig()
+  - `DataSourcePluginBlueprintV200`: MakeDataSourcePipelinePlanV200()
+  - `CloseablePluginTask`: Close() releases ApiClient
+
+#### Config UI (`config-ui/src/plugins/register/asana/`)
+- **Config** (`config.tsx`): `AsanaConfig` with:
+  - Connection fields: name, endpoint (default: `https://app.asana.com/api/1.0/`), token, proxy, rateLimitPerHour (default: 150)
+  - Data scope title: "Projects"
+  - Scope config entities: `['TICKET']`
+- **Icon** (`assets/icon.svg`): Placeholder SVG icon
+- **Registration** (`index.ts`): Exports `AsanaConfig`
+- **Plugin Registry** (`config-ui/src/plugins/register/index.ts`): Added `AsanaConfig` to `pluginConfigs` array
+
+#### Testing
+- **Table Info Test** (`backend/plugins/table_info_test.go`): Added `asana` import and `checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo)`
+- **E2E Test** (`backend/plugins/asana/e2e/task_test.go`): 
+  - Imports raw task CSV (`e2e/raw_tables/_raw_asana_tasks.csv`)
+  - Runs `ExtractTaskMeta` subtask
+  - Verifies `_tool_asana_tasks` against snapshot (`e2e/snapshot_tables/_tool_asana_tasks.csv`)
+
+### Key Implementation Notes
+1. **Scope ID**: Uses `:scopeId` path param (not `:projectId`) to match generic scope helper expectations; scope ID is Asana project GID
+2. **Response Parsing**: Asana API wraps responses in `{"data": {...}}` or `{"data": [...], "next_page": {...}}`; collectors unwrap `data` field
+3. **Date Parsing**: `due_on` is date-only string (`YYYY-MM-DD`); `parseAsanaDate()` helper converts to `*time.Time`
+4. **Task Memberships**: Tasks can belong to multiple projects/sections; extractor uses first section from memberships array
+5. **Domain Conversion**: Task status maps: `completed=true` → `ticket.DONE`, `completed=false` → `ticket.TODO`
+6. **Default Endpoint**: Set in `api/init.go` constant and applied in `PostConnections` if missing from request
+
+### Files Created
+- **Backend**: 25+ Go files across `models/`, `api/`, `tasks/`, `impl/`, `e2e/`
+- **Config UI**: 3 TypeScript files (`config.tsx`, `index.ts`, `assets/icon.svg`)
+- **Test**: 1 CSV fixture pair (raw + snapshot), 1 E2E test file
+- **CI**: Updated `table_info_test.go`
+
+### Next Steps (Optional Enhancements)
+- Add user collection/enrichment for assignee/creator names
+- Support OAuth 2.0 authentication (currently PAT only)
+- Add task comments/subtasks collection
+- Implement incremental collection with time-based bookmarking
+- Add transformation rules for custom field mappings

backend/plugins/asana/api/blueprint_v200.go

@@ -0,0 +1,101 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+
+	"github.com/apache/incubator-devlake/plugins/asana/models"
+	"github.com/apache/incubator-devlake/plugins/asana/tasks"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+)
+
+func MakePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+	plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	return plan, scopes, err
+}
+
+func makePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig],
+	connection *models.AsanaConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+		scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		task, err := helper.MakePipelinePlanTask(
+			"asana",
+			subtaskMetas,
+			scopeConfig.Entities,
+			tasks.AsanaOptions{
+				ConnectionId:  connection.ID,
+				ProjectId:     scope.Gid,
+				ScopeConfigId: scopeConfig.ID,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, task)
+		plan[i] = stage
+	}
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig],
+	connection *models.AsanaConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0, len(scopeDetails))
+	idgen := didgen.NewDomainIdGenerator(&models.AsanaProject{})
+	for _, scopeDetail := range scopeDetails {
+		scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		id := idgen.Generate(connection.ID, scope.Gid)
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
+			scopes = append(scopes, ticket.NewBoard(id, scope.Name))
+		}
+	}
+	return scopes, nil
+}