From 15e6f59f7144e80af2912da44ce84f8c60d7e855 Mon Sep 17 00:00:00 2001 From: jawad khan Date: Tue, 17 Feb 2026 20:08:05 +0500 Subject: [PATCH 01/10] Production (#6) * fix: Fixed workflow * fix: fixed workflow * feat: Added config ui * fix: fixed test cases --- .github/workflows/deploy.yml | 36 ++++++--- .github/workflows/release.yml | 80 +++++++++++++++++-- backend/plugins/developer_telemetry/README.md | 17 ++++ backend/plugins/table_info_test.go | 2 + 4 files changed, 117 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7eeac3f3e8f..36cc639505b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,26 +15,38 @@ # limitations under the License. # -name: Deploy Production +name: Deploy to Production on: - push: - tags: - - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version tag to deploy (e.g., v1.0.0)' + required: true + type: string + workflow_run: + workflows: ["Build Release Images"] + types: + - completed jobs: deploy: runs-on: ubuntu-latest - + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: - - name: Extract version - id: vars - run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + - name: Get version + id: version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + else + echo "VERSION=${{ github.event.workflow_run.head_branch }}" >> $GITHUB_OUTPUT + fi - name: SSH Deploy uses: appleboy/ssh-action@v1.0.3 env: - DEPLOY_VERSION: ${{ steps.vars.outputs.VERSION }} + DEPLOY_VERSION: ${{ steps.version.outputs.VERSION }} with: host: ${{ secrets.SERVER_HOST }} username: ${{ secrets.SERVER_USER }} @@ -43,5 +55,7 @@ jobs: script: | cd /opt/devlake sed -i "s/^APP_VERSION=.*/APP_VERSION=${DEPLOY_VERSION}/" .env - docker pull jawad4khan/devlake:${DEPLOY_VERSION} - ./deploy.sh + sudo docker pull jawad4khan/devlake:${DEPLOY_VERSION} + sudo docker pull jawad4khan/devlake-config-ui:${DEPLOY_VERSION} + sudo ./deploy.sh + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19811429863..d4cce3479ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ # limitations under the License. # -name: Build Release Image +name: Build Release Images on: push: @@ -23,15 +23,32 @@ on: - 'v*' jobs: - build-image: + build-backend: + name: Build DevLake Backend runs-on: ubuntu-latest - steps: + - name: Free Disk Space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + docker system prune -af + docker volume prune -f + - uses: actions/checkout@v4 - name: Extract tag id: vars - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + run: | + echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub uses: docker/login-action@v3 @@ -39,7 +56,56 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push Image + - name: Build and Push devlake backend image + uses: docker/build-push-action@v5 + with: + context: ./backend + push: true + tags: jawad4khan/devlake:${{ steps.vars.outputs.TAG }} + platforms: linux/amd64 + cache-from: | + apache/devlake:amd64-builder + apache/devlake:base + build-args: | + TAG=${{ steps.vars.outputs.TAG }} + SHA=${{ steps.vars.outputs.SHORT_SHA }} + + build-config-ui: + name: Build DevLake Config UI + runs-on: ubuntu-latest + steps: + - name: Free Disk Space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + docker system prune -af + docker volume prune -f + + - uses: actions/checkout@v4 + + - name: Extract tag + id: vars run: | - docker build -t jawad4khan/devlake:${{ steps.vars.outputs.TAG }} ./backend - docker push jawad4khan/devlake:${{ steps.vars.outputs.TAG }} + echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and Push config-ui image + uses: docker/build-push-action@v5 + with: + context: ./config-ui + push: true + tags: jawad4khan/devlake-config-ui:${{ steps.vars.outputs.TAG }} + platforms: linux/amd64 diff --git a/backend/plugins/developer_telemetry/README.md b/backend/plugins/developer_telemetry/README.md index 8e5c73beb75..33f047f7f3a 100644 --- a/backend/plugins/developer_telemetry/README.md +++ b/backend/plugins/developer_telemetry/README.md @@ -1,3 +1,20 @@ + + # Developer Telemetry Plugin for Apache DevLake ## Overview diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 3bf09037255..ad1b7a27376 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -30,6 +30,7 @@ import ( circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" + developer_telemetry "github.com/apache/incubator-devlake/plugins/developer_telemetry/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" @@ -71,6 +72,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("argocd/models", argocd.ArgoCD{}.GetTablesInfo) checker.FeedIn("customize/models", customize.Customize{}.GetTablesInfo) checker.FeedIn("dbt", dbt.Dbt{}.GetTablesInfo) + checker.FeedIn("developer_telemetry/models", developer_telemetry.DeveloperTelemetry{}.GetTablesInfo) checker.FeedIn("dora/models", dora.Dora{}.GetTablesInfo) checker.FeedIn("feishu/models", feishu.Feishu{}.GetTablesInfo) checker.FeedIn("gitee/models", gitee.Gitee{}.GetTablesInfo) From 3fd25327421bdc923f2219635ba6c887e1daf7cc Mon Sep 17 00:00:00 2001 From: irfanuddinahmad Date: Wed, 18 Feb 2026 11:05:10 +0500 Subject: [PATCH 02/10] feat(developer_telemetry): Add API key authentication support - Add apikeyhelper to manage API keys for developer telemetry connections - Automatically generate and return API key when creating a connection - Implement API key deletion when connection is deleted - Use transactions to ensure atomicity of connection and API key operations - API key authentication can be used via /rest/plugins/developer_telemetry/* endpoints - Follows the same pattern as the built-in webhook plugin The API key is returned in the connection creation response with: - Secure 128-character random key - Scoped access pattern: /plugins/developer_telemetry/connections/{id}/.* - Proper metadata (creator, expiration, allowed path) This enables secure authenticated access to telemetry endpoints using DevLake's existing API key infrastructure and middleware. --- .../developer_telemetry/api/connection_api.go | 75 ++++++++++++++++++- .../plugins/developer_telemetry/api/init.go | 8 ++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/backend/plugins/developer_telemetry/api/connection_api.go b/backend/plugins/developer_telemetry/api/connection_api.go index 5d70cc456cd..5d9fbcfdc62 100644 --- a/backend/plugins/developer_telemetry/api/connection_api.go +++ b/backend/plugins/developer_telemetry/api/connection_api.go @@ -18,9 +18,13 @@ limitations under the License. package api import ( + "fmt" "net/http" + "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + coremodels "github.com/apache/incubator-devlake/core/models" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/plugins/developer_telemetry/models" "github.com/apache/incubator-devlake/server/api/shared" @@ -31,6 +35,11 @@ type DeveloperTelemetryTestConnResponse struct { Connection *models.DeveloperTelemetryConnection `json:"connection"` } +type DeveloperTelemetryConnectionResponse struct { + *models.DeveloperTelemetryConnection + ApiKey *coremodels.ApiKey `json:"apiKey,omitempty"` +} + func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.DeveloperTelemetryConnection{} err := connectionHelper.Create(connection, input) @@ -50,11 +59,40 @@ func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.DeveloperTelemetryConnection{} - err := connectionHelper.Create(connection, input) + tx := basicRes.GetDal().Begin() + err := connectionHelper.CreateWithTx(tx, connection, input) if err != nil { + if err := tx.Rollback(); err != nil { + logger.Error(err, "transaction Rollback") + } + if strings.Contains(err.Error(), "the connection name already exists (400)") { + return nil, errors.BadInput.New(fmt.Sprintf("A developer telemetry connection with name %s already exists.", connection.Name)) + } return nil, err } - return &plugin.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil + logger.Info("connection: %+v", connection) + name := apiKeyHelper.GenApiKeyNameForPlugin(pluginName, connection.ID) + allowedPath := fmt.Sprintf("/plugins/%s/connections/%d/.*", pluginName, connection.ID) + extra := fmt.Sprintf("connectionId:%d", connection.ID) + apiKeyRecord, err := apiKeyHelper.CreateForPlugin(tx, input.User, name, pluginName, allowedPath, extra) + if err != nil { + if err := tx.Rollback(); err != nil { + logger.Error(err, "transaction Rollback") + } + logger.Error(err, "CreateForPlugin") + return nil, err + } + if err := tx.Commit(); err != nil { + logger.Info("transaction commit: %s", err) + } + + response := &DeveloperTelemetryConnectionResponse{ + DeveloperTelemetryConnection: connection, + ApiKey: apiKeyRecord, + } + logger.Info("api output connection: %+v", response) + + return &plugin.ApiResourceOutput{Body: response, Status: http.StatusOK}, nil } func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { @@ -68,7 +106,38 @@ func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.DeveloperTelemetryConnection{} - return connectionHelper.Delete(connection, input) + err := connectionHelper.First(connection, input.Params) + if err != nil { + logger.Error(err, "query connection") + return nil, err + } + + tx := basicRes.GetDal().Begin() + + err = tx.Delete(connection, dal.Where("id = ?", connection.ID)) + if err != nil { + if err := tx.Rollback(); err != nil { + logger.Error(err, "transaction Rollback") + } + logger.Error(err, "delete connection: %d", connection.ID) + return nil, err + } + + extra := fmt.Sprintf("connectionId:%d", connection.ID) + err = apiKeyHelper.DeleteForPlugin(tx, pluginName, extra) + if err != nil { + if err := tx.Rollback(); err != nil { + logger.Error(err, "transaction Rollback") + } + logger.Error(err, "DeleteForPlugin") + return nil, err + } + + if err := tx.Commit(); err != nil { + logger.Info("transaction commit: %s", err) + } + + return &plugin.ApiResourceOutput{Status: http.StatusOK}, nil } func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { diff --git a/backend/plugins/developer_telemetry/api/init.go b/backend/plugins/developer_telemetry/api/init.go index 062ccc09f9d..9db7b476790 100644 --- a/backend/plugins/developer_telemetry/api/init.go +++ b/backend/plugins/developer_telemetry/api/init.go @@ -19,17 +19,25 @@ package api import ( "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/apikeyhelper" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/go-playground/validator/v10" ) +const pluginName = "developer_telemetry" + var basicRes context.BasicRes var connectionHelper *api.ConnectionApiHelper +var apiKeyHelper *apikeyhelper.ApiKeyHelper var vld *validator.Validate +var logger log.Logger func Init(br context.BasicRes, pm plugin.PluginMeta) { basicRes = br + logger = basicRes.GetLogger() vld = validator.New() connectionHelper = api.NewConnectionHelper(br, vld, pm.Name()) + apiKeyHelper = apikeyhelper.NewApiKeyHelper(basicRes, logger) } From 3f0efa11895c4dd4b585b6fe1f21cd22e83a5c4d Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Fri, 6 Feb 2026 17:28:35 +0500 Subject: [PATCH 03/10] feat: Added asana plugin --- backend/plugins/asana/api/blueprint_v200.go | 101 +++++++++ backend/plugins/asana/api/connection_api.go | 124 ++++++++++++ backend/plugins/asana/api/init.go | 52 +++++ backend/plugins/asana/api/remote_api.go | 27 +++ backend/plugins/asana/api/scope_api.go | 48 +++++ backend/plugins/asana/api/scope_config_api.go | 47 +++++ backend/plugins/asana/api/swagger.go | 32 +++ backend/plugins/asana/asana.go | 42 ++++ .../asana/e2e/raw_tables/_raw_asana_tasks.csv | 2 + .../e2e/snapshot_tables/_tool_asana_tasks.csv | 2 + backend/plugins/asana/e2e/task_test.go | 49 +++++ backend/plugins/asana/impl/impl.go | 191 ++++++++++++++++++ backend/plugins/asana/models/connection.go | 71 +++++++ .../20250203_add_init_tables.go | 47 +++++ .../asana/models/migrationscripts/register.go | 29 +++ backend/plugins/asana/models/project.go | 64 ++++++ backend/plugins/asana/models/scope_config.go | 35 ++++ backend/plugins/asana/models/section.go | 35 ++++ backend/plugins/asana/models/task.go | 52 +++++ backend/plugins/asana/models/user.go | 35 ++++ backend/plugins/asana/tasks/api_client.go | 40 ++++ .../plugins/asana/tasks/project_collector.go | 74 +++++++ .../plugins/asana/tasks/project_extractor.go | 87 ++++++++ .../plugins/asana/tasks/section_collector.go | 71 +++++++ .../plugins/asana/tasks/section_extractor.go | 82 ++++++++ backend/plugins/asana/tasks/task_collector.go | 99 +++++++++ backend/plugins/asana/tasks/task_convertor.go | 101 +++++++++ backend/plugins/asana/tasks/task_data.go | 49 +++++ backend/plugins/asana/tasks/task_extractor.go | 157 ++++++++++++++ 29 files changed, 1845 insertions(+) create mode 100644 backend/plugins/asana/api/blueprint_v200.go create mode 100644 backend/plugins/asana/api/connection_api.go create mode 100644 backend/plugins/asana/api/init.go create mode 100644 backend/plugins/asana/api/remote_api.go create mode 100644 backend/plugins/asana/api/scope_api.go create mode 100644 backend/plugins/asana/api/scope_config_api.go create mode 100644 backend/plugins/asana/api/swagger.go create mode 100644 backend/plugins/asana/asana.go create mode 100644 backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv create mode 100644 backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv create mode 100644 backend/plugins/asana/e2e/task_test.go create mode 100644 backend/plugins/asana/impl/impl.go create mode 100644 backend/plugins/asana/models/connection.go create mode 100644 backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go create mode 100644 backend/plugins/asana/models/migrationscripts/register.go create mode 100644 backend/plugins/asana/models/project.go create mode 100644 backend/plugins/asana/models/scope_config.go create mode 100644 backend/plugins/asana/models/section.go create mode 100644 backend/plugins/asana/models/task.go create mode 100644 backend/plugins/asana/models/user.go create mode 100644 backend/plugins/asana/tasks/api_client.go create mode 100644 backend/plugins/asana/tasks/project_collector.go create mode 100644 backend/plugins/asana/tasks/project_extractor.go create mode 100644 backend/plugins/asana/tasks/section_collector.go create mode 100644 backend/plugins/asana/tasks/section_extractor.go create mode 100644 backend/plugins/asana/tasks/task_collector.go create mode 100644 backend/plugins/asana/tasks/task_convertor.go create mode 100644 backend/plugins/asana/tasks/task_data.go create mode 100644 backend/plugins/asana/tasks/task_extractor.go diff --git a/backend/plugins/asana/api/blueprint_v200.go b/backend/plugins/asana/api/blueprint_v200.go new file mode 100644 index 00000000000..546fd12ab21 --- /dev/null +++ b/backend/plugins/asana/api/blueprint_v200.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/helpers/srvhelper" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig], + connection *models.AsanaConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "asana", + subtaskMetas, + scopeConfig.Entities, + tasks.AsanaOptions{ + ConnectionId: connection.ID, + ProjectId: scope.Gid, + ScopeConfigId: scopeConfig.ID, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig], + connection *models.AsanaConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.Gid) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/asana/api/connection_api.go b/backend/plugins/asana/api/connection_api.go new file mode 100644 index 00000000000..6d69c8c4ee4 --- /dev/null +++ b/backend/plugins/asana/api/connection_api.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type AsanaTestConnResponse struct { + shared.ApiBody + Connection *models.AsanaConn +} + +func testConnection(ctx context.Context, connection models.AsanaConn) (*AsanaTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.GetEndpoint() == "" { + connection.Endpoint = defaultAsanaEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + res, err := apiClient.Get("users/me", nil, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized { + return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error while testing connection") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := AsanaTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.AsanaConn + err := helper.Decode(input.Body, &connection, vld) + if err != nil { + return nil, err + } + if connection.Endpoint == "" { + connection.Endpoint = defaultAsanaEndpoint + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + if connection.Endpoint == "" { + connection.Endpoint = defaultAsanaEndpoint + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.AsanaConn) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return &plugin.ApiResourceOutput{Body: testConnectionResult, Status: http.StatusOK}, nil +} + +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + if input.Body != nil { + if endpoint, ok := input.Body["endpoint"]; !ok || endpoint == nil || endpoint == "" { + input.Body["endpoint"] = defaultAsanaEndpoint + } + } + return dsHelper.ConnApi.Post(input) +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/asana/api/init.go b/backend/plugins/asana/api/init.go new file mode 100644 index 00000000000..1ec8740d177 --- /dev/null +++ b/backend/plugins/asana/api/init.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/go-playground/validator/v10" +) + +const defaultAsanaEndpoint = "https://app.asana.com/api/1.0/" + +var vld *validator.Validate +var basicRes context.BasicRes + +var dsHelper *api.DsHelper[models.AsanaConnection, models.AsanaProject, models.AsanaScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.AsanaConnection] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.AsanaConnection, models.AsanaProject, models.AsanaScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.AsanaConnection) models.AsanaConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.AsanaConnection](dsHelper.ConnApi.ModelApiHelper) +} diff --git a/backend/plugins/asana/api/remote_api.go b/backend/plugins/asana/api/remote_api.go new file mode 100644 index 00000000000..7f84420e6b4 --- /dev/null +++ b/backend/plugins/asana/api/remote_api.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/asana/api/scope_api.go b/backend/plugins/asana/api/scope_api.go new file mode 100644 index 00000000000..afbc8486583 --- /dev/null +++ b/backend/plugins/asana/api/scope_api.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.AsanaProject] +type ScopeDetail api.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig] + +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/asana/api/scope_config_api.go b/backend/plugins/asana/api/scope_config_api.go new file mode 100644 index 00000000000..d7e5b541b11 --- /dev/null +++ b/backend/plugins/asana/api/scope_config_api.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} + +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/asana/api/swagger.go b/backend/plugins/asana/api/swagger.go new file mode 100644 index 00000000000..43897adf9ba --- /dev/null +++ b/backend/plugins/asana/api/swagger.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +type AsanaTaskOptions tasks.AsanaOptions + +// @Summary asana task options for pipelines +// @Description Task options for asana pipelines +// @Tags plugins/asana +// @Accept application/json +// @Param pipeline body AsanaTaskOptions true "json" +// @Router /pipelines/asana/pipeline-task [post] +func _() {} diff --git a/backend/plugins/asana/asana.go b/backend/plugins/asana/asana.go new file mode 100644 index 00000000000..4bfbbf5b098 --- /dev/null +++ b/backend/plugins/asana/asana.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Asana //nolint + +func main() { + cmd := &cobra.Command{Use: "asana"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "asana connection id") + projectId := cmd.Flags().StringP("project", "p", "", "asana project gid") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("project") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "projectId": *projectId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv b/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv new file mode 100644 index 00000000000..59815e7308c --- /dev/null +++ b/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":""123456789""}","{""gid"":""987654321"",""name"":""Test Task"",""notes"":""Notes here"",""resource_type"":""task"",""resource_subtype"":""default_task"",""completed"":false,""due_on"":""2025-02-15"",""created_at"":""2025-02-01T10:00:00.000Z"",""permalink_url"":""https://app.asana.com/0/123/987654321"",""memberships"":[{""project"":{""gid"":""123456789""},""section"":{""gid"":""111""}}]}",https://app.asana.com/api/1.0/projects/123456789/tasks,null,2025-02-03 12:00:00 diff --git a/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv new file mode 100644 index 00000000000..6d09c271cbf --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv @@ -0,0 +1,2 @@ +connection_id,gid,name,notes,resource_type,resource_subtype,completed,project_gid,section_gid,permalink_url,num_subtasks +1,987654321,Test Task,Notes here,task,default_task,0,123456789,111,https://app.asana.com/0/123/987654321,0 diff --git a/backend/plugins/asana/e2e/task_test.go b/backend/plugins/asana/e2e/task_test.go new file mode 100644 index 00000000000..f51433063dd --- /dev/null +++ b/backend/plugins/asana/e2e/task_test.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaTaskDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "123456789", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_tasks.csv", "_raw_asana_tasks") + + dataflowTester.FlushTabler(&models.AsanaTask{}) + dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.AsanaTask{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_tasks.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/asana/impl/impl.go b/backend/plugins/asana/impl/impl.go new file mode 100644 index 00000000000..f92d0f2fd80 --- /dev/null +++ b/backend/plugins/asana/impl/impl.go @@ -0,0 +1,191 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +var _ interface { + plugin.PluginTask + plugin.PluginMeta + plugin.PluginInit + plugin.PluginApi + plugin.PluginModel + plugin.PluginMigration + plugin.PluginSource + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Asana)(nil) + +type Asana struct{} + +func (p Asana) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Asana) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.AsanaConnection{}, + &models.AsanaProject{}, + &models.AsanaScopeConfig{}, + &models.AsanaTask{}, + &models.AsanaSection{}, + &models.AsanaUser{}, + } +} + +func (p Asana) Description() string { + return "To collect and enrich data from Asana" +} + +func (p Asana) Name() string { + return "asana" +} + +func (p Asana) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectProjectMeta, + tasks.ExtractProjectMeta, + tasks.CollectSectionMeta, + tasks.ExtractSectionMeta, + tasks.CollectTaskMeta, + tasks.ExtractTaskMeta, + tasks.ConvertTaskMeta, + } +} + +func (p Asana) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.AsanaOptions + err := helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "Asana plugin could not decode options") + } + if op.ProjectId == "" { + return nil, errors.BadInput.New("asana projectId is required") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("asana connectionId is invalid") + } + connection := &models.AsanaConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Asana plugin") + } + apiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + return &tasks.AsanaTaskData{ + Options: &op, + ApiClient: apiClient, + }, nil +} + +func (p Asana) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/asana" +} + +func (p Asana) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Asana) Connection() dal.Tabler { + return &models.AsanaConnection{} +} + +func (p Asana) Scope() plugin.ToolLayerScope { + return &models.AsanaProject{} +} + +func (p Asana) ScopeConfig() dal.Tabler { + return &models.AsanaScopeConfig{} +} + +func (p Asana) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Asana) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Asana) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.AsanaTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/asana/models/connection.go b/backend/plugins/asana/models/connection.go new file mode 100644 index 00000000000..ecd49f1aa7a --- /dev/null +++ b/backend/plugins/asana/models/connection.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// AsanaConn holds the essential information to connect to the Asana API +type AsanaConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" json:"token" encrypt:"yes"` +} + +func (ac *AsanaConn) Sanitize() AsanaConn { + ac.Token = utils.SanitizeString(ac.Token) + return *ac +} + +// AsanaConnection holds AsanaConn plus ID/Name for database storage +type AsanaConnection struct { + helper.BaseConnection `mapstructure:",squash"` + AsanaConn `mapstructure:",squash"` +} + +func (connection *AsanaConnection) MergeFromRequest(target *AsanaConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (connection AsanaConnection) Sanitize() AsanaConnection { + connection.AsanaConn = connection.AsanaConn.Sanitize() + return connection +} + +// SetupAuthentication sets up the HTTP Request Authentication (Bearer token) +func (ac *AsanaConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ac.Token)) + return nil +} + +func (AsanaConnection) TableName() string { + return "_tool_asana_connections" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go new file mode 100644 index 00000000000..8816068d044 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.AsanaConnection{}, + &models.AsanaProject{}, + &models.AsanaScopeConfig{}, + &models.AsanaTask{}, + &models.AsanaSection{}, + &models.AsanaUser{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20250203000001 +} + +func (*addInitTables) Name() string { + return "asana init schemas" +} diff --git a/backend/plugins/asana/models/migrationscripts/register.go b/backend/plugins/asana/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/asana/models/project.go b/backend/plugins/asana/models/project.go new file mode 100644 index 00000000000..9be242dbe4a --- /dev/null +++ b/backend/plugins/asana/models/project.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*AsanaProject)(nil) + +type AsanaProject struct { + common.Scope `mapstructure:",squash"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"` + Gid string `json:"gid" mapstructure:"gid" gorm:"type:varchar(255);primaryKey"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + ResourceType string `json:"resourceType" mapstructure:"resourceType" gorm:"type:varchar(32)"` + Archived bool `json:"archived" mapstructure:"archived"` + WorkspaceGid string `json:"workspaceGid" mapstructure:"workspaceGid" gorm:"type:varchar(255)"` + PermalinkUrl string `json:"permalinkUrl" mapstructure:"permalinkUrl" gorm:"type:varchar(512)"` +} + +func (p AsanaProject) ScopeId() string { + return p.Gid +} + +func (p AsanaProject) ScopeName() string { + return p.Name +} + +func (p AsanaProject) ScopeFullName() string { + return p.Name +} + +func (p AsanaProject) ScopeParams() interface{} { + return &AsanaApiParams{ + ConnectionId: p.ConnectionId, + ProjectId: p.Gid, + } +} + +func (AsanaProject) TableName() string { + return "_tool_asana_projects" +} + +type AsanaApiParams struct { + ConnectionId uint64 + ProjectId string +} diff --git a/backend/plugins/asana/models/scope_config.go b/backend/plugins/asana/models/scope_config.go new file mode 100644 index 00000000000..6c2666c0713 --- /dev/null +++ b/backend/plugins/asana/models/scope_config.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (AsanaScopeConfig) TableName() string { + return "_tool_asana_scope_configs" +} + +func (a *AsanaScopeConfig) SetConnectionId(c *AsanaScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/asana/models/section.go b/backend/plugins/asana/models/section.go new file mode 100644 index 00000000000..3ebc17b8a1b --- /dev/null +++ b/backend/plugins/asana/models/section.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaSection struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ProjectGid string `gorm:"type:varchar(255);index"` + common.NoPKModel +} + +func (AsanaSection) TableName() string { + return "_tool_asana_sections" +} diff --git a/backend/plugins/asana/models/task.go b/backend/plugins/asana/models/task.go new file mode 100644 index 00000000000..5c64a319629 --- /dev/null +++ b/backend/plugins/asana/models/task.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(512)"` + Notes string `gorm:"type:text"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completedAt"` + DueOn *time.Time `gorm:"type:date" json:"dueOn"` + CreatedAt time.Time `json:"createdAt"` + ModifiedAt *time.Time `json:"modifiedAt"` + PermalinkUrl string `gorm:"type:varchar(512)"` + ProjectGid string `gorm:"type:varchar(255);index"` + SectionGid string `gorm:"type:varchar(255);index"` + AssigneeGid string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatorGid string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + ParentGid string `gorm:"type:varchar(255);index"` + NumSubtasks int `json:"numSubtasks"` + common.NoPKModel +} + +func (AsanaTask) TableName() string { + return "_tool_asana_tasks" +} diff --git a/backend/plugins/asana/models/user.go b/backend/plugins/asana/models/user.go new file mode 100644 index 00000000000..242be27a79e --- /dev/null +++ b/backend/plugins/asana/models/user.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + common.NoPKModel +} + +func (AsanaUser) TableName() string { + return "_tool_asana_users" +} diff --git a/backend/plugins/asana/tasks/api_client.go b/backend/plugins/asana/tasks/api_client.go new file mode 100644 index 00000000000..25abd421bdc --- /dev/null +++ b/backend/plugins/asana/tasks/api_client.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.AsanaConnection) (*api.ApiAsyncClient, errors.Error) { + if connection.GetEndpoint() == "" { + connection.Endpoint = "https://app.asana.com/api/1.0/" + } + apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + asyncApiClient, err := api.CreateAsyncApiClient(taskCtx, apiClient, nil) + if err != nil { + return nil, err + } + return asyncApiClient, nil +} diff --git a/backend/plugins/asana/tasks/project_collector.go b/backend/plugins/asana/tasks/project_collector.go new file mode 100644 index 00000000000..0155af9ecf7 --- /dev/null +++ b/backend/plugins/asana/tasks/project_collector.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawProjectTable = "asana_projects" + +var _ plugin.SubTaskEntryPoint = CollectProject + +var CollectProjectMeta = plugin.SubTaskMeta{ + Name: "CollectProject", + EntryPoint: CollectProject, + EnabledByDefault: true, + Description: "Collect project data from Asana API", + DomainTypes: []string{}, +} + +type asanaDataWrapper struct { + Data json.RawMessage `json:"data"` +} + +func CollectProject(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawProjectTable, + }, + ApiClient: data.ApiClient, + UrlTemplate: "projects/{{ .Params.ProjectId }}", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaDataWrapper + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + if len(w.Data) == 0 { + return nil, nil + } + return []json.RawMessage{w.Data}, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/project_extractor.go b/backend/plugins/asana/tasks/project_extractor.go new file mode 100644 index 00000000000..5eb1734fe2c --- /dev/null +++ b/backend/plugins/asana/tasks/project_extractor.go @@ -0,0 +1,87 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProject + +var ExtractProjectMeta = plugin.SubTaskMeta{ + Name: "ExtractProject", + EntryPoint: ExtractProject, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_projects", +} + +type asanaApiProject struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Archived bool `json:"archived"` + PermalinkUrl string `json:"permalink_url"` + Workspace *struct { + Gid string `json:"gid"` + } `json:"workspace"` +} + +func ExtractProject(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawProjectTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiProject := &asanaApiProject{} + err := errors.Convert(json.Unmarshal(resData.Data, apiProject)) + if err != nil { + return nil, err + } + workspaceGid := "" + if apiProject.Workspace != nil { + workspaceGid = apiProject.Workspace.Gid + } + toolProject := &models.AsanaProject{ + ConnectionId: taskData.Options.ConnectionId, + ScopeConfigId: taskData.Options.ScopeConfigId, + Gid: apiProject.Gid, + Name: apiProject.Name, + ResourceType: apiProject.ResourceType, + Archived: apiProject.Archived, + PermalinkUrl: apiProject.PermalinkUrl, + WorkspaceGid: workspaceGid, + } + return []interface{}{toolProject}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/section_collector.go b/backend/plugins/asana/tasks/section_collector.go new file mode 100644 index 00000000000..0051b5380af --- /dev/null +++ b/backend/plugins/asana/tasks/section_collector.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawSectionTable = "asana_sections" + +var _ plugin.SubTaskEntryPoint = CollectSection + +var CollectSectionMeta = plugin.SubTaskMeta{ + Name: "CollectSection", + EntryPoint: CollectSection, + EnabledByDefault: true, + Description: "Collect section data from Asana API", + DomainTypes: []string{}, +} + +type asanaListWrapper struct { + Data []json.RawMessage `json:"data"` +} + +func CollectSection(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawSectionTable, + }, + ApiClient: data.ApiClient, + UrlTemplate: "projects/{{ .Params.ProjectId }}/sections", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaListWrapper + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + return w.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/section_extractor.go b/backend/plugins/asana/tasks/section_extractor.go new file mode 100644 index 00000000000..fe06daf287f --- /dev/null +++ b/backend/plugins/asana/tasks/section_extractor.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractSection + +var ExtractSectionMeta = plugin.SubTaskMeta{ + Name: "ExtractSection", + EntryPoint: ExtractSection, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_sections", +} + +type asanaApiSection struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Project *struct { + Gid string `json:"gid"` + } `json:"project"` +} + +func ExtractSection(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawSectionTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiSection := &asanaApiSection{} + err := errors.Convert(json.Unmarshal(resData.Data, apiSection)) + if err != nil { + return nil, err + } + projectGid := taskData.Options.ProjectId + if apiSection.Project != nil { + projectGid = apiSection.Project.Gid + } + toolSection := &models.AsanaSection{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiSection.Gid, + Name: apiSection.Name, + ResourceType: apiSection.ResourceType, + ProjectGid: projectGid, + } + return []interface{}{toolSection}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go new file mode 100644 index 00000000000..42fe95d3521 --- /dev/null +++ b/backend/plugins/asana/tasks/task_collector.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawTaskTable = "asana_tasks" + +var _ plugin.SubTaskEntryPoint = CollectTask + +var CollectTaskMeta = plugin.SubTaskMeta{ + Name: "CollectTask", + EntryPoint: CollectTask, + EnabledByDefault: true, + Description: "Collect task data from Asana API", + DomainTypes: []string{}, +} + +type asanaTaskListResponse struct { + Data []json.RawMessage `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} + +func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawTaskTable, + }, + ApiClient: data.ApiClient, + PageSize: 100, + UrlTemplate: "projects/{{ .Params.ProjectId }}/tasks", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("limit", "100") + if reqData.CustomData != nil { + if offset, ok := reqData.CustomData.(string); ok && offset != "" { + query.Set("offset", offset) + } + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp asanaTaskListResponse + err := api.UnmarshalResponse(prevPageResponse, &resp) + if err != nil { + return nil, err + } + if resp.NextPage != nil && resp.NextPage.Offset != "" { + return resp.NextPage.Offset, nil + } + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaTaskListResponse + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + return w.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/task_convertor.go b/backend/plugins/asana/tasks/task_convertor.go new file mode 100644 index 00000000000..b790111f69c --- /dev/null +++ b/backend/plugins/asana/tasks/task_convertor.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertTask + +var ConvertTaskMeta = plugin.SubTaskMeta{ + Name: "ConvertTask", + EntryPoint: ConvertTask, + EnabledByDefault: true, + Description: "Convert tool layer Asana tasks into domain layer issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawTaskTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + clauses := []dal.Clause{ + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", connectionId, projectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + taskIdGen := didgen.NewDomainIdGenerator(&models.AsanaTask{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaTask{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolTask := inputRow.(*models.AsanaTask) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: taskIdGen.Generate(toolTask.ConnectionId, toolTask.Gid)}, + IssueKey: toolTask.Gid, + Title: toolTask.Name, + Description: toolTask.Notes, + Url: toolTask.PermalinkUrl, + CreatedDate: &toolTask.CreatedAt, + UpdatedDate: toolTask.ModifiedAt, + ResolutionDate: toolTask.CompletedAt, + DueDate: toolTask.DueOn, + CreatorName: toolTask.CreatorName, + AssigneeName: toolTask.AssigneeName, + OriginalType: toolTask.ResourceSubtype, + } + if toolTask.Completed { + domainIssue.Status = ticket.DONE + domainIssue.OriginalStatus = "completed" + } else { + domainIssue.Status = ticket.TODO + domainIssue.OriginalStatus = "incomplete" + } + boardId := boardIdGen.Generate(connectionId, toolTask.ProjectGid) + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + return []interface{}{domainIssue, boardIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/asana/tasks/task_data.go b/backend/plugins/asana/tasks/task_data.go new file mode 100644 index 00000000000..c5ed8e570d6 --- /dev/null +++ b/backend/plugins/asana/tasks/task_data.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string) (*api.RawDataSubTaskArgs, *AsanaTaskData) { + data := taskCtx.GetData().(*AsanaTaskData) + params := AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + } + return &api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: params, + Table: rawTable, + }, data +} + +type AsanaOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` + ProjectId string `json:"projectId" mapstructure:"projectId"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` +} + +type AsanaTaskData struct { + Options *AsanaOptions + ApiClient *api.ApiAsyncClient + Project *models.AsanaProject +} diff --git a/backend/plugins/asana/tasks/task_extractor.go b/backend/plugins/asana/tasks/task_extractor.go new file mode 100644 index 00000000000..506828b7869 --- /dev/null +++ b/backend/plugins/asana/tasks/task_extractor.go @@ -0,0 +1,157 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTask + +var ExtractTaskMeta = plugin.SubTaskMeta{ + Name: "ExtractTask", + EntryPoint: ExtractTask, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_tasks", +} + +type asanaApiTask struct { + Gid string `json:"gid"` + Name string `json:"name"` + Notes string `json:"notes"` + ResourceType string `json:"resource_type"` + ResourceSubtype string `json:"resource_subtype"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completed_at"` + DueOn string `json:"due_on"` + CreatedAt time.Time `json:"created_at"` + ModifiedAt *time.Time `json:"modified_at"` + PermalinkUrl string `json:"permalink_url"` + Assignee *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"assignee"` + CreatedBy *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"created_by"` + Parent *struct { + Gid string `json:"gid"` + } `json:"parent"` + NumSubtasks int `json:"num_subtasks"` + Memberships []struct { + Section *struct { + Gid string `json:"gid"` + } `json:"section"` + Project *struct { + Gid string `json:"gid"` + } `json:"project"` + } `json:"memberships"` +} + +func parseAsanaDate(s string) *time.Time { + if s == "" { + return nil + } + t, err := time.Parse("2006-01-02", s) + if err != nil { + return nil + } + return &t +} + +func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawTaskTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTask := &asanaApiTask{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTask)) + if err != nil { + return nil, err + } + assigneeGid := "" + assigneeName := "" + if apiTask.Assignee != nil { + assigneeGid = apiTask.Assignee.Gid + assigneeName = apiTask.Assignee.Name + } + creatorGid := "" + creatorName := "" + if apiTask.CreatedBy != nil { + creatorGid = apiTask.CreatedBy.Gid + creatorName = apiTask.CreatedBy.Name + } + parentGid := "" + if apiTask.Parent != nil { + parentGid = apiTask.Parent.Gid + } + sectionGid := "" + projectGid := taskData.Options.ProjectId + for _, m := range apiTask.Memberships { + if m.Project != nil { + projectGid = m.Project.Gid + } + if m.Section != nil && m.Section.Gid != "" { + sectionGid = m.Section.Gid + break + } + } + toolTask := &models.AsanaTask{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTask.Gid, + Name: apiTask.Name, + Notes: apiTask.Notes, + ResourceType: apiTask.ResourceType, + ResourceSubtype: apiTask.ResourceSubtype, + Completed: apiTask.Completed, + CompletedAt: apiTask.CompletedAt, + DueOn: parseAsanaDate(apiTask.DueOn), + CreatedAt: apiTask.CreatedAt, + ModifiedAt: apiTask.ModifiedAt, + PermalinkUrl: apiTask.PermalinkUrl, + ProjectGid: projectGid, + SectionGid: sectionGid, + AssigneeGid: assigneeGid, + AssigneeName: assigneeName, + CreatorGid: creatorGid, + CreatorName: creatorName, + ParentGid: parentGid, + NumSubtasks: apiTask.NumSubtasks, + } + return []interface{}{toolTask}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} From 7bb1d7edc85cebee048b669057f2c0e34abfe834 Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Wed, 11 Feb 2026 19:38:36 +0500 Subject: [PATCH 04/10] feat: Added asana plugin suppor for Devlake --- AGENTS.md | 90 ++ backend/plugins/asana/api/init.go | 2 + backend/plugins/asana/api/remote_api.go | 189 +++ backend/plugins/asana/impl/impl.go | 4 + backend/plugins/asana/models/project.go | 15 +- .../plugins/asana/tasks/project_collector.go | 5 +- .../plugins/asana/tasks/project_convertor.go | 81 ++ .../plugins/asana/tasks/project_extractor.go | 19 +- .../plugins/asana/tasks/section_collector.go | 5 +- .../plugins/asana/tasks/section_extractor.go | 3 +- backend/plugins/asana/tasks/task_collector.go | 5 +- backend/plugins/asana/tasks/task_data.go | 2 +- backend/plugins/asana/tasks/task_extractor.go | 3 +- backend/plugins/table_info_test.go | 2 + .../plugins/register/asana/assets/icon.svg | 20 + .../src/plugins/register/asana/config.tsx | 55 + config-ui/src/plugins/register/asana/index.ts | 19 + config-ui/src/plugins/register/index.ts | 2 + config-ui/src/plugins/utils.ts | 2 + config-ui/src/release/stable.ts | 4 + grafana/dashboards/Asana.json | 1192 +++++++++++++++++ 21 files changed, 1693 insertions(+), 26 deletions(-) create mode 100644 backend/plugins/asana/tasks/project_convertor.go create mode 100644 config-ui/src/plugins/register/asana/assets/icon.svg create mode 100644 config-ui/src/plugins/register/asana/config.tsx create mode 100644 config-ui/src/plugins/register/asana/index.ts create mode 100644 grafana/dashboards/Asana.json diff --git a/AGENTS.md b/AGENTS.md index b896b7af69c..46d7bf3f852 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,3 +130,93 @@ Located in `backend/python/plugins/`. Use Poetry for dependencies. See [backend/ - Migration scripts must be added to `All()` in `register.go` - API changes require running `make swag` to update Swagger docs - Python plugins require `libgit2` for gitextractor functionality + +## Asana Plugin Implementation (2025-02-03) + +### Overview +Implemented a complete Asana plugin (`backend/plugins/asana/`) to collect projects (boards), sections, and tasks from Asana's REST API and map them to DevLake's ticket/board domain model. + +### Architecture Decisions +- **Scope Model**: Asana **Project** = DevLake **Board** (scope) +- **Authentication**: Personal Access Token (PAT) via Bearer token (`Authorization: Bearer `) +- **API Base URL**: `https://app.asana.com/api/1.0/` (default endpoint) +- **Pagination**: Asana uses offset-based pagination via `next_page.offset` in response; implemented sequential fetching with `GetNextPageCustomData` + +### Implementation Details + +#### Models (`backend/plugins/asana/models/`) +- **Connection** (`connection.go`): `AsanaConn` (Token, RestConnection), `AsanaConnection` (BaseConnection + AsanaConn) +- **Scope** (`project.go`): `AsanaProject` implements `ToolLayerScope`; primary key: `(ConnectionId, Gid)` +- **Scope Config** (`scope_config.go`): `AsanaScopeConfig` embeds `common.ScopeConfig` with `Entities: ["TICKET"]` +- **Tool Layer**: + - `task.go`: `AsanaTask` (Gid, Name, Notes, Completed, DueOn, ProjectGid, SectionGid, AssigneeGid, CreatorGid, etc.) + - `section.go`: `AsanaSection` (Gid, Name, ProjectGid) + - `user.go`: `AsanaUser` (Gid, Name, Email) - optional, for assignee/creator enrichment +- **Migration**: `20250203_add_init_tables.go` creates all tables via `migrationhelper.AutoMigrateTables` + +#### API Layer (`backend/plugins/asana/api/`) +- **Connection API** (`connection_api.go`): Test connection via `GET users/me`, CRUD operations, default endpoint handling +- **Scope API** (`scope_api.go`): PutScopes, GetScopeList, GetScope, PatchScope, DeleteScope (uses `:scopeId` path param, not `:projectId`) +- **Scope Config API** (`scope_config_api.go`): Full CRUD for scope configs +- **Blueprint V200** (`blueprint_v200.go`): Maps Asana projects to `ticket.Board` domain entities when scope config includes `DOMAIN_TYPE_TICKET` +- **Remote API** (`remote_api.go`): Proxy endpoint for direct API access +- **Init** (`init.go`): Sets up `DsHelper[AsanaConnection, AsanaProject, AsanaScopeConfig]` with default endpoint + +#### Tasks (`backend/plugins/asana/tasks/`) +- **Project**: `project_collector.go` (GET `/projects/{gid}`), `project_extractor.go` (extracts to `_tool_asana_projects`) +- **Section**: `section_collector.go` (GET `/projects/{gid}/sections`), `section_extractor.go` (extracts to `_tool_asana_sections`) +- **Task**: + - `task_collector.go`: Collects tasks with offset pagination (limit=100, offset from `next_page.offset`) + - `task_extractor.go`: Extracts task data including memberships (project/section), assignee, creator, parent + - `task_convertor.go`: Converts `AsanaTask` → `ticket.Issue` + `ticket.BoardIssue` using `didgen` for domain IDs +- **API Client** (`api_client.go`): Creates `ApiAsyncClient` with Bearer auth, sets default endpoint if missing +- **Task Data** (`task_data.go`): `AsanaOptions` (ConnectionId, ProjectId, ScopeConfigId), `AsanaTaskData`, `CreateRawDataSubTaskArgs` helper + +#### Plugin Entry (`backend/plugins/asana/`) +- **Main** (`asana.go`): `PluginEntry impl.Asana` for plugin loading +- **Impl** (`impl/impl.go`): Implements all required interfaces: + - `PluginMeta`: Name="asana", Description, RootPkgPath + - `PluginTask`: SubTaskMetas (CollectProject, ExtractProject, CollectSection, ExtractSection, CollectTask, ExtractTask, ConvertTask) + - `PluginModel`: GetTablesInfo() returns all 6 models + - `PluginMigration`: MigrationScripts() from migrationscripts.All() + - `PluginApi`: ApiResources() with connections, scopes, scope-configs, test, proxy routes + - `PluginSource`: Connection(), Scope(), ScopeConfig() + - `DataSourcePluginBlueprintV200`: MakeDataSourcePipelinePlanV200() + - `CloseablePluginTask`: Close() releases ApiClient + +#### Config UI (`config-ui/src/plugins/register/asana/`) +- **Config** (`config.tsx`): `AsanaConfig` with: + - Connection fields: name, endpoint (default: `https://app.asana.com/api/1.0/`), token, proxy, rateLimitPerHour (default: 150) + - Data scope title: "Projects" + - Scope config entities: `['TICKET']` +- **Icon** (`assets/icon.svg`): Placeholder SVG icon +- **Registration** (`index.ts`): Exports `AsanaConfig` +- **Plugin Registry** (`config-ui/src/plugins/register/index.ts`): Added `AsanaConfig` to `pluginConfigs` array + +#### Testing +- **Table Info Test** (`backend/plugins/table_info_test.go`): Added `asana` import and `checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo)` +- **E2E Test** (`backend/plugins/asana/e2e/task_test.go`): + - Imports raw task CSV (`e2e/raw_tables/_raw_asana_tasks.csv`) + - Runs `ExtractTaskMeta` subtask + - Verifies `_tool_asana_tasks` against snapshot (`e2e/snapshot_tables/_tool_asana_tasks.csv`) + +### Key Implementation Notes +1. **Scope ID**: Uses `:scopeId` path param (not `:projectId`) to match generic scope helper expectations; scope ID is Asana project GID +2. **Response Parsing**: Asana API wraps responses in `{"data": {...}}` or `{"data": [...], "next_page": {...}}`; collectors unwrap `data` field +3. **Date Parsing**: `due_on` is date-only string (`YYYY-MM-DD`); `parseAsanaDate()` helper converts to `*time.Time` +4. **Task Memberships**: Tasks can belong to multiple projects/sections; extractor uses first section from memberships array +5. **Domain Conversion**: Task status maps: `completed=true` → `ticket.DONE`, `completed=false` → `ticket.TODO` +6. **Default Endpoint**: Set in `api/init.go` constant and applied in `PostConnections` if missing from request + +### Files Created +- **Backend**: 25+ Go files across `models/`, `api/`, `tasks/`, `impl/`, `e2e/` +- **Config UI**: 3 TypeScript files (`config.tsx`, `index.ts`, `assets/icon.svg`) +- **Test**: 1 CSV fixture pair (raw + snapshot), 1 E2E test file +- **CI**: Updated `table_info_test.go` + +### Next Steps (Optional Enhancements) +- Add user collection/enrichment for assignee/creator names +- Support OAuth 2.0 authentication (currently PAT only) +- Add task comments/subtasks collection +- Implement incremental collection with time-based bookmarking +- Add transformation rules for custom field mappings diff --git a/backend/plugins/asana/api/init.go b/backend/plugins/asana/api/init.go index 1ec8740d177..d13865b184c 100644 --- a/backend/plugins/asana/api/init.go +++ b/backend/plugins/asana/api/init.go @@ -32,6 +32,7 @@ var basicRes context.BasicRes var dsHelper *api.DsHelper[models.AsanaConnection, models.AsanaProject, models.AsanaScopeConfig] var raProxy *api.DsRemoteApiProxyHelper[models.AsanaConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.AsanaConnection, models.AsanaProject, AsanaRemotePagination] func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br @@ -49,4 +50,5 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { nil, ) raProxy = api.NewDsRemoteApiProxyHelper[models.AsanaConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.AsanaConnection, models.AsanaProject, AsanaRemotePagination](raProxy, listAsanaRemoteScopes) } diff --git a/backend/plugins/asana/api/remote_api.go b/backend/plugins/asana/api/remote_api.go index 7f84420e6b4..45e53eeba28 100644 --- a/backend/plugins/asana/api/remote_api.go +++ b/backend/plugins/asana/api/remote_api.go @@ -18,10 +18,199 @@ limitations under the License. package api import ( + "fmt" + "net/url" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/asana/models" ) +type AsanaRemotePagination struct { + Offset string `json:"offset"` + Limit int `json:"limit"` +} + +type asanaWorkspaceResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` +} + +type asanaWorkspacesListResponse struct { + Data []asanaWorkspaceResponse `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} + +type asanaProjectResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Archived bool `json:"archived"` + PermalinkUrl string `json:"permalink_url"` + Workspace *struct { + Gid string `json:"gid"` + } `json:"workspace"` +} + +type asanaProjectsListResponse struct { + Data []asanaProjectResponse `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} + +func listAsanaRemoteScopes( + connection *models.AsanaConnection, + apiClient plugin.ApiClient, + groupId string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + if page.Limit == 0 { + page.Limit = 100 + } + + // If no groupId, list workspaces as groups first + if groupId == "" { + return listAsanaWorkspaces(apiClient, page) + } + + // If groupId is provided, it's a workspace GID — list projects in that workspace + return listAsanaProjects(apiClient, groupId, page) +} + +func listAsanaWorkspaces( + apiClient plugin.ApiClient, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + res, err := apiClient.Get("workspaces", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch workspaces from Asana API") + } + + var response asanaWorkspacesListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana workspaces response") + } + + for _, workspace := range response.Data { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + Id: workspace.Gid, + Name: workspace.Name, + FullName: workspace.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +func listAsanaProjects( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("workspaces/%s/projects", workspaceGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch projects from Asana API") + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana projects response") + } + + for _, project := range response.Data { + workspaceGidVal := workspaceGid + if project.Workspace != nil { + workspaceGidVal = project.Workspace.Gid + } + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: project.Gid, + Name: project.Name, + FullName: project.Name, + Data: &models.AsanaProject{ + Gid: project.Gid, + Name: project.Name, + ResourceType: project.ResourceType, + Archived: project.Archived, + PermalinkUrl: project.PermalinkUrl, + WorkspaceGid: workspaceGidVal, + }, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// RemoteScopes list all available scopes (projects) for this connection +// @Summary list all available scopes (projects) for this connection +// @Description list all available scopes (projects) for this connection +// @Tags plugins/asana +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} RemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/asana/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return raProxy.Proxy(input) } diff --git a/backend/plugins/asana/impl/impl.go b/backend/plugins/asana/impl/impl.go index f92d0f2fd80..fc4b4123f05 100644 --- a/backend/plugins/asana/impl/impl.go +++ b/backend/plugins/asana/impl/impl.go @@ -78,6 +78,7 @@ func (p Asana) SubTaskMetas() []plugin.SubTaskMeta { tasks.ExtractSectionMeta, tasks.CollectTaskMeta, tasks.ExtractTaskMeta, + tasks.ConvertProjectMeta, tasks.ConvertTaskMeta, } } @@ -150,6 +151,9 @@ func (p Asana) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "connections/:connectionId/proxy/rest/*path": { "GET": api.Proxy, }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, "connections/:connectionId/scope-configs": { "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, diff --git a/backend/plugins/asana/models/project.go b/backend/plugins/asana/models/project.go index 9be242dbe4a..498bcd97ef1 100644 --- a/backend/plugins/asana/models/project.go +++ b/backend/plugins/asana/models/project.go @@ -25,14 +25,13 @@ import ( var _ plugin.ToolLayerScope = (*AsanaProject)(nil) type AsanaProject struct { - common.Scope `mapstructure:",squash"` - ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"` - Gid string `json:"gid" mapstructure:"gid" gorm:"type:varchar(255);primaryKey"` - Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` - ResourceType string `json:"resourceType" mapstructure:"resourceType" gorm:"type:varchar(32)"` - Archived bool `json:"archived" mapstructure:"archived"` - WorkspaceGid string `json:"workspaceGid" mapstructure:"workspaceGid" gorm:"type:varchar(255)"` - PermalinkUrl string `json:"permalinkUrl" mapstructure:"permalinkUrl" gorm:"type:varchar(512)"` + common.Scope `mapstructure:",squash"` + Gid string `json:"gid" mapstructure:"gid" gorm:"type:varchar(255);primaryKey"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + ResourceType string `json:"resourceType" mapstructure:"resourceType" gorm:"type:varchar(32)"` + Archived bool `json:"archived" mapstructure:"archived"` + WorkspaceGid string `json:"workspaceGid" mapstructure:"workspaceGid" gorm:"type:varchar(255)"` + PermalinkUrl string `json:"permalinkUrl" mapstructure:"permalinkUrl" gorm:"type:varchar(512)"` } func (p AsanaProject) ScopeId() string { diff --git a/backend/plugins/asana/tasks/project_collector.go b/backend/plugins/asana/tasks/project_collector.go index 0155af9ecf7..4e8416ba394 100644 --- a/backend/plugins/asana/tasks/project_collector.go +++ b/backend/plugins/asana/tasks/project_collector.go @@ -24,6 +24,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" ) const rawProjectTable = "asana_projects" @@ -35,7 +36,7 @@ var CollectProjectMeta = plugin.SubTaskMeta{ EntryPoint: CollectProject, EnabledByDefault: true, Description: "Collect project data from Asana API", - DomainTypes: []string{}, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaDataWrapper struct { @@ -47,7 +48,7 @@ func CollectProject(taskCtx plugin.SubTaskContext) errors.Error { collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: data.Options.ConnectionId, ProjectId: data.Options.ProjectId, }, diff --git a/backend/plugins/asana/tasks/project_convertor.go b/backend/plugins/asana/tasks/project_convertor.go new file mode 100644 index 00000000000..4f086b76f57 --- /dev/null +++ b/backend/plugins/asana/tasks/project_convertor.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertProject + +var ConvertProjectMeta = plugin.SubTaskMeta{ + Name: "ConvertProject", + EntryPoint: ConvertProject, + EnabledByDefault: true, + Description: "Convert tool layer Asana projects into domain layer boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertProject(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawProjectTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + clauses := []dal.Clause{ + dal.From(&models.AsanaProject{}), + dal.Where("connection_id = ? AND gid = ?", connectionId, projectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + boardIdGen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaProject{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolProject := inputRow.(*models.AsanaProject) + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(toolProject.ConnectionId, toolProject.Gid)}, + Name: toolProject.Name, + Url: toolProject.PermalinkUrl, + Type: "asana", + } + return []interface{}{domainBoard}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + diff --git a/backend/plugins/asana/tasks/project_extractor.go b/backend/plugins/asana/tasks/project_extractor.go index 5eb1734fe2c..dba73b8a654 100644 --- a/backend/plugins/asana/tasks/project_extractor.go +++ b/backend/plugins/asana/tasks/project_extractor.go @@ -33,6 +33,7 @@ var ExtractProjectMeta = plugin.SubTaskMeta{ EntryPoint: ExtractProject, EnabledByDefault: true, Description: "Extract raw data into tool layer table _tool_asana_projects", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaApiProject struct { @@ -51,7 +52,7 @@ func ExtractProject(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: taskData.Options.ConnectionId, ProjectId: taskData.Options.ProjectId, }, @@ -68,15 +69,15 @@ func ExtractProject(taskCtx plugin.SubTaskContext) errors.Error { workspaceGid = apiProject.Workspace.Gid } toolProject := &models.AsanaProject{ - ConnectionId: taskData.Options.ConnectionId, - ScopeConfigId: taskData.Options.ScopeConfigId, - Gid: apiProject.Gid, - Name: apiProject.Name, - ResourceType: apiProject.ResourceType, - Archived: apiProject.Archived, - PermalinkUrl: apiProject.PermalinkUrl, - WorkspaceGid: workspaceGid, + Gid: apiProject.Gid, + Name: apiProject.Name, + ResourceType: apiProject.ResourceType, + Archived: apiProject.Archived, + PermalinkUrl: apiProject.PermalinkUrl, + WorkspaceGid: workspaceGid, } + toolProject.ConnectionId = taskData.Options.ConnectionId + toolProject.ScopeConfigId = taskData.Options.ScopeConfigId return []interface{}{toolProject}, nil }, }) diff --git a/backend/plugins/asana/tasks/section_collector.go b/backend/plugins/asana/tasks/section_collector.go index 0051b5380af..782f47a90b7 100644 --- a/backend/plugins/asana/tasks/section_collector.go +++ b/backend/plugins/asana/tasks/section_collector.go @@ -24,6 +24,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" ) const rawSectionTable = "asana_sections" @@ -35,7 +36,7 @@ var CollectSectionMeta = plugin.SubTaskMeta{ EntryPoint: CollectSection, EnabledByDefault: true, Description: "Collect section data from Asana API", - DomainTypes: []string{}, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaListWrapper struct { @@ -47,7 +48,7 @@ func CollectSection(taskCtx plugin.SubTaskContext) errors.Error { collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: data.Options.ConnectionId, ProjectId: data.Options.ProjectId, }, diff --git a/backend/plugins/asana/tasks/section_extractor.go b/backend/plugins/asana/tasks/section_extractor.go index fe06daf287f..3c6e90a9b55 100644 --- a/backend/plugins/asana/tasks/section_extractor.go +++ b/backend/plugins/asana/tasks/section_extractor.go @@ -33,6 +33,7 @@ var ExtractSectionMeta = plugin.SubTaskMeta{ EntryPoint: ExtractSection, EnabledByDefault: true, Description: "Extract raw data into tool layer table _tool_asana_sections", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaApiSection struct { @@ -49,7 +50,7 @@ func ExtractSection(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: taskData.Options.ConnectionId, ProjectId: taskData.Options.ProjectId, }, diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go index 42fe95d3521..aad1188537d 100644 --- a/backend/plugins/asana/tasks/task_collector.go +++ b/backend/plugins/asana/tasks/task_collector.go @@ -25,6 +25,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" ) const rawTaskTable = "asana_tasks" @@ -36,7 +37,7 @@ var CollectTaskMeta = plugin.SubTaskMeta{ EntryPoint: CollectTask, EnabledByDefault: true, Description: "Collect task data from Asana API", - DomainTypes: []string{}, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaTaskListResponse struct { @@ -53,7 +54,7 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: data.Options.ConnectionId, ProjectId: data.Options.ProjectId, }, diff --git a/backend/plugins/asana/tasks/task_data.go b/backend/plugins/asana/tasks/task_data.go index c5ed8e570d6..e8770e7a714 100644 --- a/backend/plugins/asana/tasks/task_data.go +++ b/backend/plugins/asana/tasks/task_data.go @@ -25,7 +25,7 @@ import ( func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string) (*api.RawDataSubTaskArgs, *AsanaTaskData) { data := taskCtx.GetData().(*AsanaTaskData) - params := AsanaApiParams{ + params := models.AsanaApiParams{ ConnectionId: data.Options.ConnectionId, ProjectId: data.Options.ProjectId, } diff --git a/backend/plugins/asana/tasks/task_extractor.go b/backend/plugins/asana/tasks/task_extractor.go index 506828b7869..43219c2a2a2 100644 --- a/backend/plugins/asana/tasks/task_extractor.go +++ b/backend/plugins/asana/tasks/task_extractor.go @@ -34,6 +34,7 @@ var ExtractTaskMeta = plugin.SubTaskMeta{ EntryPoint: ExtractTask, EnabledByDefault: true, Description: "Extract raw data into tool layer table _tool_asana_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } type asanaApiTask struct { @@ -86,7 +87,7 @@ func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, - Params: AsanaApiParams{ + Params: models.AsanaApiParams{ ConnectionId: taskData.Options.ConnectionId, ProjectId: taskData.Options.ProjectId, }, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index ad1b7a27376..0516cb5b3b2 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -23,6 +23,7 @@ import ( "github.com/apache/incubator-devlake/helpers/unithelper" ae "github.com/apache/incubator-devlake/plugins/ae/impl" argocd "github.com/apache/incubator-devlake/plugins/argocd/impl" + asana "github.com/apache/incubator-devlake/plugins/asana/impl" azuredevops "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" @@ -70,6 +71,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("bitbucket/models", bitbucket.Bitbucket{}.GetTablesInfo) checker.FeedIn("bitbucket_server/models", bitbucket_server.BitbucketServer{}.GetTablesInfo) checker.FeedIn("argocd/models", argocd.ArgoCD{}.GetTablesInfo) + checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo) checker.FeedIn("customize/models", customize.Customize{}.GetTablesInfo) checker.FeedIn("dbt", dbt.Dbt{}.GetTablesInfo) checker.FeedIn("developer_telemetry/models", developer_telemetry.DeveloperTelemetry{}.GetTablesInfo) diff --git a/config-ui/src/plugins/register/asana/assets/icon.svg b/config-ui/src/plugins/register/asana/assets/icon.svg new file mode 100644 index 00000000000..daf46f52ea4 --- /dev/null +++ b/config-ui/src/plugins/register/asana/assets/icon.svg @@ -0,0 +1,20 @@ + + + + + diff --git a/config-ui/src/plugins/register/asana/config.tsx b/config-ui/src/plugins/register/asana/config.tsx new file mode 100644 index 00000000000..26720e8d5a3 --- /dev/null +++ b/config-ui/src/plugins/register/asana/config.tsx @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const AsanaConfig: IPluginConfig = { + plugin: 'asana', + name: 'Asana', + icon: ({ color }) => , + sort: 12, + connection: { + initialValues: { + endpoint: 'https://app.asana.com/api/1.0/', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Asana API base URL.', + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for default.', + defaultValue: 150, + }, + ], + }, + dataScope: { + title: 'Projects', + }, + scopeConfig: { + entities: ['TICKET'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/asana/index.ts b/config-ui/src/plugins/register/asana/index.ts new file mode 100644 index 00000000000..ee80317ec68 --- /dev/null +++ b/config-ui/src/plugins/register/asana/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export { AsanaConfig } from './config'; diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 225032a5b67..5a20f339fc3 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -19,6 +19,7 @@ import { IPluginConfig } from '@/types'; import { ArgoCDConfig } from './argocd'; +import { AsanaConfig } from './asana'; import { AzureConfig, AzureGoConfig } from './azure'; import { BambooConfig } from './bamboo'; import { BitbucketConfig } from './bitbucket'; @@ -41,6 +42,7 @@ import { SlackConfig } from './slack/config'; export const pluginConfigs: IPluginConfig[] = [ ArgoCDConfig, + AsanaConfig, AzureConfig, AzureGoConfig, BambooConfig, diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index c008f4b4da1..02bbd839d86 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -41,6 +41,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.planKey}`; case 'argocd': return `${scope.name}`; + case 'asana': + return `${scope.gid}`; default: return `${scope.id}`; } diff --git a/config-ui/src/release/stable.ts b/config-ui/src/release/stable.ts index c31162fdb85..bceb187b6b5 100644 --- a/config-ui/src/release/stable.ts +++ b/config-ui/src/release/stable.ts @@ -23,6 +23,10 @@ const URLS = { }, DORA: 'https://devlake.apache.org/docs/DORA/', PLUGIN: { + ASANA: { + BASIS: 'https://devlake.apache.org/docs/Configuration/Asana', + TRANSFORMATION: 'https://devlake.apache.org/docs/Configuration/Asana', + }, ARGOCD: { BASIS: 'https://devlake.apache.org/docs/Configuration/ArgoCD', TRANSFORMATION: 'https://devlake.apache.org/docs/Configuration/ArgoCD#step-3---adding-transformation-rules-optional', diff --git a/grafana/dashboards/Asana.json b/grafana/dashboards/Asana.json new file mode 100644 index 00000000000..665bb0672e2 --- /dev/null +++ b/grafana/dashboards/Asana.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Asana", + "url": "https://devlake.apache.org/docs/Configuration/Asana" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Asana.\n- Data Source Required: Asana", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "mysql", + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "9.5.15", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'asana%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'asana%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Asana", + "uid": "asana-dashboard", + "version": 1, + "weekStart": "" +} From 7c592a817a46186c36851da2986373c9c4fd87b1 Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Thu, 12 Feb 2026 17:21:55 +0500 Subject: [PATCH 05/10] fix: Add other asana objects --- backend/plugins/asana/api/remote_api.go | 478 ++++++++++++++++-- backend/plugins/asana/impl/impl.go | 28 + backend/plugins/asana/models/custom_field.go | 60 +++ backend/plugins/asana/models/membership.go | 49 ++ .../20250203_add_init_tables.go | 9 + .../20250212_add_missing_tables.go | 52 ++ ...20250212_add_task_transformation_fields.go | 75 +++ .../20250212_add_user_photo_url.go | 52 ++ .../asana/models/migrationscripts/register.go | 3 + backend/plugins/asana/models/scope_config.go | 37 ++ backend/plugins/asana/models/story.go | 48 ++ backend/plugins/asana/models/tag.go | 51 ++ backend/plugins/asana/models/task.go | 54 +- backend/plugins/asana/models/team.go | 39 ++ backend/plugins/asana/models/user.go | 12 +- backend/plugins/asana/models/workspace.go | 36 ++ .../plugins/asana/tasks/story_collector.go | 101 ++++ .../plugins/asana/tasks/story_convertor.go | 89 ++++ .../plugins/asana/tasks/story_extractor.go | 120 +++++ .../plugins/asana/tasks/subtask_collector.go | 98 ++++ .../plugins/asana/tasks/subtask_extractor.go | 125 +++++ backend/plugins/asana/tasks/tag_collector.go | 97 ++++ backend/plugins/asana/tasks/tag_extractor.go | 99 ++++ backend/plugins/asana/tasks/task_collector.go | 2 + backend/plugins/asana/tasks/task_convertor.go | 150 +++++- backend/plugins/asana/tasks/task_extractor.go | 6 +- backend/plugins/asana/tasks/user_collector.go | 102 ++++ backend/plugins/asana/tasks/user_convertor.go | 81 +++ backend/plugins/asana/tasks/user_extractor.go | 86 ++++ .../src/plugins/register/asana/config.tsx | 26 +- 30 files changed, 2199 insertions(+), 66 deletions(-) create mode 100644 backend/plugins/asana/models/custom_field.go create mode 100644 backend/plugins/asana/models/membership.go create mode 100644 backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go create mode 100644 backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go create mode 100644 backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go create mode 100644 backend/plugins/asana/models/story.go create mode 100644 backend/plugins/asana/models/tag.go create mode 100644 backend/plugins/asana/models/team.go create mode 100644 backend/plugins/asana/models/workspace.go create mode 100644 backend/plugins/asana/tasks/story_collector.go create mode 100644 backend/plugins/asana/tasks/story_convertor.go create mode 100644 backend/plugins/asana/tasks/story_extractor.go create mode 100644 backend/plugins/asana/tasks/subtask_collector.go create mode 100644 backend/plugins/asana/tasks/subtask_extractor.go create mode 100644 backend/plugins/asana/tasks/tag_collector.go create mode 100644 backend/plugins/asana/tasks/tag_extractor.go create mode 100644 backend/plugins/asana/tasks/user_collector.go create mode 100644 backend/plugins/asana/tasks/user_convertor.go create mode 100644 backend/plugins/asana/tasks/user_extractor.go diff --git a/backend/plugins/asana/api/remote_api.go b/backend/plugins/asana/api/remote_api.go index 45e53eeba28..0245cb6008b 100644 --- a/backend/plugins/asana/api/remote_api.go +++ b/backend/plugins/asana/api/remote_api.go @@ -20,6 +20,7 @@ package api import ( "fmt" "net/url" + "strings" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -33,19 +34,60 @@ type AsanaRemotePagination struct { Limit int `json:"limit"` } +// Response types for Asana API type asanaWorkspaceResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + IsOrganization bool `json:"is_organization"` +} + +type asanaWorkspacesListResponse struct { + Data []asanaWorkspaceResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaNextPage struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` +} + +type asanaTeamResponse struct { Gid string `json:"gid"` Name string `json:"name"` ResourceType string `json:"resource_type"` + Description string `json:"description"` + PermalinkUrl string `json:"permalink_url"` } -type asanaWorkspacesListResponse struct { - Data []asanaWorkspaceResponse `json:"data"` - NextPage *struct { - Offset string `json:"offset"` - Path string `json:"path"` - URI string `json:"uri"` - } `json:"next_page"` +type asanaTeamsListResponse struct { + Data []asanaTeamResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaPortfolioResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + PermalinkUrl string `json:"permalink_url"` +} + +type asanaPortfoliosListResponse struct { + Data []asanaPortfolioResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaGoalResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Notes string `json:"notes"` +} + +type asanaGoalsListResponse struct { + Data []asanaGoalResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` } type asanaProjectResponse struct { @@ -57,17 +99,24 @@ type asanaProjectResponse struct { Workspace *struct { Gid string `json:"gid"` } `json:"workspace"` + Team *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"team"` } type asanaProjectsListResponse struct { Data []asanaProjectResponse `json:"data"` - NextPage *struct { - Offset string `json:"offset"` - Path string `json:"path"` - URI string `json:"uri"` - } `json:"next_page"` + NextPage *asanaNextPage `json:"next_page"` } +// Scope type constants +const ( + ScopeTypeTeam = "team" + ScopeTypePortfolio = "portfolio" + ScopeTypeGoal = "goal" +) + func listAsanaRemoteScopes( connection *models.AsanaConnection, apiClient plugin.ApiClient, @@ -82,15 +131,67 @@ func listAsanaRemoteScopes( page.Limit = 100 } - // If no groupId, list workspaces as groups first + // Level 1: No groupId - list workspaces if groupId == "" { return listAsanaWorkspaces(apiClient, page) } - // If groupId is provided, it's a workspace GID — list projects in that workspace - return listAsanaProjects(apiClient, groupId, page) + // Parse hierarchical groupId + // Format examples: + // - "workspace/{gid}" -> show Teams, Portfolios, Goals options + // - "workspace/{gid}/team/{teamGid}" -> list projects in team + // - "workspace/{gid}/portfolio/{portfolioGid}" -> list projects in portfolio + // - "workspace/{gid}/goal/{goalGid}" -> list projects for goal + + if strings.HasPrefix(groupId, "workspace/") { + parts := strings.Split(groupId[10:], "/") // Remove "workspace/" prefix + + if len(parts) == 1 { + // Level 2: Workspace selected - show Teams, Portfolios, Goals as categories + workspaceGid := parts[0] + return listAsanaScopeCategories(apiClient, workspaceGid, page) + } + + if len(parts) >= 3 { + workspaceGid := parts[0] + scopeType := parts[1] + scopeGid := parts[2] + + switch scopeType { + case ScopeTypeTeam: + // Level 4: List projects in team + return listAsanaTeamProjects(apiClient, workspaceGid, scopeGid, page) + case ScopeTypePortfolio: + // Level 4: List projects in portfolio + return listAsanaPortfolioProjects(apiClient, workspaceGid, scopeGid, page) + case ScopeTypeGoal: + // Level 4: List projects for goal + return listAsanaGoalProjects(apiClient, workspaceGid, scopeGid, page) + } + } + + if len(parts) == 2 { + workspaceGid := parts[0] + scopeType := parts[1] + + switch scopeType { + case ScopeTypeTeam: + // Level 3: List all teams + return listAsanaTeams(apiClient, workspaceGid, page) + case ScopeTypePortfolio: + // Level 3: List all portfolios + return listAsanaPortfolios(apiClient, workspaceGid, page) + case ScopeTypeGoal: + // Level 3: List all goals + return listAsanaGoals(apiClient, workspaceGid, page) + } + } + } + + return nil, nil, errors.BadInput.New("invalid groupId format") } +// Level 1: List workspaces func listAsanaWorkspaces( apiClient plugin.ApiClient, page AsanaRemotePagination, @@ -101,7 +202,7 @@ func listAsanaWorkspaces( ) { query := url.Values{} query.Set("limit", fmt.Sprintf("%d", page.Limit)) - query.Set("opt_fields", "name,resource_type") + query.Set("opt_fields", "name,resource_type,is_organization") if page.Offset != "" { query.Set("offset", page.Offset) } @@ -118,9 +219,11 @@ func listAsanaWorkspaces( } for _, workspace := range response.Data { + groupId := fmt.Sprintf("workspace/%s", workspace.Gid) children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: workspace.Gid, + ParentId: nil, // Root level, no parent + Id: groupId, Name: workspace.Name, FullName: workspace.Name, }) @@ -136,7 +239,156 @@ func listAsanaWorkspaces( return children, nextPage, nil } -func listAsanaProjects( +// Level 2: Show scope categories (Teams, Portfolios, Goals) +func listAsanaScopeCategories( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the workspace + parentId := fmt.Sprintf("workspace/%s", workspaceGid) + + // Return the three main categories: Teams, Portfolios, Goals + children = []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeTeam), + Name: "🏢 Teams", + FullName: "Teams", + }, + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypePortfolio), + Name: "📁 Portfolios", + FullName: "Portfolios", + }, + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeGoal), + Name: "🎯 Goals", + FullName: "Goals", + }, + } + + return children, nil, nil +} + +// Level 3: List teams in workspace +func listAsanaTeams( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the "Teams" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeTeam) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,description,permalink_url") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("workspaces/%s/teams", workspaceGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch teams from Asana API") + } + + var response asanaTeamsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana teams response") + } + + for _, team := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeTeam, team.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: team.Name, + FullName: team.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 3: List portfolios in workspace +func listAsanaPortfolios( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the "Portfolios" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypePortfolio) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("workspace", workspaceGid) + query.Set("owner", "me") + query.Set("opt_fields", "name,resource_type,permalink_url") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + res, err := apiClient.Get("portfolios", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch portfolios from Asana API") + } + + var response asanaPortfoliosListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana portfolios response") + } + + for _, portfolio := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypePortfolio, portfolio.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: portfolio.Name, + FullName: portfolio.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 3: List goals in workspace +func listAsanaGoals( apiClient plugin.ApiClient, workspaceGid string, page AsanaRemotePagination, @@ -145,32 +397,203 @@ func listAsanaProjects( nextPage *AsanaRemotePagination, err errors.Error, ) { + // Parent is the "Goals" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeGoal) + query := url.Values{} query.Set("limit", fmt.Sprintf("%d", page.Limit)) - query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace") + query.Set("workspace", workspaceGid) + query.Set("opt_fields", "name,resource_type,notes") if page.Offset != "" { query.Set("offset", page.Offset) } - apiPath := fmt.Sprintf("workspaces/%s/projects", workspaceGid) + res, err := apiClient.Get("goals", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch goals from Asana API") + } + + var response asanaGoalsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana goals response") + } + + for _, goal := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeGoal, goal.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: goal.Name, + FullName: goal.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects in a team +func listAsanaTeamProjects( + apiClient plugin.ApiClient, + workspaceGid string, + teamGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific team + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeTeam, teamGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("teams/%s/projects", teamGid) res, err := apiClient.Get(apiPath, query, nil) if err != nil { - return nil, nil, errors.Default.Wrap(err, "failed to fetch projects from Asana API") + return nil, nil, errors.Default.Wrap(err, "failed to fetch team projects from Asana API") } var response asanaProjectsListResponse err = api.UnmarshalResponse(res, &response) if err != nil { - return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana projects response") + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana team projects response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects in a portfolio +func listAsanaPortfolioProjects( + apiClient plugin.ApiClient, + workspaceGid string, + portfolioGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific portfolio + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypePortfolio, portfolioGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) } - for _, project := range response.Data { + apiPath := fmt.Sprintf("portfolios/%s/items", portfolioGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch portfolio items from Asana API") + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana portfolio items response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects associated with a goal +func listAsanaGoalProjects( + apiClient plugin.ApiClient, + workspaceGid string, + goalGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific goal + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeGoal, goalGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + // Goals API: GET /goals/{goal_gid}/parentGoals for related projects + // Note: Asana's goal-to-project relationship is through supporting work + apiPath := fmt.Sprintf("goals/%s/supportingWork", goalGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + // If supporting work API fails, return empty list + return children, nil, nil + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana goal supporting work response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Helper function to convert Asana projects to scope entries +func convertProjectsToScopes( + projects []asanaProjectResponse, + workspaceGid string, + parentId *string, +) []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject] { + var children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject] + + for _, project := range projects { workspaceGidVal := workspaceGid if project.Workspace != nil { workspaceGidVal = project.Workspace.Gid } children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: parentId, Id: project.Gid, Name: project.Name, FullName: project.Name, @@ -185,14 +608,7 @@ func listAsanaProjects( }) } - if response.NextPage != nil && response.NextPage.Offset != "" { - nextPage = &AsanaRemotePagination{ - Offset: response.NextPage.Offset, - Limit: page.Limit, - } - } - - return children, nextPage, nil + return children } // RemoteScopes list all available scopes (projects) for this connection diff --git a/backend/plugins/asana/impl/impl.go b/backend/plugins/asana/impl/impl.go index fc4b4123f05..42ac786ae5f 100644 --- a/backend/plugins/asana/impl/impl.go +++ b/backend/plugins/asana/impl/impl.go @@ -59,6 +59,15 @@ func (p Asana) GetTablesInfo() []dal.Tabler { &models.AsanaTask{}, &models.AsanaSection{}, &models.AsanaUser{}, + &models.AsanaWorkspace{}, + &models.AsanaTeam{}, + &models.AsanaStory{}, + &models.AsanaTag{}, + &models.AsanaTaskTag{}, + &models.AsanaCustomField{}, + &models.AsanaTaskCustomFieldValue{}, + &models.AsanaProjectMembership{}, + &models.AsanaTeamMembership{}, } } @@ -72,14 +81,33 @@ func (p Asana) Name() string { func (p Asana) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ + // Collect and extract in hierarchical order + // 1. Project (scope) tasks.CollectProjectMeta, tasks.ExtractProjectMeta, + // 2. Users (project members) + tasks.CollectUserMeta, + tasks.ExtractUserMeta, + // 3. Sections tasks.CollectSectionMeta, tasks.ExtractSectionMeta, + // 4. Tasks tasks.CollectTaskMeta, tasks.ExtractTaskMeta, + // 5. Subtasks (children of tasks) + tasks.CollectSubtaskMeta, + tasks.ExtractSubtaskMeta, + // 6. Stories (comments on tasks) + tasks.CollectStoryMeta, + tasks.ExtractStoryMeta, + // 7. Tags (on tasks) + tasks.CollectTagMeta, + tasks.ExtractTagMeta, + // Convert to domain layer tasks.ConvertProjectMeta, + tasks.ConvertUserMeta, tasks.ConvertTaskMeta, + tasks.ConvertStoryMeta, } } diff --git a/backend/plugins/asana/models/custom_field.go b/backend/plugins/asana/models/custom_field.go new file mode 100644 index 00000000000..f313d7abe40 --- /dev/null +++ b/backend/plugins/asana/models/custom_field.go @@ -0,0 +1,60 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaCustomField represents a custom field definition +type AsanaCustomField struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Type string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + Precision int `json:"precision"` + IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` + HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` + common.NoPKModel +} + +func (AsanaCustomField) TableName() string { + return "_tool_asana_custom_fields" +} + +// AsanaTaskCustomFieldValue represents a custom field value on a task +type AsanaTaskCustomFieldValue struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldName string `gorm:"type:varchar(255)"` + DisplayValue string `gorm:"type:text"` + TextValue string `gorm:"type:text"` + NumberValue *float64 `json:"numberValue"` + EnumValueGid string `gorm:"type:varchar(255)"` + EnumValueName string `gorm:"type:varchar(255)"` + common.NoPKModel +} + +func (AsanaTaskCustomFieldValue) TableName() string { + return "_tool_asana_task_custom_field_values" +} + diff --git a/backend/plugins/asana/models/membership.go b/backend/plugins/asana/models/membership.go new file mode 100644 index 00000000000..f3469093fa2 --- /dev/null +++ b/backend/plugins/asana/models/membership.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaProjectMembership links users to projects with their role +type AsanaProjectMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + Role string `gorm:"type:varchar(32)"` + common.NoPKModel +} + +func (AsanaProjectMembership) TableName() string { + return "_tool_asana_project_memberships" +} + +// AsanaTeamMembership links users to teams +type AsanaTeamMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + IsGuest bool `json:"isGuest"` + common.NoPKModel +} + +func (AsanaTeamMembership) TableName() string { + return "_tool_asana_team_memberships" +} + diff --git a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go index 8816068d044..3b26a7dffed 100644 --- a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go +++ b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go @@ -35,6 +35,15 @@ func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { &models.AsanaTask{}, &models.AsanaSection{}, &models.AsanaUser{}, + &models.AsanaWorkspace{}, + &models.AsanaTeam{}, + &models.AsanaStory{}, + &models.AsanaTag{}, + &models.AsanaTaskTag{}, + &models.AsanaCustomField{}, + &models.AsanaTaskCustomFieldValue{}, + &models.AsanaProjectMembership{}, + &models.AsanaTeamMembership{}, ) } diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go new file mode 100644 index 00000000000..ad9d2866c34 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +type addMissingTables struct{} + +func (*addMissingTables) Up(basicRes context.BasicRes) errors.Error { + // Add all the new tables that were added after the initial migration + return migrationhelper.AutoMigrateTables( + basicRes, + &models.AsanaWorkspace{}, + &models.AsanaTeam{}, + &models.AsanaStory{}, + &models.AsanaTag{}, + &models.AsanaTaskTag{}, + &models.AsanaCustomField{}, + &models.AsanaTaskCustomFieldValue{}, + &models.AsanaProjectMembership{}, + &models.AsanaTeamMembership{}, + ) +} + +func (*addMissingTables) Version() uint64 { + return 20250212000002 +} + +func (*addMissingTables) Name() string { + return "asana add missing tables for hierarchical data" +} + diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go new file mode 100644 index 00000000000..e895cc7367e --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addTaskTransformationFields)(nil) + +type addTaskTransformationFields struct{} + +type asanaTask20250212 struct { + SectionName string `gorm:"type:varchar(255)"` + StdType string `gorm:"type:varchar(255)"` + StdStatus string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + StoryPoint *float64 `gorm:"type:double"` + Severity string `gorm:"type:varchar(255)"` + LeadTimeMinutes *uint `gorm:"type:int unsigned"` +} + +func (asanaTask20250212) TableName() string { + return "_tool_asana_tasks" +} + +type asanaScopeConfig20250212 struct { + TypeMappings string `gorm:"type:json"` + ApplicationType string `gorm:"type:varchar(255)"` + StoryPointField string `gorm:"type:varchar(255)"` + PriorityField string `gorm:"type:varchar(255)"` + EpicField string `gorm:"type:varchar(255)"` + SeverityField string `gorm:"type:varchar(255)"` + DueDateField string `gorm:"type:varchar(255)"` +} + +func (asanaScopeConfig20250212) TableName() string { + return "_tool_asana_scope_configs" +} + +func (*addTaskTransformationFields) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + // Add transformation fields to tasks table + if err := db.AutoMigrate(&asanaTask20250212{}); err != nil { + return err + } + // Add transformation config fields to scope_configs table + return db.AutoMigrate(&asanaScopeConfig20250212{}) +} + +func (*addTaskTransformationFields) Version() uint64 { + return 20250212000003 +} + +func (*addTaskTransformationFields) Name() string { + return "asana add task transformation fields for issue tracking" +} + diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go new file mode 100644 index 00000000000..48acd0ae629 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addUserPhotoUrl)(nil) + +type addUserPhotoUrl struct{} + +type asanaUser20250212 struct { + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` +} + +func (asanaUser20250212) TableName() string { + return "_tool_asana_users" +} + +func (*addUserPhotoUrl) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + // Add photo_url and workspace_gids columns to _tool_asana_users table + return db.AutoMigrate(&asanaUser20250212{}) +} + +func (*addUserPhotoUrl) Version() uint64 { + return 20250212000001 +} + +func (*addUserPhotoUrl) Name() string { + return "asana add photo_url and workspace_gids to users table" +} + diff --git a/backend/plugins/asana/models/migrationscripts/register.go b/backend/plugins/asana/models/migrationscripts/register.go index ec054748c27..2d98d7d3ff6 100644 --- a/backend/plugins/asana/models/migrationscripts/register.go +++ b/backend/plugins/asana/models/migrationscripts/register.go @@ -25,5 +25,8 @@ import ( func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), + new(addUserPhotoUrl), + new(addMissingTables), + new(addTaskTransformationFields), } } diff --git a/backend/plugins/asana/models/scope_config.go b/backend/plugins/asana/models/scope_config.go index 6c2666c0713..ed39bc2afda 100644 --- a/backend/plugins/asana/models/scope_config.go +++ b/backend/plugins/asana/models/scope_config.go @@ -21,8 +21,45 @@ import ( "github.com/apache/incubator-devlake/core/models/common" ) +// StatusMapping maps Asana statuses to standard statuses +type AsanaStatusMapping struct { + StandardStatus string `json:"standardStatus"` +} + +// AsanaStatusMappings is a map of section/status names to their mappings +type AsanaStatusMappings map[string]AsanaStatusMapping + +// AsanaTypeMapping maps Asana task types to standard types with their status mappings +type AsanaTypeMapping struct { + StandardType string `json:"standardType"` + StatusMappings AsanaStatusMappings `json:"statusMappings"` +} + type AsanaScopeConfig struct { common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + + // Type and Status Mappings (like Jira) + // Maps Asana resource_subtype (default_task, milestone, section, approval) to standard types + // Standard types: REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK + TypeMappings map[string]AsanaTypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` + + // Application type for categorization + ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"` + + // Story Point field - custom field name/gid that contains story points + StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"` + + // Priority field - custom field name/gid that contains priority + PriorityField string `mapstructure:"priorityField,omitempty" json:"priorityField" gorm:"type:varchar(255)"` + + // Epic field - custom field name/gid that links tasks to epics + EpicField string `mapstructure:"epicField,omitempty" json:"epicField" gorm:"type:varchar(255)"` + + // Severity field - custom field name/gid for severity (used for bugs/incidents) + SeverityField string `mapstructure:"severityField,omitempty" json:"severityField" gorm:"type:varchar(255)"` + + // Due date handling + DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"` } func (AsanaScopeConfig) TableName() string { diff --git a/backend/plugins/asana/models/story.go b/backend/plugins/asana/models/story.go new file mode 100644 index 00000000000..88e62a68346 --- /dev/null +++ b/backend/plugins/asana/models/story.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaStory represents comments and system-generated stories on tasks +type AsanaStory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(64)"` + Text string `gorm:"type:text"` + HtmlText string `gorm:"type:text"` + IsPinned bool `json:"isPinned"` + IsEdited bool `json:"isEdited"` + StickerName string `gorm:"type:varchar(64)"` + CreatedAt time.Time `json:"createdAt"` + CreatedByGid string `gorm:"type:varchar(255)"` + CreatedByName string `gorm:"type:varchar(255)"` + TaskGid string `gorm:"type:varchar(255);index"` + TargetGid string `gorm:"type:varchar(255);index"` + common.NoPKModel +} + +func (AsanaStory) TableName() string { + return "_tool_asana_stories" +} + diff --git a/backend/plugins/asana/models/tag.go b/backend/plugins/asana/models/tag.go new file mode 100644 index 00000000000..f97cb77f0cb --- /dev/null +++ b/backend/plugins/asana/models/tag.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Color string `gorm:"type:varchar(32)"` + Notes string `gorm:"type:text"` + WorkspaceGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + common.NoPKModel +} + +func (AsanaTag) TableName() string { + return "_tool_asana_tags" +} + +// AsanaTaskTag is a many-to-many relationship between tasks and tags +type AsanaTaskTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + TagGid string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (AsanaTaskTag) TableName() string { + return "_tool_asana_task_tags" +} + diff --git a/backend/plugins/asana/models/task.go b/backend/plugins/asana/models/task.go index 5c64a319629..fce6bd9388b 100644 --- a/backend/plugins/asana/models/task.go +++ b/backend/plugins/asana/models/task.go @@ -24,26 +24,40 @@ import ( ) type AsanaTask struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(512)"` - Notes string `gorm:"type:text"` - ResourceType string `gorm:"type:varchar(32)"` - ResourceSubtype string `gorm:"type:varchar(32)"` - Completed bool `json:"completed"` - CompletedAt *time.Time `json:"completedAt"` - DueOn *time.Time `gorm:"type:date" json:"dueOn"` - CreatedAt time.Time `json:"createdAt"` - ModifiedAt *time.Time `json:"modifiedAt"` - PermalinkUrl string `gorm:"type:varchar(512)"` - ProjectGid string `gorm:"type:varchar(255);index"` - SectionGid string `gorm:"type:varchar(255);index"` - AssigneeGid string `gorm:"type:varchar(255)"` - AssigneeName string `gorm:"type:varchar(255)"` - CreatorGid string `gorm:"type:varchar(255)"` - CreatorName string `gorm:"type:varchar(255)"` - ParentGid string `gorm:"type:varchar(255);index"` - NumSubtasks int `json:"numSubtasks"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(512)"` + Notes string `gorm:"type:text"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` // default_task, milestone, section, approval + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completedAt"` + DueOn *time.Time `gorm:"type:date" json:"dueOn"` + CreatedAt time.Time `json:"createdAt"` + ModifiedAt *time.Time `json:"modifiedAt"` + PermalinkUrl string `gorm:"type:varchar(512)"` + ProjectGid string `gorm:"type:varchar(255);index"` + SectionGid string `gorm:"type:varchar(255);index"` + SectionName string `gorm:"type:varchar(255)"` // For status mapping + AssigneeGid string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatorGid string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + ParentGid string `gorm:"type:varchar(255);index"` + NumSubtasks int `json:"numSubtasks"` + + // Transformed fields for domain layer + StdType string `gorm:"type:varchar(255)"` // Standard type: REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK + StdStatus string `gorm:"type:varchar(255)"` // Standard status: TODO, IN_PROGRESS, DONE + + // Custom field values (extracted during transformation) + Priority string `gorm:"type:varchar(255)"` // Priority from custom field + StoryPoint *float64 `json:"storyPoint"` // Story points from custom field + Severity string `gorm:"type:varchar(255)"` // Severity from custom field + + // Lead time tracking + LeadTimeMinutes *uint `json:"leadTimeMinutes"` + common.NoPKModel } diff --git a/backend/plugins/asana/models/team.go b/backend/plugins/asana/models/team.go new file mode 100644 index 00000000000..a05d8e05282 --- /dev/null +++ b/backend/plugins/asana/models/team.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + HtmlDescription string `gorm:"type:text"` + OrganizationGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + common.NoPKModel +} + +func (AsanaTeam) TableName() string { + return "_tool_asana_teams" +} + diff --git a/backend/plugins/asana/models/user.go b/backend/plugins/asana/models/user.go index 242be27a79e..59ce8342162 100644 --- a/backend/plugins/asana/models/user.go +++ b/backend/plugins/asana/models/user.go @@ -22,11 +22,13 @@ import ( ) type AsanaUser struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - Email string `gorm:"type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` // JSON array of workspace GIDs common.NoPKModel } diff --git a/backend/plugins/asana/models/workspace.go b/backend/plugins/asana/models/workspace.go new file mode 100644 index 00000000000..e64db849701 --- /dev/null +++ b/backend/plugins/asana/models/workspace.go @@ -0,0 +1,36 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaWorkspace struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + IsOrganization bool `json:"isOrganization"` + common.NoPKModel +} + +func (AsanaWorkspace) TableName() string { + return "_tool_asana_workspaces" +} + diff --git a/backend/plugins/asana/tasks/story_collector.go b/backend/plugins/asana/tasks/story_collector.go new file mode 100644 index 00000000000..ea164da9641 --- /dev/null +++ b/backend/plugins/asana/tasks/story_collector.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawStoryTable = "asana_stories" + +var _ plugin.SubTaskEntryPoint = CollectStory + +var CollectStoryMeta = plugin.SubTaskMeta{ + Name: "CollectStory", + EntryPoint: CollectStory, + EnabledByDefault: true, + Description: "Collect story/comment data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectStory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks for this project + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawStoryTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/stories", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,resource_type,resource_subtype,text,html_text,is_pinned,is_edited,sticker_name,created_at,created_by,target") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +type simpleTask struct { + Gid string +} + diff --git a/backend/plugins/asana/tasks/story_convertor.go b/backend/plugins/asana/tasks/story_convertor.go new file mode 100644 index 00000000000..8d8d09e9630 --- /dev/null +++ b/backend/plugins/asana/tasks/story_convertor.go @@ -0,0 +1,89 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertStory + +var ConvertStoryMeta = plugin.SubTaskMeta{ + Name: "ConvertStory", + EntryPoint: ConvertStory, + EnabledByDefault: true, + Description: "Convert tool layer Asana stories into domain layer issue comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertStory(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawStoryTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + // Only convert comment-type stories (not system-generated ones) + clauses := []dal.Clause{ + dal.From(&models.AsanaStory{}), + dal.Join("LEFT JOIN _tool_asana_tasks ON _tool_asana_stories.task_gid = _tool_asana_tasks.gid AND _tool_asana_stories.connection_id = _tool_asana_tasks.connection_id"), + dal.Where("_tool_asana_stories.connection_id = ? AND _tool_asana_tasks.project_gid = ? AND _tool_asana_stories.resource_subtype = ?", + connectionId, projectId, "comment_added"), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + commentIdGen := didgen.NewDomainIdGenerator(&models.AsanaStory{}) + taskIdGen := didgen.NewDomainIdGenerator(&models.AsanaTask{}) + userIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaStory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolStory := inputRow.(*models.AsanaStory) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(toolStory.ConnectionId, toolStory.Gid)}, + IssueId: taskIdGen.Generate(toolStory.ConnectionId, toolStory.TaskGid), + Body: toolStory.Text, + CreatedDate: toolStory.CreatedAt, + } + if toolStory.CreatedByGid != "" { + domainComment.AccountId = userIdGen.Generate(toolStory.ConnectionId, toolStory.CreatedByGid) + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + diff --git a/backend/plugins/asana/tasks/story_extractor.go b/backend/plugins/asana/tasks/story_extractor.go new file mode 100644 index 00000000000..9c1e2315c25 --- /dev/null +++ b/backend/plugins/asana/tasks/story_extractor.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractStory + +var ExtractStoryMeta = plugin.SubTaskMeta{ + Name: "ExtractStory", + EntryPoint: ExtractStory, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_stories", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiStory struct { + Gid string `json:"gid"` + ResourceType string `json:"resource_type"` + ResourceSubtype string `json:"resource_subtype"` + Text string `json:"text"` + HtmlText string `json:"html_text"` + IsPinned bool `json:"is_pinned"` + IsEdited bool `json:"is_edited"` + StickerName string `json:"sticker_name"` + CreatedAt time.Time `json:"created_at"` + CreatedBy *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"created_by"` + Target *struct { + Gid string `json:"gid"` + } `json:"target"` +} + +func ExtractStory(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawStoryTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiStory := &asanaApiStory{} + err := errors.Convert(json.Unmarshal(resData.Data, apiStory)) + if err != nil { + return nil, err + } + + // Extract task GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + createdByGid := "" + createdByName := "" + if apiStory.CreatedBy != nil { + createdByGid = apiStory.CreatedBy.Gid + createdByName = apiStory.CreatedBy.Name + } + targetGid := "" + if apiStory.Target != nil { + targetGid = apiStory.Target.Gid + } + + toolStory := &models.AsanaStory{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiStory.Gid, + ResourceType: apiStory.ResourceType, + ResourceSubtype: apiStory.ResourceSubtype, + Text: apiStory.Text, + HtmlText: apiStory.HtmlText, + IsPinned: apiStory.IsPinned, + IsEdited: apiStory.IsEdited, + StickerName: apiStory.StickerName, + CreatedAt: apiStory.CreatedAt, + CreatedByGid: createdByGid, + CreatedByName: createdByName, + TaskGid: input.Gid, + TargetGid: targetGid, + } + return []interface{}{toolStory}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + diff --git a/backend/plugins/asana/tasks/subtask_collector.go b/backend/plugins/asana/tasks/subtask_collector.go new file mode 100644 index 00000000000..aab12e314d7 --- /dev/null +++ b/backend/plugins/asana/tasks/subtask_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawSubtaskTable = "asana_subtasks" + +var _ plugin.SubTaskEntryPoint = CollectSubtask + +var CollectSubtaskMeta = plugin.SubTaskMeta{ + Name: "CollectSubtask", + EntryPoint: CollectSubtask, + EnabledByDefault: true, + Description: "Collect subtask data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectSubtask(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks that have subtasks + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ? AND num_subtasks > 0", + data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawSubtaskTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/subtasks", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,created_by,parent,num_subtasks,memberships.section,memberships.project") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + diff --git a/backend/plugins/asana/tasks/subtask_extractor.go b/backend/plugins/asana/tasks/subtask_extractor.go new file mode 100644 index 00000000000..3f5b1d496d2 --- /dev/null +++ b/backend/plugins/asana/tasks/subtask_extractor.go @@ -0,0 +1,125 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractSubtask + +var ExtractSubtaskMeta = plugin.SubTaskMeta{ + Name: "ExtractSubtask", + EntryPoint: ExtractSubtask, + EnabledByDefault: true, + Description: "Extract raw subtask data into tool layer table _tool_asana_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractSubtask(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawSubtaskTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTask := &asanaApiTask{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTask)) + if err != nil { + return nil, err + } + + // Get parent GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + assigneeGid := "" + assigneeName := "" + if apiTask.Assignee != nil { + assigneeGid = apiTask.Assignee.Gid + assigneeName = apiTask.Assignee.Name + } + creatorGid := "" + creatorName := "" + if apiTask.CreatedBy != nil { + creatorGid = apiTask.CreatedBy.Gid + creatorName = apiTask.CreatedBy.Name + } + sectionGid := "" + projectGid := taskData.Options.ProjectId + for _, m := range apiTask.Memberships { + if m.Project != nil { + projectGid = m.Project.Gid + } + if m.Section != nil && m.Section.Gid != "" { + sectionGid = m.Section.Gid + break + } + } + + var dueOn *time.Time + if apiTask.DueOn != "" { + dueOn = parseAsanaDate(apiTask.DueOn) + } + + toolTask := &models.AsanaTask{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTask.Gid, + Name: apiTask.Name, + Notes: apiTask.Notes, + ResourceType: apiTask.ResourceType, + ResourceSubtype: apiTask.ResourceSubtype, + Completed: apiTask.Completed, + CompletedAt: apiTask.CompletedAt, + DueOn: dueOn, + CreatedAt: apiTask.CreatedAt, + ModifiedAt: apiTask.ModifiedAt, + PermalinkUrl: apiTask.PermalinkUrl, + ProjectGid: projectGid, + SectionGid: sectionGid, + AssigneeGid: assigneeGid, + AssigneeName: assigneeName, + CreatorGid: creatorGid, + CreatorName: creatorName, + ParentGid: input.Gid, // Parent is the task that has subtasks + NumSubtasks: apiTask.NumSubtasks, + } + return []interface{}{toolTask}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + diff --git a/backend/plugins/asana/tasks/tag_collector.go b/backend/plugins/asana/tasks/tag_collector.go new file mode 100644 index 00000000000..7772d14e617 --- /dev/null +++ b/backend/plugins/asana/tasks/tag_collector.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawTagTable = "asana_tags" + +var _ plugin.SubTaskEntryPoint = CollectTag + +var CollectTagMeta = plugin.SubTaskMeta{ + Name: "CollectTag", + EntryPoint: CollectTag, + EnabledByDefault: true, + Description: "Collect tag data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTag(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks for this project + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawTagTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/tags", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,resource_type,color,notes,permalink_url") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + diff --git a/backend/plugins/asana/tasks/tag_extractor.go b/backend/plugins/asana/tasks/tag_extractor.go new file mode 100644 index 00000000000..f9ac94e0b4a --- /dev/null +++ b/backend/plugins/asana/tasks/tag_extractor.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTag + +var ExtractTagMeta = plugin.SubTaskMeta{ + Name: "ExtractTag", + EntryPoint: ExtractTag, + EnabledByDefault: true, + Description: "Extract raw data into tool layer tables _tool_asana_tags and _tool_asana_task_tags", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiTag struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Color string `json:"color"` + Notes string `json:"notes"` + PermalinkUrl string `json:"permalink_url"` +} + +func ExtractTag(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawTagTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTag := &asanaApiTag{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTag)) + if err != nil { + return nil, err + } + + // Get task GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + toolTag := &models.AsanaTag{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTag.Gid, + Name: apiTag.Name, + ResourceType: apiTag.ResourceType, + Color: apiTag.Color, + Notes: apiTag.Notes, + PermalinkUrl: apiTag.PermalinkUrl, + } + + // Create the task-tag relationship + taskTag := &models.AsanaTaskTag{ + ConnectionId: taskData.Options.ConnectionId, + TaskGid: input.Gid, + TagGid: apiTag.Gid, + } + + return []interface{}{toolTag, taskTag}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go index aad1188537d..1550f59211c 100644 --- a/backend/plugins/asana/tasks/task_collector.go +++ b/backend/plugins/asana/tasks/task_collector.go @@ -66,6 +66,8 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("limit", "100") + // Request all fields needed for transformation including section name + query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,assignee.name,created_by,created_by.name,parent,num_subtasks,memberships.section,memberships.section.name,memberships.project") if reqData.CustomData != nil { if offset, ok := reqData.CustomData.(string); ok && offset != "" { query.Set("offset", offset) diff --git a/backend/plugins/asana/tasks/task_convertor.go b/backend/plugins/asana/tasks/task_convertor.go index b790111f69c..a3a393f82df 100644 --- a/backend/plugins/asana/tasks/task_convertor.go +++ b/backend/plugins/asana/tasks/task_convertor.go @@ -46,6 +46,9 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId projectId := data.Options.ProjectId + // Get scope config for type/status mappings + scopeConfig := getScopeConfig(taskCtx) + clauses := []dal.Clause{ dal.From(&models.AsanaTask{}), dal.Where("connection_id = ? AND project_gid = ?", connectionId, projectId), @@ -58,6 +61,7 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { taskIdGen := didgen.NewDomainIdGenerator(&models.AsanaTask{}) boardIdGen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) converter, err := helper.NewDataConverter(helper.DataConverterArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, @@ -65,33 +69,73 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { toolTask := inputRow.(*models.AsanaTask) + + // Map type and status using scope config + stdType, stdStatus := getStdTypeAndStatus(toolTask, scopeConfig) + domainIssue := &ticket.Issue{ DomainEntity: domainlayer.DomainEntity{Id: taskIdGen.Generate(toolTask.ConnectionId, toolTask.Gid)}, IssueKey: toolTask.Gid, Title: toolTask.Name, Description: toolTask.Notes, Url: toolTask.PermalinkUrl, + Type: stdType, + OriginalType: toolTask.ResourceSubtype, + Status: stdStatus, + OriginalStatus: getOriginalStatus(toolTask), + Priority: toolTask.Priority, + StoryPoint: toolTask.StoryPoint, CreatedDate: &toolTask.CreatedAt, UpdatedDate: toolTask.ModifiedAt, ResolutionDate: toolTask.CompletedAt, DueDate: toolTask.DueOn, CreatorName: toolTask.CreatorName, AssigneeName: toolTask.AssigneeName, - OriginalType: toolTask.ResourceSubtype, + LeadTimeMinutes: toolTask.LeadTimeMinutes, } - if toolTask.Completed { - domainIssue.Status = ticket.DONE - domainIssue.OriginalStatus = "completed" - } else { - domainIssue.Status = ticket.TODO - domainIssue.OriginalStatus = "incomplete" + + // Set creator and assignee IDs + if toolTask.CreatorGid != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, toolTask.CreatorGid) + } + if toolTask.AssigneeGid != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, toolTask.AssigneeGid) } + + // Set parent issue ID if this is a subtask + if toolTask.ParentGid != "" { + domainIssue.ParentIssueId = taskIdGen.Generate(connectionId, toolTask.ParentGid) + // If no type mapping and has parent, it's a subtask + if stdType == "" { + domainIssue.Type = ticket.SUBTASK + } + } + + // Set subtask flag + domainIssue.IsSubtask = toolTask.ParentGid != "" + + var result []interface{} + result = append(result, domainIssue) + + // Create board issue relationship boardId := boardIdGen.Generate(connectionId, toolTask.ProjectGid) boardIssue := &ticket.BoardIssue{ BoardId: boardId, IssueId: domainIssue.Id, } - return []interface{}{domainIssue, boardIssue}, nil + result = append(result, boardIssue) + + // Create issue assignee if assignee exists + if toolTask.AssigneeGid != "" { + issueAssignee := &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: toolTask.AssigneeName, + } + result = append(result, issueAssignee) + } + + return result, nil }, }) if err != nil { @@ -99,3 +143,93 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { } return converter.Execute() } + +// getScopeConfig retrieves the scope config for transformation rules +func getScopeConfig(taskCtx plugin.SubTaskContext) *models.AsanaScopeConfig { + if taskCtx.GetData() == nil { + return nil + } + data := taskCtx.GetData().(*AsanaTaskData) + if data.Options.ScopeConfigId == 0 { + return nil + } + db := taskCtx.GetDal() + var scopeConfig models.AsanaScopeConfig + err := db.First(&scopeConfig, dal.Where("id = ?", data.Options.ScopeConfigId)) + if err != nil { + return nil + } + return &scopeConfig +} + +// getStdTypeAndStatus maps Asana task to standard type and status +func getStdTypeAndStatus(task *models.AsanaTask, scopeConfig *models.AsanaScopeConfig) (string, string) { + stdType := "" + stdStatus := "" + + // Default status based on completion + if task.Completed { + stdStatus = ticket.DONE + } else { + stdStatus = ticket.TODO + } + + // Use pre-computed values if available + if task.StdType != "" { + stdType = task.StdType + } + if task.StdStatus != "" { + stdStatus = task.StdStatus + } + + // Apply scope config mappings if available + if scopeConfig != nil && scopeConfig.TypeMappings != nil { + // Map resource_subtype to standard type + if typeMapping, ok := scopeConfig.TypeMappings[task.ResourceSubtype]; ok { + if typeMapping.StandardType != "" { + stdType = typeMapping.StandardType + } + // Map section name to status if available + if task.SectionName != "" && typeMapping.StatusMappings != nil { + if statusMapping, ok := typeMapping.StatusMappings[task.SectionName]; ok { + if statusMapping.StandardStatus != "" { + stdStatus = statusMapping.StandardStatus + } + } + } + } + } + + // Default type mapping based on resource_subtype + // Asana terminology: default_task, milestone, approval, section + if stdType == "" { + switch task.ResourceSubtype { + case "milestone": + stdType = ticket.REQUIREMENT // Milestones represent key deliverables + case "approval": + stdType = ticket.TASK + case "section": + stdType = ticket.TASK // Sections are just groupings, not tasks themselves + default: + if task.ParentGid != "" { + stdType = ticket.SUBTASK + } else { + stdType = ticket.TASK + } + } + } + + return stdType, stdStatus +} + +// getOriginalStatus returns the original status string +func getOriginalStatus(task *models.AsanaTask) string { + if task.Completed { + return "completed" + } + if task.SectionName != "" { + return task.SectionName + } + return "incomplete" +} + diff --git a/backend/plugins/asana/tasks/task_extractor.go b/backend/plugins/asana/tasks/task_extractor.go index 43219c2a2a2..09ef7a6d538 100644 --- a/backend/plugins/asana/tasks/task_extractor.go +++ b/backend/plugins/asana/tasks/task_extractor.go @@ -63,7 +63,8 @@ type asanaApiTask struct { NumSubtasks int `json:"num_subtasks"` Memberships []struct { Section *struct { - Gid string `json:"gid"` + Gid string `json:"gid"` + Name string `json:"name"` } `json:"section"` Project *struct { Gid string `json:"gid"` @@ -116,6 +117,7 @@ func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { parentGid = apiTask.Parent.Gid } sectionGid := "" + sectionName := "" projectGid := taskData.Options.ProjectId for _, m := range apiTask.Memberships { if m.Project != nil { @@ -123,6 +125,7 @@ func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { } if m.Section != nil && m.Section.Gid != "" { sectionGid = m.Section.Gid + sectionName = m.Section.Name break } } @@ -141,6 +144,7 @@ func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { PermalinkUrl: apiTask.PermalinkUrl, ProjectGid: projectGid, SectionGid: sectionGid, + SectionName: sectionName, AssigneeGid: assigneeGid, AssigneeName: assigneeName, CreatorGid: creatorGid, diff --git a/backend/plugins/asana/tasks/user_collector.go b/backend/plugins/asana/tasks/user_collector.go new file mode 100644 index 00000000000..8303195f21a --- /dev/null +++ b/backend/plugins/asana/tasks/user_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawUserTable = "asana_users" + +var _ plugin.SubTaskEntryPoint = CollectUser + +var CollectUserMeta = plugin.SubTaskMeta{ + Name: "CollectUser", + EntryPoint: CollectUser, + EnabledByDefault: true, + Description: "Collect user data from Asana API (project members)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func CollectUser(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawUserTable, + }, + ApiClient: data.ApiClient, + PageSize: 100, + UrlTemplate: "projects/{{ .Params.ProjectId }}/members", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,email,resource_type,photo.image_128x128") + query.Set("limit", "100") + if reqData.CustomData != nil { + if offset, ok := reqData.CustomData.(string); ok && offset != "" { + query.Set("offset", offset) + } + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(prevPageResponse, &resp) + if err != nil { + return nil, err + } + if resp.NextPage != nil && resp.NextPage.Offset != "" { + return resp.NextPage.Offset, nil + } + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +type asanaListResponse struct { + Data []json.RawMessage `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} + diff --git a/backend/plugins/asana/tasks/user_convertor.go b/backend/plugins/asana/tasks/user_convertor.go new file mode 100644 index 00000000000..20905410436 --- /dev/null +++ b/backend/plugins/asana/tasks/user_convertor.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertUser + +var ConvertUserMeta = plugin.SubTaskMeta{ + Name: "ConvertUser", + EntryPoint: ConvertUser, + EnabledByDefault: true, + Description: "Convert tool layer Asana users into domain layer accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func ConvertUser(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawUserTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + + clauses := []dal.Clause{ + dal.From(&models.AsanaUser{}), + dal.Where("connection_id = ?", connectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + accountIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaUser{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolUser := inputRow.(*models.AsanaUser) + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{Id: accountIdGen.Generate(toolUser.ConnectionId, toolUser.Gid)}, + Email: toolUser.Email, + FullName: toolUser.Name, + UserName: toolUser.Name, + AvatarUrl: toolUser.PhotoUrl, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + diff --git a/backend/plugins/asana/tasks/user_extractor.go b/backend/plugins/asana/tasks/user_extractor.go new file mode 100644 index 00000000000..3804df07bb5 --- /dev/null +++ b/backend/plugins/asana/tasks/user_extractor.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractUser + +var ExtractUserMeta = plugin.SubTaskMeta{ + Name: "ExtractUser", + EntryPoint: ExtractUser, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_users", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +type asanaApiUser struct { + Gid string `json:"gid"` + Name string `json:"name"` + Email string `json:"email"` + ResourceType string `json:"resource_type"` + Photo *struct { + Image128x128 string `json:"image_128x128"` + } `json:"photo"` +} + +func ExtractUser(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawUserTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiUser := &asanaApiUser{} + err := errors.Convert(json.Unmarshal(resData.Data, apiUser)) + if err != nil { + return nil, err + } + photoUrl := "" + if apiUser.Photo != nil { + photoUrl = apiUser.Photo.Image128x128 + } + toolUser := &models.AsanaUser{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiUser.Gid, + Name: apiUser.Name, + Email: apiUser.Email, + ResourceType: apiUser.ResourceType, + PhotoUrl: photoUrl, + } + return []interface{}{toolUser}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + diff --git a/config-ui/src/plugins/register/asana/config.tsx b/config-ui/src/plugins/register/asana/config.tsx index 26720e8d5a3..975d5786428 100644 --- a/config-ui/src/plugins/register/asana/config.tsx +++ b/config-ui/src/plugins/register/asana/config.tsx @@ -16,6 +16,7 @@ * */ +import { DOC_URL } from '@/release'; import { IPluginConfig } from '@/types'; import Icon from './assets/icon.svg?react'; @@ -26,6 +27,7 @@ export const AsanaConfig: IPluginConfig = { icon: ({ color }) => , sort: 12, connection: { + docLink: DOC_URL.PLUGIN.ASANA.BASIS, initialValues: { endpoint: 'https://app.asana.com/api/1.0/', }, @@ -47,9 +49,31 @@ export const AsanaConfig: IPluginConfig = { }, dataScope: { title: 'Projects', + millerColumn: { + columnCount: 4, + firstColumnTitle: 'Workspaces', + }, + searchPlaceholder: 'Search projects...', }, scopeConfig: { entities: ['TICKET'], - transformation: {}, + transformation: { + typeMappings: { + label: 'Issue Type Mappings', + externalInfo: 'Map Asana task types (default_task, milestone, approval) to standard types (REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK)', + }, + storyPointField: { + label: 'Story Point Field', + subLabel: 'Custom field name containing story points', + }, + priorityField: { + label: 'Priority Field', + subLabel: 'Custom field name containing priority', + }, + applicationField: { + label: 'Application Type', + subLabel: 'Application type for categorization', + }, + }, }, }; From 578bc47e849db9081bbd9c3fc4df69344a6b026d Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Fri, 13 Feb 2026 23:27:26 +0500 Subject: [PATCH 06/10] fix: Added required fixes --- ...0212_add_scope_config_issue_type_fields.go | 52 +++++ ...20250212_add_task_transformation_fields.go | 11 +- .../asana/models/migrationscripts/register.go | 1 + backend/plugins/asana/models/scope_config.go | 41 +--- backend/plugins/asana/tasks/task_convertor.go | 211 ++++++++++++------ .../src/plugins/register/asana/config.tsx | 19 +- config-ui/src/plugins/register/asana/index.ts | 3 +- .../plugins/register/asana/transformation.tsx | 116 ++++++++++ 8 files changed, 321 insertions(+), 133 deletions(-) create mode 100644 backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go create mode 100644 config-ui/src/plugins/register/asana/transformation.tsx diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go new file mode 100644 index 00000000000..737d32e7b59 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addScopeConfigIssueTypeFields)(nil) + +type addScopeConfigIssueTypeFields struct{} + +type asanaScopeConfig20250212v2 struct { + IssueTypeRequirement string `gorm:"type:varchar(255)"` + IssueTypeBug string `gorm:"type:varchar(255)"` + IssueTypeIncident string `gorm:"type:varchar(255)"` +} + +func (asanaScopeConfig20250212v2) TableName() string { + return "_tool_asana_scope_configs" +} + +func (*addScopeConfigIssueTypeFields) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + return db.AutoMigrate(&asanaScopeConfig20250212v2{}) +} + +func (*addScopeConfigIssueTypeFields) Version() uint64 { + return 20250212000004 +} + +func (*addScopeConfigIssueTypeFields) Name() string { + return "asana add issue type fields to scope config" +} + diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go index e895cc7367e..65d96e73e5d 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go @@ -42,13 +42,10 @@ func (asanaTask20250212) TableName() string { } type asanaScopeConfig20250212 struct { - TypeMappings string `gorm:"type:json"` - ApplicationType string `gorm:"type:varchar(255)"` - StoryPointField string `gorm:"type:varchar(255)"` - PriorityField string `gorm:"type:varchar(255)"` - EpicField string `gorm:"type:varchar(255)"` - SeverityField string `gorm:"type:varchar(255)"` - DueDateField string `gorm:"type:varchar(255)"` + // Regex patterns for tag-based type classification (like GitHub) + IssueTypeRequirement string `gorm:"type:varchar(255)"` + IssueTypeBug string `gorm:"type:varchar(255)"` + IssueTypeIncident string `gorm:"type:varchar(255)"` } func (asanaScopeConfig20250212) TableName() string { diff --git a/backend/plugins/asana/models/migrationscripts/register.go b/backend/plugins/asana/models/migrationscripts/register.go index 2d98d7d3ff6..0c41babcbf4 100644 --- a/backend/plugins/asana/models/migrationscripts/register.go +++ b/backend/plugins/asana/models/migrationscripts/register.go @@ -28,5 +28,6 @@ func All() []plugin.MigrationScript { new(addUserPhotoUrl), new(addMissingTables), new(addTaskTransformationFields), + new(addScopeConfigIssueTypeFields), } } diff --git a/backend/plugins/asana/models/scope_config.go b/backend/plugins/asana/models/scope_config.go index ed39bc2afda..560e645a5e5 100644 --- a/backend/plugins/asana/models/scope_config.go +++ b/backend/plugins/asana/models/scope_config.go @@ -21,45 +21,14 @@ import ( "github.com/apache/incubator-devlake/core/models/common" ) -// StatusMapping maps Asana statuses to standard statuses -type AsanaStatusMapping struct { - StandardStatus string `json:"standardStatus"` -} - -// AsanaStatusMappings is a map of section/status names to their mappings -type AsanaStatusMappings map[string]AsanaStatusMapping - -// AsanaTypeMapping maps Asana task types to standard types with their status mappings -type AsanaTypeMapping struct { - StandardType string `json:"standardType"` - StatusMappings AsanaStatusMappings `json:"statusMappings"` -} - type AsanaScopeConfig struct { common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` - // Type and Status Mappings (like Jira) - // Maps Asana resource_subtype (default_task, milestone, section, approval) to standard types - // Standard types: REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK - TypeMappings map[string]AsanaTypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` - - // Application type for categorization - ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"` - - // Story Point field - custom field name/gid that contains story points - StoryPointField string `mapstructure:"storyPointField,omitempty" json:"storyPointField" gorm:"type:varchar(255)"` - - // Priority field - custom field name/gid that contains priority - PriorityField string `mapstructure:"priorityField,omitempty" json:"priorityField" gorm:"type:varchar(255)"` - - // Epic field - custom field name/gid that links tasks to epics - EpicField string `mapstructure:"epicField,omitempty" json:"epicField" gorm:"type:varchar(255)"` - - // Severity field - custom field name/gid for severity (used for bugs/incidents) - SeverityField string `mapstructure:"severityField,omitempty" json:"severityField" gorm:"type:varchar(255)"` - - // Due date handling - DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"` + // Issue type mapping using regex patterns (like GitHub) + // Tags matching these patterns will classify the task type + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` } func (AsanaScopeConfig) TableName() string { diff --git a/backend/plugins/asana/tasks/task_convertor.go b/backend/plugins/asana/tasks/task_convertor.go index a3a393f82df..a23ba2af493 100644 --- a/backend/plugins/asana/tasks/task_convertor.go +++ b/backend/plugins/asana/tasks/task_convertor.go @@ -19,6 +19,8 @@ package tasks import ( "reflect" + "regexp" + "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -46,9 +48,12 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId projectId := data.Options.ProjectId - // Get scope config for type/status mappings + // Get scope config for transformation rules scopeConfig := getScopeConfig(taskCtx) + // Get tags for tasks + taskTags := getTaskTags(db, connectionId) + clauses := []dal.Clause{ dal.From(&models.AsanaTask{}), dal.Where("connection_id = ? AND project_gid = ?", connectionId, projectId), @@ -70,27 +75,29 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { toolTask := inputRow.(*models.AsanaTask) - // Map type and status using scope config - stdType, stdStatus := getStdTypeAndStatus(toolTask, scopeConfig) + // Get tags for this task + tags := taskTags[toolTask.Gid] + + // Map type and status using scope config and tags + stdType, stdStatus := getStdTypeAndStatus(toolTask, scopeConfig, tags) domainIssue := &ticket.Issue{ - DomainEntity: domainlayer.DomainEntity{Id: taskIdGen.Generate(toolTask.ConnectionId, toolTask.Gid)}, - IssueKey: toolTask.Gid, - Title: toolTask.Name, - Description: toolTask.Notes, - Url: toolTask.PermalinkUrl, - Type: stdType, - OriginalType: toolTask.ResourceSubtype, - Status: stdStatus, - OriginalStatus: getOriginalStatus(toolTask), - Priority: toolTask.Priority, - StoryPoint: toolTask.StoryPoint, - CreatedDate: &toolTask.CreatedAt, - UpdatedDate: toolTask.ModifiedAt, - ResolutionDate: toolTask.CompletedAt, - DueDate: toolTask.DueOn, - CreatorName: toolTask.CreatorName, - AssigneeName: toolTask.AssigneeName, + DomainEntity: domainlayer.DomainEntity{Id: taskIdGen.Generate(toolTask.ConnectionId, toolTask.Gid)}, + IssueKey: toolTask.Gid, + Title: toolTask.Name, + Description: toolTask.Notes, + Url: toolTask.PermalinkUrl, + Type: stdType, + OriginalType: toolTask.ResourceSubtype, + Status: stdStatus, + OriginalStatus: getOriginalStatus(toolTask), + StoryPoint: toolTask.StoryPoint, + CreatedDate: &toolTask.CreatedAt, + UpdatedDate: toolTask.ModifiedAt, + ResolutionDate: toolTask.CompletedAt, + DueDate: toolTask.DueOn, + CreatorName: toolTask.CreatorName, + AssigneeName: toolTask.AssigneeName, LeadTimeMinutes: toolTask.LeadTimeMinutes, } @@ -105,8 +112,8 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { // Set parent issue ID if this is a subtask if toolTask.ParentGid != "" { domainIssue.ParentIssueId = taskIdGen.Generate(connectionId, toolTask.ParentGid) - // If no type mapping and has parent, it's a subtask - if stdType == "" { + // If no type determined and has parent, it's a subtask + if stdType == "" || stdType == ticket.TASK { domainIssue.Type = ticket.SUBTASK } } @@ -146,82 +153,140 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { // getScopeConfig retrieves the scope config for transformation rules func getScopeConfig(taskCtx plugin.SubTaskContext) *models.AsanaScopeConfig { + logger := taskCtx.GetLogger() if taskCtx.GetData() == nil { + logger.Info("getScopeConfig: taskCtx.GetData() is nil") return nil } data := taskCtx.GetData().(*AsanaTaskData) - if data.Options.ScopeConfigId == 0 { - return nil - } db := taskCtx.GetDal() - var scopeConfig models.AsanaScopeConfig - err := db.First(&scopeConfig, dal.Where("id = ?", data.Options.ScopeConfigId)) + + // First try to get by ScopeConfigId from options + if data.Options.ScopeConfigId != 0 { + var scopeConfig models.AsanaScopeConfig + err := db.First(&scopeConfig, dal.Where("id = ?", data.Options.ScopeConfigId)) + if err == nil { + logger.Info("getScopeConfig: Found scope config by ID %d, IssueTypeRequirement=%s, IssueTypeBug=%s, IssueTypeIncident=%s", + data.Options.ScopeConfigId, scopeConfig.IssueTypeRequirement, scopeConfig.IssueTypeBug, scopeConfig.IssueTypeIncident) + return &scopeConfig + } + logger.Info("getScopeConfig: Failed to get scope config by ID %d: %v", data.Options.ScopeConfigId, err) + } else { + logger.Info("getScopeConfig: ScopeConfigId is 0, trying to get from project") + } + + // Try to get scope config from project's scope_config_id + var project models.AsanaProject + err := db.First(&project, dal.Where("connection_id = ? AND gid = ?", data.Options.ConnectionId, data.Options.ProjectId)) if err != nil { + logger.Info("getScopeConfig: Failed to get project: %v", err) return nil } - return &scopeConfig + + if project.ScopeConfigId != 0 { + var scopeConfig models.AsanaScopeConfig + err := db.First(&scopeConfig, dal.Where("id = ?", project.ScopeConfigId)) + if err == nil { + logger.Info("getScopeConfig: Found scope config from project, IssueTypeRequirement=%s, IssueTypeBug=%s, IssueTypeIncident=%s", + scopeConfig.IssueTypeRequirement, scopeConfig.IssueTypeBug, scopeConfig.IssueTypeIncident) + return &scopeConfig + } + logger.Info("getScopeConfig: Failed to get scope config from project: %v", err) + } else { + logger.Info("getScopeConfig: Project has no scope_config_id") + } + + return nil +} + +// getTaskTags retrieves all tags for tasks and returns a map of taskGid -> []tagName +func getTaskTags(db dal.Dal, connectionId uint64) map[string][]string { + result := make(map[string][]string) + + var taskTags []models.AsanaTaskTag + err := db.All(&taskTags, dal.Where("connection_id = ?", connectionId)) + if err != nil { + return result + } + + // Get all tag names + tagNames := make(map[string]string) + var tags []models.AsanaTag + err = db.All(&tags, dal.Where("connection_id = ?", connectionId)) + if err == nil { + for _, tag := range tags { + tagNames[tag.Gid] = tag.Name + } + } + + // Build taskGid -> []tagName map + for _, tt := range taskTags { + if tagName, ok := tagNames[tt.TagGid]; ok { + result[tt.TaskGid] = append(result[tt.TaskGid], tagName) + } + } + + return result } -// getStdTypeAndStatus maps Asana task to standard type and status -func getStdTypeAndStatus(task *models.AsanaTask, scopeConfig *models.AsanaScopeConfig) (string, string) { - stdType := "" - stdStatus := "" +// getStdTypeAndStatus maps Asana task to standard type and status using regex patterns (like GitHub) +func getStdTypeAndStatus(task *models.AsanaTask, scopeConfig *models.AsanaScopeConfig, tags []string) (string, string) { + stdType := ticket.TASK + stdStatus := ticket.TODO // Default status based on completion if task.Completed { stdStatus = ticket.DONE - } else { - stdStatus = ticket.TODO } - // Use pre-computed values if available - if task.StdType != "" { - stdType = task.StdType - } - if task.StdStatus != "" { - stdStatus = task.StdStatus + // If no scope config, return defaults + if scopeConfig == nil { + return getDefaultType(task), stdStatus } - // Apply scope config mappings if available - if scopeConfig != nil && scopeConfig.TypeMappings != nil { - // Map resource_subtype to standard type - if typeMapping, ok := scopeConfig.TypeMappings[task.ResourceSubtype]; ok { - if typeMapping.StandardType != "" { - stdType = typeMapping.StandardType - } - // Map section name to status if available - if task.SectionName != "" && typeMapping.StatusMappings != nil { - if statusMapping, ok := typeMapping.StatusMappings[task.SectionName]; ok { - if statusMapping.StandardStatus != "" { - stdStatus = statusMapping.StandardStatus - } - } - } - } + // Combine all tags into a single string for matching + tagString := strings.ToLower(strings.Join(tags, " ")) + + // Match issue type using regex patterns (like GitHub) + if scopeConfig.IssueTypeRequirement != "" && matchPattern(tagString, scopeConfig.IssueTypeRequirement) { + stdType = ticket.REQUIREMENT + } + if scopeConfig.IssueTypeBug != "" && matchPattern(tagString, scopeConfig.IssueTypeBug) { + stdType = ticket.BUG + } + if scopeConfig.IssueTypeIncident != "" && matchPattern(tagString, scopeConfig.IssueTypeIncident) { + stdType = ticket.INCIDENT } - // Default type mapping based on resource_subtype - // Asana terminology: default_task, milestone, approval, section - if stdType == "" { - switch task.ResourceSubtype { - case "milestone": - stdType = ticket.REQUIREMENT // Milestones represent key deliverables - case "approval": - stdType = ticket.TASK - case "section": - stdType = ticket.TASK // Sections are just groupings, not tasks themselves - default: - if task.ParentGid != "" { - stdType = ticket.SUBTASK - } else { - stdType = ticket.TASK - } - } + // If no type matched and task is a subtask, mark it as subtask + if stdType == ticket.TASK && task.ParentGid != "" { + stdType = ticket.SUBTASK } return stdType, stdStatus } +// getDefaultType returns the default type based on task properties +func getDefaultType(task *models.AsanaTask) string { + if task.ParentGid != "" { + return ticket.SUBTASK + } + return ticket.TASK +} + +// matchPattern checks if the input string matches the regex pattern +func matchPattern(input, pattern string) bool { + if pattern == "" { + return false + } + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + return false + } + return re.MatchString(input) +} + + // getOriginalStatus returns the original status string func getOriginalStatus(task *models.AsanaTask) string { if task.Completed { diff --git a/config-ui/src/plugins/register/asana/config.tsx b/config-ui/src/plugins/register/asana/config.tsx index 975d5786428..af570c0e80e 100644 --- a/config-ui/src/plugins/register/asana/config.tsx +++ b/config-ui/src/plugins/register/asana/config.tsx @@ -58,22 +58,9 @@ export const AsanaConfig: IPluginConfig = { scopeConfig: { entities: ['TICKET'], transformation: { - typeMappings: { - label: 'Issue Type Mappings', - externalInfo: 'Map Asana task types (default_task, milestone, approval) to standard types (REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK)', - }, - storyPointField: { - label: 'Story Point Field', - subLabel: 'Custom field name containing story points', - }, - priorityField: { - label: 'Priority Field', - subLabel: 'Custom field name containing priority', - }, - applicationField: { - label: 'Application Type', - subLabel: 'Application type for categorization', - }, + issueTypeRequirement: '(feat|feature|story|requirement)', + issueTypeBug: '(bug|defect|broken)', + issueTypeIncident: '(incident|outage|failure)', }, }, }; diff --git a/config-ui/src/plugins/register/asana/index.ts b/config-ui/src/plugins/register/asana/index.ts index ee80317ec68..5f16858cbe4 100644 --- a/config-ui/src/plugins/register/asana/index.ts +++ b/config-ui/src/plugins/register/asana/index.ts @@ -16,4 +16,5 @@ * */ -export { AsanaConfig } from './config'; +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/asana/transformation.tsx b/config-ui/src/plugins/register/asana/transformation.tsx new file mode 100644 index 00000000000..3277ee8e787 --- /dev/null +++ b/config-ui/src/plugins/register/asana/transformation.tsx @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { CaretRightOutlined } from '@ant-design/icons'; +import { theme, Collapse, Tag, Form, Input } from 'antd'; + +import { ExternalLink } from '@/components'; +import { DOC_URL } from '@/release'; + +interface Props { + entities: string[]; + connectionId: ID; + transformation: any; + setTransformation: React.Dispatch>; +} + +export const AsanaTransformation = ({ entities, transformation, setTransformation }: Props) => { + const { token } = theme.useToken(); + + const panelStyle: React.CSSProperties = { + marginBottom: 24, + background: token.colorFillAlter, + borderRadius: token.borderRadiusLG, + border: 'none', + }; + + return ( + } + style={{ background: token.colorBgContainer }} + size="large" + items={[ + { + key: 'TICKET', + label: 'Issue Tracking', + style: panelStyle, + children: ( + <> +

+ Tell DevLake what your Asana tags mean to view metrics such as{' '} + Bug Age,{' '} + DORA - Median Time to Restore Service, etc. +

+

+ DevLake defines three standard types of issues: REQUIREMENT, BUG and INCIDENT. Classify your Asana tasks + using tags that match the RegEx patterns below. +

+ + + setTransformation({ + ...transformation, + issueTypeRequirement: e.target.value, + }) + } + /> + + + + setTransformation({ + ...transformation, + issueTypeBug: e.target.value, + }) + } + /> + + + Incident + + DORA + + + } + > + + setTransformation({ + ...transformation, + issueTypeIncident: e.target.value, + }) + } + /> + + + ), + }, + ].filter((it) => entities.includes(it.key))} + /> + ); +}; From 614b7898281c91ab7ae3dd642c3de3683c2008ba Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Tue, 17 Feb 2026 20:20:49 +0500 Subject: [PATCH 07/10] fix: added missing files --- backend/plugins/asana/e2e/project_test.go | 97 ++++++++++ backend/plugins/asana/e2e/story_test.go | 0 backend/plugins/asana/e2e/tag_test.go | 89 +++++++++ .../plugins/asana/e2e/task_conversion_test.go | 182 ++++++++++++++++++ backend/plugins/asana/e2e/user_test.go | 95 +++++++++ .../components/scope-config-form/index.tsx | 12 +- 6 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/asana/e2e/project_test.go create mode 100644 backend/plugins/asana/e2e/story_test.go create mode 100644 backend/plugins/asana/e2e/tag_test.go create mode 100644 backend/plugins/asana/e2e/task_conversion_test.go create mode 100644 backend/plugins/asana/e2e/user_test.go diff --git a/backend/plugins/asana/e2e/project_test.go b/backend/plugins/asana/e2e/project_test.go new file mode 100644 index 00000000000..721b4aecdd7 --- /dev/null +++ b/backend/plugins/asana/e2e/project_test.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaProjectDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import raw data for projects + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_projects.csv", "_raw_asana_projects") + + // Verify project extraction + dataflowTester.FlushTabler(&models.AsanaProject{}) + dataflowTester.Subtask(tasks.ExtractProjectMeta, taskData) + + dataflowTester.VerifyTableWithOptions(models.AsanaProject{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_projects.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // Verify project conversion to domain layer board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertProjectMeta, taskData) + + dataflowTester.VerifyTable( + ticket.Board{}, + "./snapshot_tables/boards.csv", + []string{ + "id", + "name", + "description", + "url", + "created_date", + }, + ) +} + +func TestAsanaProjectWithScopeConfig(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + ScopeConfigId: 1, + }, + } + + // Import project with scope config association + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_projects.csv", "_raw_asana_projects") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_scope_configs.csv", &models.AsanaScopeConfig{}) + + // Extract project + dataflowTester.FlushTabler(&models.AsanaProject{}) + dataflowTester.Subtask(tasks.ExtractProjectMeta, taskData) + + // Verify project has scope_config_id + dataflowTester.VerifyTableWithOptions(models.AsanaProject{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_projects_with_scope_config.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} + diff --git a/backend/plugins/asana/e2e/story_test.go b/backend/plugins/asana/e2e/story_test.go new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/plugins/asana/e2e/tag_test.go b/backend/plugins/asana/e2e/tag_test.go new file mode 100644 index 00000000000..64294883d29 --- /dev/null +++ b/backend/plugins/asana/e2e/tag_test.go @@ -0,0 +1,89 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaTagDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import raw data for tags + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_tags.csv", "_raw_asana_tags") + + // Import tasks needed for tag collection (tags are collected per task) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_tasks_for_tags.csv", &models.AsanaTask{}) + + // Verify tag extraction + dataflowTester.FlushTabler(&models.AsanaTag{}) + dataflowTester.FlushTabler(&models.AsanaTaskTag{}) + dataflowTester.Subtask(tasks.ExtractTagMeta, taskData) + + dataflowTester.VerifyTableWithOptions(models.AsanaTag{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_tags.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.VerifyTableWithOptions(models.AsanaTaskTag{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_task_tags.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} + +func TestAsanaTagWithMultipleTasks(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import raw data with multiple tasks having tags + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_tags_multiple.csv", "_raw_asana_tags") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_tasks_multiple.csv", &models.AsanaTask{}) + + // Extract tags + dataflowTester.FlushTabler(&models.AsanaTag{}) + dataflowTester.FlushTabler(&models.AsanaTaskTag{}) + dataflowTester.Subtask(tasks.ExtractTagMeta, taskData) + + // Verify multiple task-tag relationships are created + dataflowTester.VerifyTableWithOptions(models.AsanaTaskTag{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_task_tags_multiple.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} + diff --git a/backend/plugins/asana/e2e/task_conversion_test.go b/backend/plugins/asana/e2e/task_conversion_test.go new file mode 100644 index 00000000000..a10c99fe212 --- /dev/null +++ b/backend/plugins/asana/e2e/task_conversion_test.go @@ -0,0 +1,182 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaTaskConversionDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + ScopeConfigId: 1, + }, + } + + // Import raw data tables + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_tasks.csv", "_raw_asana_tasks") + + // Import tool layer data needed for conversion + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_projects.csv", &models.AsanaProject{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_scope_configs.csv", &models.AsanaScopeConfig{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_tags.csv", &models.AsanaTag{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_task_tags.csv", &models.AsanaTaskTag{}) + + // Verify task extraction + dataflowTester.FlushTabler(&models.AsanaTask{}) + dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.AsanaTask{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_tasks.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // Verify task conversion to domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertTaskMeta, taskData) + + dataflowTester.VerifyTable( + ticket.Issue{}, + "./snapshot_tables/issues.csv", + []string{ + "id", + "url", + "issue_key", + "title", + "description", + "type", + "original_type", + "status", + "original_status", + "story_point", + "resolution_date", + "created_date", + "updated_date", + "lead_time_minutes", + "parent_issue_id", + "creator_id", + "creator_name", + "assignee_id", + "assignee_name", + "due_date", + }, + ) + + dataflowTester.VerifyTable( + ticket.BoardIssue{}, + "./snapshot_tables/board_issues.csv", + []string{"board_id", "issue_id"}, + ) + + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} + +func TestAsanaTaskWithTypeMapping(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + // Test with scope config that has issue type mappings + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + ScopeConfigId: 1, + }, + } + + // Import raw data and tool layer data + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_tasks_with_tags.csv", "_raw_asana_tasks") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_projects.csv", &models.AsanaProject{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_scope_configs_with_mappings.csv", &models.AsanaScopeConfig{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_tags.csv", &models.AsanaTag{}) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_task_tags_with_types.csv", &models.AsanaTaskTag{}) + + // Extract and convert + dataflowTester.FlushTabler(&models.AsanaTask{}) + dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData) + + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertTaskMeta, taskData) + + // Verify issues have correct type based on tag matching + dataflowTester.VerifyTable( + ticket.Issue{}, + "./snapshot_tables/issues_with_type_mapping.csv", + []string{ + "id", + "type", + "original_type", + "status", + }, + ) +} + +func TestAsanaSubtaskConversion(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + ScopeConfigId: 0, // No scope config + }, + } + + // Import subtask data + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_subtasks.csv", "_raw_asana_tasks") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_asana_projects.csv", &models.AsanaProject{}) + + // Extract + dataflowTester.FlushTabler(&models.AsanaTask{}) + dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData) + + // Convert + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertTaskMeta, taskData) + + // Verify subtasks have correct parent_issue_id and type=SUBTASK + dataflowTester.VerifyTable( + ticket.Issue{}, + "./snapshot_tables/issues_subtasks.csv", + []string{ + "id", + "type", + "parent_issue_id", + }, + ) +} + diff --git a/backend/plugins/asana/e2e/user_test.go b/backend/plugins/asana/e2e/user_test.go new file mode 100644 index 00000000000..b911997c48a --- /dev/null +++ b/backend/plugins/asana/e2e/user_test.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaUserDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import raw data for users + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_users.csv", "_raw_asana_users") + + // Verify user extraction + dataflowTester.FlushTabler(&models.AsanaUser{}) + dataflowTester.Subtask(tasks.ExtractUserMeta, taskData) + + dataflowTester.VerifyTableWithOptions(models.AsanaUser{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_users.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // Verify user conversion to domain layer accounts + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertUserMeta, taskData) + + dataflowTester.VerifyTable( + crossdomain.Account{}, + "./snapshot_tables/accounts.csv", + []string{ + "id", + "email", + "full_name", + "user_name", + "avatar_url", + }, + ) +} + +func TestAsanaUserWithPhotoUrl(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import users with photo URLs + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_asana_users_with_photos.csv", "_raw_asana_users") + + // Extract users + dataflowTester.FlushTabler(&models.AsanaUser{}) + dataflowTester.Subtask(tasks.ExtractUserMeta, taskData) + + // Verify photo_url is extracted + dataflowTester.VerifyTableWithOptions(models.AsanaUser{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_asana_users_with_photos.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} + diff --git a/config-ui/src/plugins/components/scope-config-form/index.tsx b/config-ui/src/plugins/components/scope-config-form/index.tsx index 0f45a5708ff..eeeb9e86f06 100644 --- a/config-ui/src/plugins/components/scope-config-form/index.tsx +++ b/config-ui/src/plugins/components/scope-config-form/index.tsx @@ -35,6 +35,7 @@ import { TapdTransformation } from '@/plugins/register/tapd'; import { BambooTransformation } from '@/plugins/register/bamboo'; import { CircleCITransformation } from '@/plugins/register/circleci'; import { ArgoCDTransformation } from '@/plugins/register/argocd'; +import { AsanaTransformation } from '@/plugins/register/asana'; import { DOC_URL } from '@/release'; import { operator } from '@/utils'; @@ -87,7 +88,7 @@ export const ScopeConfigForm = ({ setName(forceCreate ? `${res.name}-copy` : res.name); setEntities(res.entities ?? []); setTransformation(omit(res, ['id', 'connectionId', 'name', 'entities', 'createdAt', 'updatedAt'])); - } catch { } + } catch {} })(); }, [scopeConfigId]); @@ -285,6 +286,15 @@ export const ScopeConfigForm = ({ /> )} + {plugin === 'asana' && ( + + )} + {plugin === 'tapd' && scopeId && ( Date: Wed, 18 Feb 2026 12:48:18 +0500 Subject: [PATCH 08/10] fix: removed unneccessaey file --- backend/plugins/asana/e2e/story_test.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/plugins/asana/e2e/story_test.go diff --git a/backend/plugins/asana/e2e/story_test.go b/backend/plugins/asana/e2e/story_test.go deleted file mode 100644 index e69de29bb2d..00000000000 From 3c8ebad9fddee4ac08111abcc04ac4ad67f5759f Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Wed, 18 Feb 2026 12:57:07 +0500 Subject: [PATCH 09/10] fix: formatted files --- backend/plugins/asana/api/blueprint_v200.go | 8 ++-- backend/plugins/asana/e2e/project_test.go | 1 - backend/plugins/asana/e2e/tag_test.go | 1 - .../plugins/asana/e2e/task_conversion_test.go | 1 - backend/plugins/asana/e2e/user_test.go | 1 - backend/plugins/asana/models/custom_field.go | 37 +++++++++---------- backend/plugins/asana/models/membership.go | 1 - .../20250212_add_missing_tables.go | 1 - ...0212_add_scope_config_issue_type_fields.go | 1 - ...20250212_add_task_transformation_fields.go | 1 - .../20250212_add_user_photo_url.go | 1 - backend/plugins/asana/models/story.go | 29 +++++++-------- backend/plugins/asana/models/tag.go | 1 - backend/plugins/asana/models/team.go | 13 +++---- backend/plugins/asana/models/user.go | 14 +++---- backend/plugins/asana/models/workspace.go | 11 +++--- .../plugins/asana/tasks/project_collector.go | 2 +- .../plugins/asana/tasks/project_convertor.go | 1 - .../plugins/asana/tasks/story_collector.go | 1 - .../plugins/asana/tasks/story_convertor.go | 1 - .../plugins/asana/tasks/story_extractor.go | 1 - .../plugins/asana/tasks/subtask_collector.go | 1 - .../plugins/asana/tasks/subtask_extractor.go | 1 - backend/plugins/asana/tasks/tag_collector.go | 1 - backend/plugins/asana/tasks/tag_extractor.go | 1 - backend/plugins/asana/tasks/task_collector.go | 2 +- backend/plugins/asana/tasks/task_convertor.go | 2 - backend/plugins/asana/tasks/task_data.go | 4 +- backend/plugins/asana/tasks/task_extractor.go | 24 ++++++------ backend/plugins/asana/tasks/user_collector.go | 1 - backend/plugins/asana/tasks/user_convertor.go | 1 - backend/plugins/asana/tasks/user_extractor.go | 1 - 32 files changed, 70 insertions(+), 97 deletions(-) diff --git a/backend/plugins/asana/api/blueprint_v200.go b/backend/plugins/asana/api/blueprint_v200.go index 546fd12ab21..2820593e350 100644 --- a/backend/plugins/asana/api/blueprint_v200.go +++ b/backend/plugins/asana/api/blueprint_v200.go @@ -27,8 +27,8 @@ import ( coreModels "github.com/apache/incubator-devlake/core/models" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/helpers/srvhelper" ) @@ -70,9 +70,9 @@ func makePipelinePlanV200( subtaskMetas, scopeConfig.Entities, tasks.AsanaOptions{ - ConnectionId: connection.ID, - ProjectId: scope.Gid, - ScopeConfigId: scopeConfig.ID, + ConnectionId: connection.ID, + ProjectId: scope.Gid, + ScopeConfigId: scopeConfig.ID, }, ) if err != nil { diff --git a/backend/plugins/asana/e2e/project_test.go b/backend/plugins/asana/e2e/project_test.go index 721b4aecdd7..37db18d7c0d 100644 --- a/backend/plugins/asana/e2e/project_test.go +++ b/backend/plugins/asana/e2e/project_test.go @@ -94,4 +94,3 @@ func TestAsanaProjectWithScopeConfig(t *testing.T) { IgnoreTypes: []interface{}{common.NoPKModel{}}, }) } - diff --git a/backend/plugins/asana/e2e/tag_test.go b/backend/plugins/asana/e2e/tag_test.go index 64294883d29..411bf2f5b5d 100644 --- a/backend/plugins/asana/e2e/tag_test.go +++ b/backend/plugins/asana/e2e/tag_test.go @@ -86,4 +86,3 @@ func TestAsanaTagWithMultipleTasks(t *testing.T) { IgnoreTypes: []interface{}{common.NoPKModel{}}, }) } - diff --git a/backend/plugins/asana/e2e/task_conversion_test.go b/backend/plugins/asana/e2e/task_conversion_test.go index a10c99fe212..403a20b1b76 100644 --- a/backend/plugins/asana/e2e/task_conversion_test.go +++ b/backend/plugins/asana/e2e/task_conversion_test.go @@ -179,4 +179,3 @@ func TestAsanaSubtaskConversion(t *testing.T) { }, ) } - diff --git a/backend/plugins/asana/e2e/user_test.go b/backend/plugins/asana/e2e/user_test.go index b911997c48a..6d48d88cbfa 100644 --- a/backend/plugins/asana/e2e/user_test.go +++ b/backend/plugins/asana/e2e/user_test.go @@ -92,4 +92,3 @@ func TestAsanaUserWithPhotoUrl(t *testing.T) { IgnoreTypes: []interface{}{common.NoPKModel{}}, }) } - diff --git a/backend/plugins/asana/models/custom_field.go b/backend/plugins/asana/models/custom_field.go index f313d7abe40..aabade05f84 100644 --- a/backend/plugins/asana/models/custom_field.go +++ b/backend/plugins/asana/models/custom_field.go @@ -23,16 +23,16 @@ import ( // AsanaCustomField represents a custom field definition type AsanaCustomField struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` - ResourceSubtype string `gorm:"type:varchar(32)"` - Type string `gorm:"type:varchar(32)"` - Description string `gorm:"type:text"` - Precision int `json:"precision"` - IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` - HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Type string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + Precision int `json:"precision"` + IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` + HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` common.NoPKModel } @@ -42,19 +42,18 @@ func (AsanaCustomField) TableName() string { // AsanaTaskCustomFieldValue represents a custom field value on a task type AsanaTaskCustomFieldValue struct { - ConnectionId uint64 `gorm:"primaryKey"` - TaskGid string `gorm:"primaryKey;type:varchar(255)"` - CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` - CustomFieldName string `gorm:"type:varchar(255)"` - DisplayValue string `gorm:"type:text"` - TextValue string `gorm:"type:text"` + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldName string `gorm:"type:varchar(255)"` + DisplayValue string `gorm:"type:text"` + TextValue string `gorm:"type:text"` NumberValue *float64 `json:"numberValue"` - EnumValueGid string `gorm:"type:varchar(255)"` - EnumValueName string `gorm:"type:varchar(255)"` + EnumValueGid string `gorm:"type:varchar(255)"` + EnumValueName string `gorm:"type:varchar(255)"` common.NoPKModel } func (AsanaTaskCustomFieldValue) TableName() string { return "_tool_asana_task_custom_field_values" } - diff --git a/backend/plugins/asana/models/membership.go b/backend/plugins/asana/models/membership.go index f3469093fa2..572e2464ecd 100644 --- a/backend/plugins/asana/models/membership.go +++ b/backend/plugins/asana/models/membership.go @@ -46,4 +46,3 @@ type AsanaTeamMembership struct { func (AsanaTeamMembership) TableName() string { return "_tool_asana_team_memberships" } - diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go index ad9d2866c34..e9ede755357 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go @@ -49,4 +49,3 @@ func (*addMissingTables) Version() uint64 { func (*addMissingTables) Name() string { return "asana add missing tables for hierarchical data" } - diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go index 737d32e7b59..e092290c6ad 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go @@ -49,4 +49,3 @@ func (*addScopeConfigIssueTypeFields) Version() uint64 { func (*addScopeConfigIssueTypeFields) Name() string { return "asana add issue type fields to scope config" } - diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go index 65d96e73e5d..7af1404489b 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go @@ -69,4 +69,3 @@ func (*addTaskTransformationFields) Version() uint64 { func (*addTaskTransformationFields) Name() string { return "asana add task transformation fields for issue tracking" } - diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go index 48acd0ae629..fc87c9fa1aa 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go @@ -49,4 +49,3 @@ func (*addUserPhotoUrl) Version() uint64 { func (*addUserPhotoUrl) Name() string { return "asana add photo_url and workspace_gids to users table" } - diff --git a/backend/plugins/asana/models/story.go b/backend/plugins/asana/models/story.go index 88e62a68346..41386a41e24 100644 --- a/backend/plugins/asana/models/story.go +++ b/backend/plugins/asana/models/story.go @@ -25,24 +25,23 @@ import ( // AsanaStory represents comments and system-generated stories on tasks type AsanaStory struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` - ResourceSubtype string `gorm:"type:varchar(64)"` - Text string `gorm:"type:text"` - HtmlText string `gorm:"type:text"` - IsPinned bool `json:"isPinned"` - IsEdited bool `json:"isEdited"` - StickerName string `gorm:"type:varchar(64)"` - CreatedAt time.Time `json:"createdAt"` - CreatedByGid string `gorm:"type:varchar(255)"` - CreatedByName string `gorm:"type:varchar(255)"` - TaskGid string `gorm:"type:varchar(255);index"` - TargetGid string `gorm:"type:varchar(255);index"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(64)"` + Text string `gorm:"type:text"` + HtmlText string `gorm:"type:text"` + IsPinned bool `json:"isPinned"` + IsEdited bool `json:"isEdited"` + StickerName string `gorm:"type:varchar(64)"` + CreatedAt time.Time `json:"createdAt"` + CreatedByGid string `gorm:"type:varchar(255)"` + CreatedByName string `gorm:"type:varchar(255)"` + TaskGid string `gorm:"type:varchar(255);index"` + TargetGid string `gorm:"type:varchar(255);index"` common.NoPKModel } func (AsanaStory) TableName() string { return "_tool_asana_stories" } - diff --git a/backend/plugins/asana/models/tag.go b/backend/plugins/asana/models/tag.go index f97cb77f0cb..2ab29555ad2 100644 --- a/backend/plugins/asana/models/tag.go +++ b/backend/plugins/asana/models/tag.go @@ -48,4 +48,3 @@ type AsanaTaskTag struct { func (AsanaTaskTag) TableName() string { return "_tool_asana_task_tags" } - diff --git a/backend/plugins/asana/models/team.go b/backend/plugins/asana/models/team.go index a05d8e05282..1488a0fd00c 100644 --- a/backend/plugins/asana/models/team.go +++ b/backend/plugins/asana/models/team.go @@ -22,18 +22,17 @@ import ( ) type AsanaTeam struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` - Description string `gorm:"type:text"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` HtmlDescription string `gorm:"type:text"` OrganizationGid string `gorm:"type:varchar(255);index"` - PermalinkUrl string `gorm:"type:varchar(512)"` + PermalinkUrl string `gorm:"type:varchar(512)"` common.NoPKModel } func (AsanaTeam) TableName() string { return "_tool_asana_teams" } - diff --git a/backend/plugins/asana/models/user.go b/backend/plugins/asana/models/user.go index 59ce8342162..92681195945 100644 --- a/backend/plugins/asana/models/user.go +++ b/backend/plugins/asana/models/user.go @@ -22,13 +22,13 @@ import ( ) type AsanaUser struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - Email string `gorm:"type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` - PhotoUrl string `gorm:"type:varchar(512)"` - WorkspaceGids string `gorm:"type:text"` // JSON array of workspace GIDs + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` // JSON array of workspace GIDs common.NoPKModel } diff --git a/backend/plugins/asana/models/workspace.go b/backend/plugins/asana/models/workspace.go index e64db849701..101c249b358 100644 --- a/backend/plugins/asana/models/workspace.go +++ b/backend/plugins/asana/models/workspace.go @@ -22,15 +22,14 @@ import ( ) type AsanaWorkspace struct { - ConnectionId uint64 `gorm:"primaryKey"` - Gid string `gorm:"primaryKey;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` - ResourceType string `gorm:"type:varchar(32)"` - IsOrganization bool `json:"isOrganization"` + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + IsOrganization bool `json:"isOrganization"` common.NoPKModel } func (AsanaWorkspace) TableName() string { return "_tool_asana_workspaces" } - diff --git a/backend/plugins/asana/tasks/project_collector.go b/backend/plugins/asana/tasks/project_collector.go index 4e8416ba394..5ae69a68d63 100644 --- a/backend/plugins/asana/tasks/project_collector.go +++ b/backend/plugins/asana/tasks/project_collector.go @@ -54,7 +54,7 @@ func CollectProject(taskCtx plugin.SubTaskContext) errors.Error { }, Table: rawProjectTable, }, - ApiClient: data.ApiClient, + ApiClient: data.ApiClient, UrlTemplate: "projects/{{ .Params.ProjectId }}", ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var w asanaDataWrapper diff --git a/backend/plugins/asana/tasks/project_convertor.go b/backend/plugins/asana/tasks/project_convertor.go index 4f086b76f57..cd6f89576ec 100644 --- a/backend/plugins/asana/tasks/project_convertor.go +++ b/backend/plugins/asana/tasks/project_convertor.go @@ -78,4 +78,3 @@ func ConvertProject(taskCtx plugin.SubTaskContext) errors.Error { } return converter.Execute() } - diff --git a/backend/plugins/asana/tasks/story_collector.go b/backend/plugins/asana/tasks/story_collector.go index ea164da9641..7fa8896900e 100644 --- a/backend/plugins/asana/tasks/story_collector.go +++ b/backend/plugins/asana/tasks/story_collector.go @@ -98,4 +98,3 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error { type simpleTask struct { Gid string } - diff --git a/backend/plugins/asana/tasks/story_convertor.go b/backend/plugins/asana/tasks/story_convertor.go index 8d8d09e9630..f5df43b2ab7 100644 --- a/backend/plugins/asana/tasks/story_convertor.go +++ b/backend/plugins/asana/tasks/story_convertor.go @@ -86,4 +86,3 @@ func ConvertStory(taskCtx plugin.SubTaskContext) errors.Error { } return converter.Execute() } - diff --git a/backend/plugins/asana/tasks/story_extractor.go b/backend/plugins/asana/tasks/story_extractor.go index 9c1e2315c25..e0240f2da88 100644 --- a/backend/plugins/asana/tasks/story_extractor.go +++ b/backend/plugins/asana/tasks/story_extractor.go @@ -117,4 +117,3 @@ func ExtractStory(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } - diff --git a/backend/plugins/asana/tasks/subtask_collector.go b/backend/plugins/asana/tasks/subtask_collector.go index aab12e314d7..a2f22cd3a1a 100644 --- a/backend/plugins/asana/tasks/subtask_collector.go +++ b/backend/plugins/asana/tasks/subtask_collector.go @@ -95,4 +95,3 @@ func CollectSubtask(taskCtx plugin.SubTaskContext) errors.Error { } return collector.Execute() } - diff --git a/backend/plugins/asana/tasks/subtask_extractor.go b/backend/plugins/asana/tasks/subtask_extractor.go index 3f5b1d496d2..0e0b310facb 100644 --- a/backend/plugins/asana/tasks/subtask_extractor.go +++ b/backend/plugins/asana/tasks/subtask_extractor.go @@ -122,4 +122,3 @@ func ExtractSubtask(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } - diff --git a/backend/plugins/asana/tasks/tag_collector.go b/backend/plugins/asana/tasks/tag_collector.go index 7772d14e617..0f3d218664e 100644 --- a/backend/plugins/asana/tasks/tag_collector.go +++ b/backend/plugins/asana/tasks/tag_collector.go @@ -94,4 +94,3 @@ func CollectTag(taskCtx plugin.SubTaskContext) errors.Error { } return collector.Execute() } - diff --git a/backend/plugins/asana/tasks/tag_extractor.go b/backend/plugins/asana/tasks/tag_extractor.go index f9ac94e0b4a..3e7db18ac4c 100644 --- a/backend/plugins/asana/tasks/tag_extractor.go +++ b/backend/plugins/asana/tasks/tag_extractor.go @@ -96,4 +96,3 @@ func ExtractTag(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } - diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go index 1550f59211c..8e1b72c138b 100644 --- a/backend/plugins/asana/tasks/task_collector.go +++ b/backend/plugins/asana/tasks/task_collector.go @@ -41,7 +41,7 @@ var CollectTaskMeta = plugin.SubTaskMeta{ } type asanaTaskListResponse struct { - Data []json.RawMessage `json:"data"` + Data []json.RawMessage `json:"data"` NextPage *struct { Offset string `json:"offset"` Path string `json:"path"` diff --git a/backend/plugins/asana/tasks/task_convertor.go b/backend/plugins/asana/tasks/task_convertor.go index a23ba2af493..7c981b50d20 100644 --- a/backend/plugins/asana/tasks/task_convertor.go +++ b/backend/plugins/asana/tasks/task_convertor.go @@ -286,7 +286,6 @@ func matchPattern(input, pattern string) bool { return re.MatchString(input) } - // getOriginalStatus returns the original status string func getOriginalStatus(task *models.AsanaTask) string { if task.Completed { @@ -297,4 +296,3 @@ func getOriginalStatus(task *models.AsanaTask) string { } return "incomplete" } - diff --git a/backend/plugins/asana/tasks/task_data.go b/backend/plugins/asana/tasks/task_data.go index e8770e7a714..b6c82836b7b 100644 --- a/backend/plugins/asana/tasks/task_data.go +++ b/backend/plugins/asana/tasks/task_data.go @@ -37,8 +37,8 @@ func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string) (* } type AsanaOptions struct { - ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` - ProjectId string `json:"projectId" mapstructure:"projectId"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` + ProjectId string `json:"projectId" mapstructure:"projectId"` ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` } diff --git a/backend/plugins/asana/tasks/task_extractor.go b/backend/plugins/asana/tasks/task_extractor.go index 09ef7a6d538..11ce45af1e0 100644 --- a/backend/plugins/asana/tasks/task_extractor.go +++ b/backend/plugins/asana/tasks/task_extractor.go @@ -38,18 +38,18 @@ var ExtractTaskMeta = plugin.SubTaskMeta{ } type asanaApiTask struct { - Gid string `json:"gid"` - Name string `json:"name"` - Notes string `json:"notes"` - ResourceType string `json:"resource_type"` - ResourceSubtype string `json:"resource_subtype"` - Completed bool `json:"completed"` - CompletedAt *time.Time `json:"completed_at"` - DueOn string `json:"due_on"` - CreatedAt time.Time `json:"created_at"` - ModifiedAt *time.Time `json:"modified_at"` - PermalinkUrl string `json:"permalink_url"` - Assignee *struct { + Gid string `json:"gid"` + Name string `json:"name"` + Notes string `json:"notes"` + ResourceType string `json:"resource_type"` + ResourceSubtype string `json:"resource_subtype"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completed_at"` + DueOn string `json:"due_on"` + CreatedAt time.Time `json:"created_at"` + ModifiedAt *time.Time `json:"modified_at"` + PermalinkUrl string `json:"permalink_url"` + Assignee *struct { Gid string `json:"gid"` Name string `json:"name"` } `json:"assignee"` diff --git a/backend/plugins/asana/tasks/user_collector.go b/backend/plugins/asana/tasks/user_collector.go index 8303195f21a..9b1649aa1fb 100644 --- a/backend/plugins/asana/tasks/user_collector.go +++ b/backend/plugins/asana/tasks/user_collector.go @@ -99,4 +99,3 @@ type asanaListResponse struct { URI string `json:"uri"` } `json:"next_page"` } - diff --git a/backend/plugins/asana/tasks/user_convertor.go b/backend/plugins/asana/tasks/user_convertor.go index 20905410436..4d9f3da0802 100644 --- a/backend/plugins/asana/tasks/user_convertor.go +++ b/backend/plugins/asana/tasks/user_convertor.go @@ -78,4 +78,3 @@ func ConvertUser(taskCtx plugin.SubTaskContext) errors.Error { } return converter.Execute() } - diff --git a/backend/plugins/asana/tasks/user_extractor.go b/backend/plugins/asana/tasks/user_extractor.go index 3804df07bb5..763c5a2a766 100644 --- a/backend/plugins/asana/tasks/user_extractor.go +++ b/backend/plugins/asana/tasks/user_extractor.go @@ -83,4 +83,3 @@ func ExtractUser(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } - From 344bbabd56a75725bea9a857972063d389eebda3 Mon Sep 17 00:00:00 2001 From: jawad-khan Date: Wed, 18 Feb 2026 13:57:43 +0500 Subject: [PATCH 10/10] fux: fixed lint msg --- .../20250203_add_init_tables.go | 32 +-- .../20250212_add_missing_tables.go | 20 +- .../migrationscripts/archived/models.go | 263 ++++++++++++++++++ 3 files changed, 289 insertions(+), 26 deletions(-) create mode 100644 backend/plugins/asana/models/migrationscripts/archived/models.go diff --git a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go index 3b26a7dffed..a2dae698bd0 100644 --- a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go +++ b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go @@ -21,7 +21,7 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/helpers/migrationhelper" - "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts/archived" ) type addInitTables struct{} @@ -29,21 +29,21 @@ type addInitTables struct{} func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { return migrationhelper.AutoMigrateTables( basicRes, - &models.AsanaConnection{}, - &models.AsanaProject{}, - &models.AsanaScopeConfig{}, - &models.AsanaTask{}, - &models.AsanaSection{}, - &models.AsanaUser{}, - &models.AsanaWorkspace{}, - &models.AsanaTeam{}, - &models.AsanaStory{}, - &models.AsanaTag{}, - &models.AsanaTaskTag{}, - &models.AsanaCustomField{}, - &models.AsanaTaskCustomFieldValue{}, - &models.AsanaProjectMembership{}, - &models.AsanaTeamMembership{}, + &archived.AsanaConnection{}, + &archived.AsanaProject{}, + &archived.AsanaScopeConfig{}, + &archived.AsanaTask{}, + &archived.AsanaSection{}, + &archived.AsanaUser{}, + &archived.AsanaWorkspace{}, + &archived.AsanaTeam{}, + &archived.AsanaStory{}, + &archived.AsanaTag{}, + &archived.AsanaTaskTag{}, + &archived.AsanaCustomField{}, + &archived.AsanaTaskCustomFieldValue{}, + &archived.AsanaProjectMembership{}, + &archived.AsanaTeamMembership{}, ) } diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go index e9ede755357..35c482d271c 100644 --- a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go @@ -21,7 +21,7 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/helpers/migrationhelper" - "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts/archived" ) type addMissingTables struct{} @@ -30,15 +30,15 @@ func (*addMissingTables) Up(basicRes context.BasicRes) errors.Error { // Add all the new tables that were added after the initial migration return migrationhelper.AutoMigrateTables( basicRes, - &models.AsanaWorkspace{}, - &models.AsanaTeam{}, - &models.AsanaStory{}, - &models.AsanaTag{}, - &models.AsanaTaskTag{}, - &models.AsanaCustomField{}, - &models.AsanaTaskCustomFieldValue{}, - &models.AsanaProjectMembership{}, - &models.AsanaTeamMembership{}, + &archived.AsanaWorkspace{}, + &archived.AsanaTeam{}, + &archived.AsanaStory{}, + &archived.AsanaTag{}, + &archived.AsanaTaskTag{}, + &archived.AsanaCustomField{}, + &archived.AsanaTaskCustomFieldValue{}, + &archived.AsanaProjectMembership{}, + &archived.AsanaTeamMembership{}, ) } diff --git a/backend/plugins/asana/models/migrationscripts/archived/models.go b/backend/plugins/asana/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..5d292db9e4b --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/archived/models.go @@ -0,0 +1,263 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type AsanaConnection struct { + archived.Model + Name string `gorm:"type:varchar(150);uniqueIndex" json:"name" validate:"required"` + Endpoint string `mapstructure:"endpoint" json:"endpoint" validate:"required"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec" encrypt:"yes"` +} + +func (AsanaConnection) TableName() string { + return "_tool_asana_connections" +} + +type AsanaProject struct { + archived.ScopeModel + Gid string `json:"gid" gorm:"type:varchar(255);primaryKey"` + Name string `json:"name" gorm:"type:varchar(255)"` + ResourceType string `json:"resourceType" gorm:"type:varchar(32)"` + Archived bool `json:"archived"` + WorkspaceGid string `json:"workspaceGid" gorm:"type:varchar(255)"` + PermalinkUrl string `json:"permalinkUrl" gorm:"type:varchar(512)"` +} + +func (AsanaProject) TableName() string { + return "_tool_asana_projects" +} + +type AsanaScopeConfig struct { + archived.ScopeConfig + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (AsanaScopeConfig) TableName() string { + return "_tool_asana_scope_configs" +} + +type AsanaTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(512)"` + Notes string `gorm:"type:text"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completedAt"` + DueOn *time.Time `gorm:"type:date" json:"dueOn"` + CreatedAt time.Time `json:"createdAt"` + ModifiedAt *time.Time `json:"modifiedAt"` + PermalinkUrl string `gorm:"type:varchar(512)"` + ProjectGid string `gorm:"type:varchar(255);index"` + SectionGid string `gorm:"type:varchar(255);index"` + SectionName string `gorm:"type:varchar(255)"` + AssigneeGid string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatorGid string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + ParentGid string `gorm:"type:varchar(255);index"` + NumSubtasks int `json:"numSubtasks"` + StdType string `gorm:"type:varchar(255)"` + StdStatus string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + StoryPoint *float64 `json:"storyPoint"` + Severity string `gorm:"type:varchar(255)"` + LeadTimeMinutes *uint `json:"leadTimeMinutes"` + archived.NoPKModel +} + +func (AsanaTask) TableName() string { + return "_tool_asana_tasks" +} + +type AsanaSection struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ProjectGid string `gorm:"type:varchar(255);index"` + archived.NoPKModel +} + +func (AsanaSection) TableName() string { + return "_tool_asana_sections" +} + +type AsanaUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` + archived.NoPKModel +} + +func (AsanaUser) TableName() string { + return "_tool_asana_users" +} + +type AsanaWorkspace struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + IsOrganization bool `json:"isOrganization"` + archived.NoPKModel +} + +func (AsanaWorkspace) TableName() string { + return "_tool_asana_workspaces" +} + +type AsanaTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + HtmlDescription string `gorm:"type:text"` + OrganizationGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + archived.NoPKModel +} + +func (AsanaTeam) TableName() string { + return "_tool_asana_teams" +} + +type AsanaStory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(64)"` + Text string `gorm:"type:text"` + HtmlText string `gorm:"type:text"` + IsPinned bool `json:"isPinned"` + IsEdited bool `json:"isEdited"` + StickerName string `gorm:"type:varchar(64)"` + CreatedAt time.Time `json:"createdAt"` + CreatedByGid string `gorm:"type:varchar(255)"` + CreatedByName string `gorm:"type:varchar(255)"` + TaskGid string `gorm:"type:varchar(255);index"` + TargetGid string `gorm:"type:varchar(255);index"` + archived.NoPKModel +} + +func (AsanaStory) TableName() string { + return "_tool_asana_stories" +} + +type AsanaTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Color string `gorm:"type:varchar(32)"` + Notes string `gorm:"type:text"` + WorkspaceGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + archived.NoPKModel +} + +func (AsanaTag) TableName() string { + return "_tool_asana_tags" +} + +type AsanaTaskTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + TagGid string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (AsanaTaskTag) TableName() string { + return "_tool_asana_task_tags" +} + +type AsanaCustomField struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Type string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + Precision int `json:"precision"` + IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` + HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` + archived.NoPKModel +} + +func (AsanaCustomField) TableName() string { + return "_tool_asana_custom_fields" +} + +type AsanaTaskCustomFieldValue struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldName string `gorm:"type:varchar(255)"` + DisplayValue string `gorm:"type:text"` + TextValue string `gorm:"type:text"` + NumberValue *float64 `json:"numberValue"` + EnumValueGid string `gorm:"type:varchar(255)"` + EnumValueName string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (AsanaTaskCustomFieldValue) TableName() string { + return "_tool_asana_task_custom_field_values" +} + +type AsanaProjectMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + Role string `gorm:"type:varchar(32)"` + archived.NoPKModel +} + +func (AsanaProjectMembership) TableName() string { + return "_tool_asana_project_memberships" +} + +type AsanaTeamMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + IsGuest bool `json:"isGuest"` + archived.NoPKModel +} + +func (AsanaTeamMembership) TableName() string { + return "_tool_asana_team_memberships" +} +