diff --git a/backend/plugins/developer_telemetry/api/report_api.go b/backend/plugins/developer_telemetry/api/report_api.go index bb126139d70..dd5962991fd 100644 --- a/backend/plugins/developer_telemetry/api/report_api.go +++ b/backend/plugins/developer_telemetry/api/report_api.go @@ -42,10 +42,34 @@ type TelemetryReportRequest struct { } type TelemetryMetrics struct { - ActiveHours int `json:"active_hours" mapstructure:"active_hours"` - ToolsUsed []string `json:"tools_used" mapstructure:"tools_used"` - Commands map[string]int `json:"commands" mapstructure:"commands"` - Projects []string `json:"projects" mapstructure:"projects"` + ActiveHours int `json:"active_hours" mapstructure:"active_hours"` + ToolsUsed []string `json:"tools_used" mapstructure:"tools_used"` + Projects []string `json:"projects" mapstructure:"projects"` + GitActivity *GitActivity `json:"git_activity,omitempty" mapstructure:"git_activity"` + DevelopmentActivity *DevelopmentActivity `json:"development_activity,omitempty" mapstructure:"development_activity"` +} + +type GitActivity struct { + TotalCommits int `json:"total_commits" mapstructure:"total_commits"` + TotalLinesAdded int `json:"total_lines_added" mapstructure:"total_lines_added"` + TotalLinesDeleted int `json:"total_lines_deleted" mapstructure:"total_lines_deleted"` + TotalFilesChanged int `json:"total_files_changed" mapstructure:"total_files_changed"` + Repositories []Repository `json:"repositories" mapstructure:"repositories"` +} + +type Repository struct { + Name string `json:"name" mapstructure:"name"` + Path string `json:"path" mapstructure:"path"` + Commits int `json:"commits" mapstructure:"commits"` + LinesAdded int `json:"lines_added" mapstructure:"lines_added"` + LinesDeleted int `json:"lines_deleted" mapstructure:"lines_deleted"` + FilesChanged int `json:"files_changed" mapstructure:"files_changed"` + BranchesWorked []string `json:"branches_worked" mapstructure:"branches_worked"` +} + +type DevelopmentActivity struct { + TestRunsDetected int `json:"test_runs_detected" mapstructure:"test_runs_detected"` + BuildsDetected int `json:"build_commands_detected" mapstructure:"build_commands_detected"` } func PostReport(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { @@ -90,20 +114,22 @@ func PostReport(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro toolsUsedJSON, _ := json.Marshal(report.Metrics.ToolsUsed) projectsJSON, _ := json.Marshal(report.Metrics.Projects) - commandsJSON, _ := json.Marshal(report.Metrics.Commands) + gitActivityJSON, _ := json.Marshal(report.Metrics.GitActivity) + developmentActivityJSON, _ := json.Marshal(report.Metrics.DevelopmentActivity) metric := &models.DeveloperMetrics{ - ConnectionId: connectionId, - DeveloperId: report.Developer, - Date: reportDate, - Email: report.Email, - Name: report.Name, - Hostname: report.Hostname, - ActiveHours: report.Metrics.ActiveHours, - ToolsUsed: string(toolsUsedJSON), - ProjectContext: string(projectsJSON), - CommandCounts: string(commandsJSON), - OsInfo: "", + ConnectionId: connectionId, + DeveloperId: report.Developer, + Date: reportDate, + Email: report.Email, + Name: report.Name, + Hostname: report.Hostname, + ActiveHours: report.Metrics.ActiveHours, + ToolsUsed: string(toolsUsedJSON), + ProjectContext: string(projectsJSON), + GitActivity: string(gitActivityJSON), + DevelopmentActivity: string(developmentActivityJSON), + OsInfo: "", } // Check if record exists using Count @@ -131,7 +157,9 @@ func PostReport(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro {ColumnName: "active_hours", Value: metric.ActiveHours}, {ColumnName: "tools_used", Value: metric.ToolsUsed}, {ColumnName: "project_context", Value: metric.ProjectContext}, - {ColumnName: "command_counts", Value: metric.CommandCounts}, + + {ColumnName: "git_activity", Value: metric.GitActivity}, + {ColumnName: "development_activity", Value: metric.DevelopmentActivity}, {ColumnName: "os_info", Value: metric.OsInfo}, }, dal.Where("connection_id = ? AND developer_id = ? AND date = ?", connectionId, report.Developer, dateStr)) diff --git a/backend/plugins/developer_telemetry/models/developer_metrics.go b/backend/plugins/developer_telemetry/models/developer_metrics.go index 697f3649163..90728384f05 100644 --- a/backend/plugins/developer_telemetry/models/developer_metrics.go +++ b/backend/plugins/developer_telemetry/models/developer_metrics.go @@ -21,21 +21,48 @@ import ( "time" ) +// GitActivity represents git activity metrics for a developer +type GitActivity struct { + TotalCommits int `json:"total_commits"` + TotalLinesAdded int `json:"total_lines_added"` + TotalLinesDeleted int `json:"total_lines_deleted"` + TotalFilesChanged int `json:"total_files_changed"` + Repositories []Repository `json:"repositories"` +} + +// Repository represents per-repository git metrics +type Repository struct { + Name string `json:"name"` + Path string `json:"path"` + Commits int `json:"commits"` + LinesAdded int `json:"lines_added"` + LinesDeleted int `json:"lines_deleted"` + FilesChanged int `json:"files_changed"` + BranchesWorked []string `json:"branches_worked"` +} + +// DevelopmentActivity represents detected development activity patterns +type DevelopmentActivity struct { + TestRunsDetected int `json:"test_runs_detected"` + BuildsDetected int `json:"build_commands_detected"` +} + // DeveloperMetrics represents the tool layer table for developer telemetry data type DeveloperMetrics struct { - ConnectionId uint64 `gorm:"primaryKey;type:BIGINT;column:connection_id" json:"connection_id"` - DeveloperId string `gorm:"primaryKey;type:varchar(255);column:developer_id" json:"developer_id"` - Date time.Time `gorm:"primaryKey;type:date;column:date" json:"date"` - Email string `gorm:"type:varchar(255);index;column:email" json:"email"` - Name string `gorm:"type:varchar(255);column:name" json:"name"` - Hostname string `gorm:"type:varchar(255);column:hostname" json:"hostname"` - ActiveHours int `gorm:"column:active_hours" json:"active_hours"` - ToolsUsed string `gorm:"type:text;column:tools_used" json:"tools_used"` // JSON array stored as text - ProjectContext string `gorm:"type:text;column:project_context" json:"project_context"` // JSON array stored as text - CommandCounts string `gorm:"type:text;column:command_counts" json:"command_counts"` // JSON object stored as text - OsInfo string `gorm:"type:varchar(255);column:os_info" json:"os_info"` - CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` - UpdatedAt time.Time `gorm:"column:updated_at" json:"updatedAt"` + ConnectionId uint64 `gorm:"primaryKey;type:BIGINT;column:connection_id" json:"connection_id"` + DeveloperId string `gorm:"primaryKey;type:varchar(255);column:developer_id" json:"developer_id"` + Date time.Time `gorm:"primaryKey;type:date;column:date" json:"date"` + Email string `gorm:"type:varchar(255);index;column:email" json:"email"` + Name string `gorm:"type:varchar(255);column:name" json:"name"` + Hostname string `gorm:"type:varchar(255);column:hostname" json:"hostname"` + ActiveHours int `gorm:"column:active_hours" json:"active_hours"` + ToolsUsed string `gorm:"type:text;column:tools_used" json:"tools_used"` // JSON array stored as text + ProjectContext string `gorm:"type:text;column:project_context" json:"project_context"` // JSON array stored as text + GitActivity string `gorm:"type:text;column:git_activity" json:"git_activity"` // JSON object stored as text + DevelopmentActivity string `gorm:"type:text;column:development_activity" json:"development_activity"` // JSON object stored as text + OsInfo string `gorm:"type:varchar(255);column:os_info" json:"os_info"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` + UpdatedAt time.Time `gorm:"column:updated_at" json:"updatedAt"` } func (DeveloperMetrics) TableName() string { diff --git a/backend/plugins/developer_telemetry/models/migrationscripts/20240211_add_init_tables.go b/backend/plugins/developer_telemetry/models/migrationscripts/20240211_add_init_tables.go deleted file mode 100644 index d5ca2c1962c..00000000000 --- a/backend/plugins/developer_telemetry/models/migrationscripts/20240211_add_init_tables.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package migrationscripts - -import ( - "time" - - "github.com/apache/incubator-devlake/core/context" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" -) - -var _ plugin.MigrationScript = (*addInitTables)(nil) - -type addInitTables struct{} - -type developerTelemetryConnection20240211 struct { - ID uint64 `gorm:"primaryKey;type:BIGINT NOT NULL AUTO_INCREMENT" json:"id"` - Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` - Endpoint string `json:"endpoint"` - SecretToken string `mapstructure:"secretToken" json:"secretToken" gorm:"serializer:encdec"` - Proxy string `json:"proxy"` - RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` -} - -func (developerTelemetryConnection20240211) TableName() string { - return "_tool_developer_telemetry_connections" -} - -type developerMetrics20240211 struct { - ConnectionId uint64 `gorm:"primaryKey;type:BIGINT" json:"connection_id"` - DeveloperId string `gorm:"primaryKey;type:varchar(255)" json:"developer_id"` - Date string `gorm:"primaryKey;type:date" json:"date"` - Email string `gorm:"type:varchar(255);index" json:"email"` - Name string `gorm:"type:varchar(255)" json:"name"` - Hostname string `gorm:"type:varchar(255)" json:"hostname"` - ActiveHours int `json:"active_hours"` - ToolsUsed string `gorm:"type:text" json:"tools_used"` - ProjectContext string `gorm:"type:text" json:"project_context"` - CommandCounts string `gorm:"type:text" json:"command_counts"` - OsInfo string `gorm:"type:varchar(255)" json:"os_info"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` -} - -func (developerMetrics20240211) TableName() string { - return "_tool_developer_metrics" -} - -func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { - db := basicRes.GetDal() - err := db.AutoMigrate(&developerTelemetryConnection20240211{}) - if err != nil { - return err - } - err = db.AutoMigrate(&developerMetrics20240211{}) - if err != nil { - return err - } - return nil -} - -func (*addInitTables) Version() uint64 { - return 20240211000001 -} - -func (*addInitTables) Name() string { - return "developer_telemetry init schemas" -} diff --git a/backend/plugins/developer_telemetry/models/migrationscripts/20260219_init_schema.go b/backend/plugins/developer_telemetry/models/migrationscripts/20260219_init_schema.go new file mode 100644 index 00000000000..7a339a8470b --- /dev/null +++ b/backend/plugins/developer_telemetry/models/migrationscripts/20260219_init_schema.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*initSchema)(nil) + +type initSchema struct{} + +type developerTelemetryConnection20260219 struct { + ID uint64 `gorm:"primaryKey;type:BIGINT NOT NULL AUTO_INCREMENT" json:"id"` + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + Endpoint string `json:"endpoint"` + SecretToken string `mapstructure:"secretToken" json:"secretToken" gorm:"serializer:encdec"` + Proxy string `json:"proxy"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (developerTelemetryConnection20260219) TableName() string { + return "_tool_developer_telemetry_connections" +} + +type developerMetrics20260219 struct { + ConnectionId uint64 `gorm:"primaryKey;type:BIGINT" json:"connection_id"` + DeveloperId string `gorm:"primaryKey;type:varchar(255)" json:"developer_id"` + Date string `gorm:"primaryKey;type:date" json:"date"` + Email string `gorm:"type:varchar(255);index" json:"email"` + Name string `gorm:"type:varchar(255)" json:"name"` + Hostname string `gorm:"type:varchar(255)" json:"hostname"` + ActiveHours int `json:"active_hours"` + ToolsUsed string `gorm:"type:text" json:"tools_used"` + ProjectContext string `gorm:"type:text" json:"project_context"` + GitActivity string `gorm:"type:text" json:"git_activity"` + DevelopmentActivity string `gorm:"type:text" json:"development_activity"` + OsInfo string `gorm:"type:varchar(255)" json:"os_info"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (developerMetrics20260219) TableName() string { + return "_tool_developer_metrics" +} + +func (*initSchema) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + // Drop existing tables if they exist (dev mode - fresh start) + err := db.DropTables( + &developerTelemetryConnection20260219{}, + &developerMetrics20260219{}, + ) + if err != nil { + return err + } + + // Create fresh tables with new schema + err = db.AutoMigrate(&developerTelemetryConnection20260219{}) + if err != nil { + return err + } + err = db.AutoMigrate(&developerMetrics20260219{}) + if err != nil { + return err + } + + return nil +} + +func (*initSchema) Version() uint64 { + return 20260219000001 +} + +func (*initSchema) Name() string { + return "Initialize developer_telemetry schema with git activity support" +} diff --git a/backend/plugins/developer_telemetry/models/migrationscripts/register.go b/backend/plugins/developer_telemetry/models/migrationscripts/register.go index ec054748c27..d68e003a920 100644 --- a/backend/plugins/developer_telemetry/models/migrationscripts/register.go +++ b/backend/plugins/developer_telemetry/models/migrationscripts/register.go @@ -24,6 +24,6 @@ import ( // All return all the migration scripts func All() []plugin.MigrationScript { return []plugin.MigrationScript{ - new(addInitTables), + new(initSchema), } }