diff --git a/backend/plugins/monorepo/e2e/attribution_test.go b/backend/plugins/monorepo/e2e/attribution_test.go new file mode 100644 index 00000000000..b56d1ce4a07 --- /dev/null +++ b/backend/plugins/monorepo/e2e/attribution_test.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" + "github.com/stretchr/testify/assert" +) + +// TestMonorepoAttributionDataFlow exercises both subtasks against a monorepo containing +// serviceA and serviceB, each with its own deploy job. +// +// The fixtures deliberately include the cases that motivated this plugin: +// - pr2 (serviceB) merges at 09:00 while serviceA deploys at 10:00 and serviceB only at +// 12:00. DORA would link pr2 to the 10:00 deployment because it searches the whole +// repository; pr2 must instead link to 12:00. +// - pipeline3 runs both deploy jobs, so it must yield one row per sub-project. +// - a failed deployment and a staging deployment sit between pr1's merge and the +// deployment that actually shipped it, so neither may be linked. +func TestMonorepoAttributionDataFlow(t *testing.T) { + var plugin impl.Monorepo + dataflowTester := e2ehelper.NewDataFlowTester(t, "monorepo", plugin) + + subProjects := []tasks.SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } + matcher, err := tasks.NewSubProjectMatcher(subProjects) + assert.Nil(t, err) + + taskData := &tasks.MonorepoTaskData{ + Options: &tasks.MonorepoOptions{ + ProjectName: "monorepo", + SubProjects: subProjects, + }, + Matcher: matcher, + } + + // seed the domain layer + dataflowTester.FlushTabler(&crossdomain.ProjectMapping{}) + dataflowTester.FlushTabler(&devops.CICDTask{}) + dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{}) + dataflowTester.FlushTabler(&code.PullRequest{}) + dataflowTester.FlushTabler(&code.PullRequestLabel{}) + dataflowTester.FlushTabler(&crossdomain.ProjectPrMetric{}) + + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_mapping.csv", &crossdomain.ProjectMapping{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_tasks.csv", &devops.CICDTask{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_deployment_commits.csv", &devops.CicdDeploymentCommit{}) + dataflowTester.ImportNullableCsvIntoTabler("./monorepo_attribution/pull_requests.csv", &code.PullRequest{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/pull_request_labels.csv", &code.PullRequestLabel{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_pr_metrics.csv", &crossdomain.ProjectPrMetric{}) + + // deployments must be attributed first: the pull request subtask reads them back to + // work out which deployment shipped each merged pull request. + dataflowTester.FlushTabler(&models.SubProjectDeployment{}) + dataflowTester.Subtask(tasks.AttributeDeploymentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectDeployment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_deployments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.FlushTabler(&models.SubProjectPrMetric{}) + dataflowTester.Subtask(tasks.AttributePullRequestsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectPrMetric{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_pr_metrics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv new file mode 100644 index 00000000000..0bf5e350c27 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv @@ -0,0 +1,7 @@ +id,cicd_deployment_id,cicd_scope_id,name,result,status,environment,repo_url,commit_sha,created_date,finished_date +dc1,pipeline1,cicd1,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitA1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc2,pipeline2,cicd1,deploy-serviceB,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitB1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +dc3,pipeline3,cicd1,deploy-both,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitAB,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +dc5,pipeline5,cicd2,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/other,commitOther,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc6,pipeline6,cicd1,deploy-serviceA,FAILURE,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitFail,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +dc7,pipeline7,cicd1,deploy-serviceA,SUCCESS,DONE,STAGING,https://gitlab.example.com/acme/monorepo,commitStg,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv new file mode 100644 index 00000000000..9324d96ffb1 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv @@ -0,0 +1,9 @@ +id,name,pipeline_id,type,result,status,environment,cicd_scope_id,created_date,finished_date +task1,deploy-serviceA,pipeline1,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task1b,build,pipeline1,,SUCCESS,DONE,,cicd1,2026-08-01T09:40:00.000+00:00,2026-08-01T09:50:00.000+00:00 +task2,deploy-serviceB,pipeline2,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +task3a,deploy-serviceA,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task3b,deploy-serviceB,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task5,deploy-serviceA,pipeline5,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd2,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task6,deploy-serviceA,pipeline6,DEPLOYMENT,FAILURE,DONE,PRODUCTION,cicd1,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +task7,deploy-serviceA,pipeline7,DEPLOYMENT,SUCCESS,DONE,STAGING,cicd1,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv new file mode 100644 index 00000000000..c871e7cb114 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv @@ -0,0 +1,5 @@ +project_name,table,row_id +monorepo,cicd_scopes,cicd1 +monorepo,repos,repo1 +other,cicd_scopes,cicd2 +other,repos,repo2 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv new file mode 100644 index 00000000000..c60538d6507 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv @@ -0,0 +1,7 @@ +id,project_name,pr_coding_time,pr_pickup_time,pr_review_time +pr1,monorepo,100,20,30 +pr2,monorepo,200,40,60 +pr3,monorepo,300,60,90 +pr4,monorepo,400,80,120 +pr5,monorepo,500,100,150 +pr7,monorepo,700,140,210 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv new file mode 100644 index 00000000000..95b45e7f473 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv @@ -0,0 +1,9 @@ +pull_request_id,label_name +pr1,serviceA +pr2,serviceB +pr3,serviceB +pr3,serviceA +pr4,bug +pr5,serviceA +pr6,serviceA +pr7,serviceA diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv new file mode 100644 index 00000000000..c1809224f2b --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv @@ -0,0 +1,8 @@ +id,base_repo_id,created_date,merged_date,merge_commit_sha +pr1,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitA1 +pr2,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitB1 +pr3,repo1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,commitAB +pr4,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitBug +pr5,repo1,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00,commitLate +pr6,repo2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitOther +pr7,repo1,2026-08-01T08:00:00.000+00:00,NULL,commitOpen diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv new file mode 100644 index 00000000000..1504f88eff7 --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv @@ -0,0 +1,7 @@ +project_name,sub_project,cicd_deployment_id,commit_sha,job_name,result,environment,finished_date +monorepo,serviceA,pipeline1,commitA1,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-01T10:00:00.000+00:00 +monorepo,serviceB,pipeline2,commitB1,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-01T12:00:00.000+00:00 +monorepo,serviceA,pipeline3,commitAB,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceB,pipeline3,commitAB,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceA,pipeline6,commitFail,deploy-serviceA,FAILURE,PRODUCTION,2026-08-01T09:30:00.000+00:00 +monorepo,serviceA,pipeline7,commitStg,deploy-serviceA,SUCCESS,STAGING,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv new file mode 100644 index 00000000000..f40d9459bbf --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv @@ -0,0 +1,5 @@ +project_name,pull_request_id,sub_project,coding_time,pickup_time,review_time,deploy_time,cycle_time,deployment_id,pr_created_date,pr_merged_date,deployed_date +monorepo,pr1,serviceA,100,20,30,60,220,pipeline1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr2,serviceB,200,40,60,180,440,pipeline2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T12:00:00.000+00:00 +monorepo,pr3,serviceA,300,60,90,30,390,pipeline1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr5,serviceA,500,100,150,,560,,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00, diff --git a/backend/plugins/monorepo/impl/impl.go b/backend/plugins/monorepo/impl/impl.go new file mode 100644 index 00000000000..25ac166d51a --- /dev/null +++ b/backend/plugins/monorepo/impl/impl.go @@ -0,0 +1,188 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" +) + +// make sure interface is implemented +var _ interface { + plugin.PluginMeta + plugin.PluginTask + plugin.PluginModel + plugin.PluginMetric + plugin.PluginMigration + plugin.MetricPluginBlueprintV200 +} = (*Monorepo)(nil) + +type Monorepo struct{} + +func (p Monorepo) Description() string { + return "Split a monorepo into sub-projects and compute per-sub-project DORA metrics" +} + +func (p Monorepo) Name() string { + return "monorepo" +} + +func (p Monorepo) Dashboards() []plugin.GrafanaDashboard { + return nil +} + +func (p Monorepo) SvgIcon() string { + return ` + +` +} + +// RequiredDataEntities declares that deployments must be recognisable as CI/CD tasks of +// type Deployment, which is what sub-project attribution matches job names against. +func (p Monorepo) RequiredDataEntities() (data []map[string]interface{}, err errors.Error) { + return []map[string]interface{}{ + { + "model": "cicd_tasks", + "requiredFields": map[string]string{ + "column": "type", + "execptedValue": "Deployment", + }, + }, + }, nil +} + +func (p Monorepo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + } +} + +func (p Monorepo) IsProjectMetric() bool { + return true +} + +// RunAfter declares that this plugin should run after dora. NOTE: this is currently +// advisory metadata only (surfaced via the /plugins API) — core's blueprint plan builder +// (server/services/blueprint_makeplan_v200.go GeneratePlanJsonV200) merges all enabled +// metric plugins' plans with ParallelizePipelinePlans, which zips their stages together by +// index and does not consult RunAfter. Actual ordering against dora is enforced by stage +// padding in MakeMetricPluginPipelinePlanV200 below, not by this declaration. +func (p Monorepo) RunAfter() ([]string, errors.Error) { + return []string{"dora"}, nil +} + +func (p Monorepo) Settings() interface{} { + return nil +} + +func (p Monorepo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.AttributeDeploymentsMeta, + tasks.AttributePullRequestsMeta, + } +} + +func (p Monorepo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + matcher, err := tasks.NewSubProjectMatcher(op.SubProjects) + if err != nil { + return nil, err + } + return &tasks.MonorepoTaskData{ + Options: op, + Matcher: matcher, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Monorepo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/monorepo" +} + +func (p Monorepo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Monorepo) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { + op := &tasks.MonorepoOptions{} + if options != nil && string(options) != "\"\"" { + if err := json.Unmarshal(options, op); err != nil { + return nil, errors.Default.WrapRaw(err) + } + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New( + "the monorepo plugin requires a subProjects list in its metric plugin options") + } + // Validate eagerly so a bad regex is reported when the blueprint is saved rather + // than midway through a pipeline run. + if _, err := tasks.NewSubProjectMatcher(op.SubProjects); err != nil { + return nil, err + } + + subProjects := make([]map[string]interface{}, 0, len(op.SubProjects)) + for _, sp := range op.SubProjects { + subProjects = append(subProjects, map[string]interface{}{ + "name": sp.Name, + "prLabels": sp.PrLabels, + "deployJobPattern": sp.DeployJobPattern, + }) + } + + // attributeDeployments reads cicd_deployment_commits, and attributePullRequests reads + // project_pr_metrics — both are written by dora's own multi-stage plan (currently 3 + // stages: generate deployments, refdiff, calculate change lead time). Core's + // ParallelizePipelinePlans merges every enabled metric plugin's plan by stage index, so + // without padding, our single stage would run concurrently with dora's stage 0 instead + // of after its stage 2 — a real race that silently produces incomplete/nil-metric + // output (no error) when dora and monorepo are enabled together, since core's RunAfter + // contract above is not actually enforced by the scheduler. Padding with empty stages + // through dora's stage count (and then some, for headroom against future growth) is a + // workaround, not a fix: if dora's plan ever grows past this padding, the race returns. + // Revisit if DevLake ever adds real cross-plugin dependency scheduling. + const stagesToOutlastDora = 6 + plan := make(coreModels.PipelinePlan, stagesToOutlastDora+1) + for i := 0; i < stagesToOutlastDora; i++ { + plan[i] = coreModels.PipelineStage{} + } + plan[stagesToOutlastDora] = coreModels.PipelineStage{ + { + Plugin: "monorepo", + Options: map[string]interface{}{ + "projectName": projectName, + "subProjects": subProjects, + }, + Subtasks: []string{ + tasks.AttributeDeploymentsMeta.Name, + tasks.AttributePullRequestsMeta.Name, + }, + }, + } + return plan, nil +} diff --git a/backend/plugins/monorepo/impl/impl_test.go b/backend/plugins/monorepo/impl/impl_test.go new file mode 100644 index 00000000000..d3ee5310a0d --- /dev/null +++ b/backend/plugins/monorepo/impl/impl_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Core's blueprint plan builder (server/services/blueprint_makeplan_v200.go) merges every +// enabled metric plugin's plan with ParallelizePipelinePlans, which zips stages together by +// index and does NOT consult RunAfter(). If monorepo's real work sat in stage 0 like a naive +// single-stage plan would, it would run concurrently with dora's stage 0 instead of after +// dora's stage 2 (where project_pr_metrics/cicd_deployment_commits actually get written) — +// silently producing nil-metric output. This test locks in the stage-padding workaround so a +// future edit can't accidentally collapse the plan back to one stage. +func TestMakeMetricPluginPipelinePlanV200_StagePadding(t *testing.T) { + options, err := json.Marshal(map[string]interface{}{ + "subProjects": []map[string]interface{}{ + {"name": "serviceA", "prLabels": []string{"serviceA"}, "deployJobPattern": "^deploy-serviceA$"}, + }, + }) + require.NoError(t, err) + + var p Monorepo + plan, err2 := p.MakeMetricPluginPipelinePlanV200("test-project", options) + require.NoError(t, err2) + + require.Greater(t, len(plan), 3, "plan must have more stages than dora's plan (3), or monorepo's "+ + "work would run concurrently with dora instead of after it") + + for i := 0; i < len(plan)-1; i++ { + assert.Emptyf(t, plan[i], "stage %d should be empty padding, not real work", i) + } + + lastStage := plan[len(plan)-1] + require.Len(t, lastStage, 1) + assert.Equal(t, "monorepo", lastStage[0].Plugin) + assert.ElementsMatch(t, []string{"attributeDeployments", "attributePullRequests"}, lastStage[0].Subtasks) +} diff --git a/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go new file mode 100644 index 00000000000..ce485504013 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var _ plugin.MigrationScript = (*addInitTables)(nil) + +type addInitTables struct{} + +func (script *addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + ) +} + +func (*addInitTables) Version() uint64 { return 20260809100000 } + +func (*addInitTables) Name() string { + return "create init tables for the monorepo plugin" +} diff --git a/backend/plugins/monorepo/models/migrationscripts/register.go b/backend/plugins/monorepo/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/monorepo/models/subproject_deployment.go b/backend/plugins/monorepo/models/subproject_deployment.go new file mode 100644 index 00000000000..04f55279a06 --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_deployment.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectDeployment attributes a deployment to a single sub-project of a monorepo, +// based on the name of the CI job that performed the deployment. +// +// One deployment may produce several rows when a single pipeline runs the deploy jobs +// of several sub-projects. That is not double counting: each sub-project really was +// deployed by that pipeline. +type SubProjectDeployment struct { + common.NoPKModel + // The four primary key columns are deliberately kept narrow: MySQL caps a composite + // index at 3072 bytes, which is 768 characters under utf8mb4. + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + SubProject string `gorm:"primaryKey;type:varchar(100)"` + // CicdDeploymentId is the id of the deployment (a cicd_pipelines.id when the + // deployment was generated from a pipeline), taken from cicd_deployment_commits. + CicdDeploymentId string `gorm:"primaryKey;type:varchar(255)"` + // CommitSha is wide enough for a SHA-256 hash; the source column is varchar(255) but + // only ever holds a git object id. + CommitSha string `gorm:"primaryKey;type:varchar(64)"` + // JobName is the cicd_tasks.name that matched this sub-project's DeployJobPattern. + JobName string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + FinishedDate *time.Time +} + +func (SubProjectDeployment) TableName() string { + return "monorepo_subproject_deployments" +} diff --git a/backend/plugins/monorepo/models/subproject_pr_metric.go b/backend/plugins/monorepo/models/subproject_pr_metric.go new file mode 100644 index 00000000000..ab138230c8d --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_pr_metric.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectPrMetric holds the change-lead-time breakdown for a merged pull request, +// attributed to exactly one sub-project of a monorepo. +// +// CodingTime/PickupTime/ReviewTime are carried over from DORA's project_pr_metrics: +// they depend only on the pull request itself, so DORA already computes them correctly +// for a monorepo. Only DeployTime (and therefore CycleTime) is recomputed here, against +// the deployments of this sub-project rather than the whole repository's. +// +// All durations are in minutes, matching DORA's convention. +type SubProjectPrMetric struct { + common.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + PullRequestId string `gorm:"primaryKey;type:varchar(255)"` + SubProject string `gorm:"index;type:varchar(255)"` + + CodingTime *int64 + PickupTime *int64 + ReviewTime *int64 + DeployTime *int64 + CycleTime *int64 + + // DeploymentId is the sub-project deployment this PR was linked to, if any. + DeploymentId string `gorm:"type:varchar(255)"` + + PrCreatedDate *time.Time + PrMergedDate *time.Time + DeployedDate *time.Time +} + +func (SubProjectPrMetric) TableName() string { + return "monorepo_subproject_pr_metrics" +} diff --git a/backend/plugins/monorepo/monorepo.go b/backend/plugins/monorepo/monorepo.go new file mode 100644 index 00000000000..0b23cd9b74a --- /dev/null +++ b/backend/plugins/monorepo/monorepo.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/spf13/cobra" +) + +// PluginEntry exports for Framework to search and load +var PluginEntry impl.Monorepo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "monorepo"} + + projectName := cmd.Flags().StringP("projectName", "p", "", "project name") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("projectName") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "projectName": *projectName, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/monorepo/tasks/deployment_attributor.go b/backend/plugins/monorepo/tasks/deployment_attributor.go new file mode 100644 index 00000000000..e9ac492b612 --- /dev/null +++ b/backend/plugins/monorepo/tasks/deployment_attributor.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributeDeploymentsMeta = plugin.SubTaskMeta{ + Name: "attributeDeployments", + EntryPoint: AttributeDeployments, + EnabledByDefault: true, + Description: "Attribute each deployment to a monorepo sub-project by the name of the CI job that deployed it", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, +} + +// deploymentJobRow is one (deployment, deploy job) pair as returned by the query below. +// +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type deploymentJobRow struct { + common.RawDataOrigin + CicdDeploymentId string + CommitSha string + Result string + Environment string + FinishedDate *time.Time + JobName string +} + +func AttributeDeployments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*MonorepoTaskData) + + // Rebuild from scratch: attribution depends on configuration that may have changed + // since the last run, so stale rows cannot be reconciled incrementally. + if err := db.Exec( + "DELETE FROM monorepo_subproject_deployments WHERE project_name = ?", + data.Options.ProjectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_deployments") + } + + // Only deployments generated from pipelines can be attributed: cicd_deployment_id is + // the pipeline id, which is what cicd_tasks rows hang off. Deployments imported + // straight from a provider's deployment API carry no job and are skipped. + clauses := []dal.Clause{ + dal.Select(`dc.cicd_deployment_id, dc.commit_sha, dc.result, dc.environment, + dc.finished_date, t.name AS job_name`), + dal.From("cicd_deployment_commits dc"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)"), + dal.Join("JOIN cicd_tasks t ON (t.pipeline_id = dc.cicd_deployment_id)"), + dal.Where("pm.project_name = ? AND t.type = ?", data.Options.ProjectName, devops.DEPLOYMENT), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: data.Options.ProjectName, + }, + Table: "cicd_deployment_commits", + }, + InputRowType: reflect.TypeOf(deploymentJobRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + row := inputRow.(*deploymentJobRow) + matched := data.Matcher.MatchDeployJob(row.JobName) + results := make([]interface{}, 0, len(matched)) + for _, subProject := range matched { + results = append(results, &models.SubProjectDeployment{ + ProjectName: data.Options.ProjectName, + SubProject: subProject, + CicdDeploymentId: row.CicdDeploymentId, + CommitSha: row.CommitSha, + JobName: row.JobName, + Result: row.Result, + Environment: row.Environment, + FinishedDate: row.FinishedDate, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor.go b/backend/plugins/monorepo/tasks/pr_attributor.go new file mode 100644 index 00000000000..405c80f7219 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor.go @@ -0,0 +1,275 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "math" + "reflect" + "sort" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributePullRequestsMeta = plugin.SubTaskMeta{ + Name: "attributePullRequests", + EntryPoint: AttributePullRequests, + EnabledByDefault: true, + Description: "Attribute merged pull requests to monorepo sub-projects by label and compute their change lead time", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, +} + +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type pullRequestRow struct { + common.RawDataOrigin + Id string + CreatedDate time.Time + MergedDate *time.Time +} + +type prLabelRow struct { + PullRequestId string + LabelName string +} + +// deployedAt is the minimum information needed to link a merged PR to a deployment. +type deployedAt struct { + Id string + FinishedDate time.Time +} + +func AttributePullRequests(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + data := taskCtx.GetData().(*MonorepoTaskData) + projectName := data.Options.ProjectName + + if err := db.Exec( + "DELETE FROM monorepo_subproject_pr_metrics WHERE project_name = ?", + projectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_pr_metrics") + } + + labelsByPr, err := loadPrLabels(db, projectName) + if err != nil { + return err + } + // DORA already computes coding/pickup/review time correctly for a monorepo: they + // depend only on the pull request itself. Only the deploy leg needs recomputing. + doraMetrics, err := loadDoraPrMetrics(db, projectName) + if err != nil { + return err + } + deploymentsBySubProject, err := loadSubProjectDeployments(db, projectName) + if err != nil { + return err + } + logger.Info("monorepo: %d labelled PRs, %d DORA metric rows, %d sub-projects with deployments", + len(labelsByPr), len(doraMetrics), len(deploymentsBySubProject)) + + clauses := []dal.Clause{ + dal.Select("pr.id, pr.created_date, pr.merged_date"), + dal.From("pull_requests pr"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ? AND pr.merged_date IS NOT NULL", projectName), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + unattributed := 0 + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: projectName, + }, + Table: "pull_requests", + }, + InputRowType: reflect.TypeOf(pullRequestRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + pr := inputRow.(*pullRequestRow) + subProject := data.Matcher.MatchPrLabels(labelsByPr[pr.Id]) + if subProject == "" { + // No sub-project claims this PR; it is simply out of scope here. + unattributed++ + return nil, nil + } + + metric := &models.SubProjectPrMetric{ + ProjectName: projectName, + PullRequestId: pr.Id, + SubProject: subProject, + PrCreatedDate: &pr.CreatedDate, + PrMergedDate: pr.MergedDate, + } + if dm := doraMetrics[pr.Id]; dm != nil { + metric.CodingTime = dm.PrCodingTime + metric.PickupTime = dm.PrPickupTime + metric.ReviewTime = dm.PrReviewTime + } + + if deployment := firstDeploymentAfter(deploymentsBySubProject[subProject], pr.MergedDate); deployment != nil { + metric.DeploymentId = deployment.Id + metric.DeployedDate = &deployment.FinishedDate + metric.DeployTime = computeTimeSpan(pr.MergedDate, &deployment.FinishedDate) + } + + // Mirrors DORA's definition: coding + (merged - created) + deploy. + var cycleTime int64 + if metric.CodingTime != nil { + cycleTime += *metric.CodingTime + } + if prDuring := computeTimeSpan(&pr.CreatedDate, pr.MergedDate); prDuring != nil { + cycleTime += *prDuring + } + if metric.DeployTime != nil { + cycleTime += *metric.DeployTime + } + metric.CycleTime = &cycleTime + + return []interface{}{metric}, nil + }, + }) + if err != nil { + return err + } + + if err := converter.Execute(); err != nil { + return err + } + if unattributed > 0 { + logger.Info("monorepo: %d merged PRs matched no sub-project label and were skipped", unattributed) + } + return nil +} + +// firstDeploymentAfter returns the earliest deployment that finished after mergedDate. +// +// This is an approximation: it assumes a merged change is shipped by the next successful +// production deployment of its sub-project. Hotfixes, cherry-picks, rollbacks and re-runs +// can break that assumption. Exact attribution would need each deployment's commit range +// from the refdiff plugin's commits_diffs table; this function is the seam where that +// swap would happen. +func firstDeploymentAfter(deployments []deployedAt, mergedDate *time.Time) *deployedAt { + if mergedDate == nil || len(deployments) == 0 { + return nil + } + // deployments is sorted by FinishedDate ascending. + i := sort.Search(len(deployments), func(i int) bool { + return deployments[i].FinishedDate.After(*mergedDate) + }) + if i >= len(deployments) { + return nil + } + return &deployments[i] +} + +func loadPrLabels(db dal.Dal, projectName string) (map[string][]string, errors.Error) { + var rows []prLabelRow + err := db.All(&rows, + dal.Select("prl.pull_request_id, prl.label_name"), + dal.From("pull_request_labels prl"), + dal.Join("JOIN pull_requests pr ON (pr.id = prl.pull_request_id)"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading pull request labels") + } + byPr := make(map[string][]string) + for _, r := range rows { + byPr[r.PullRequestId] = append(byPr[r.PullRequestId], r.LabelName) + } + return byPr, nil +} + +func loadDoraPrMetrics(db dal.Dal, projectName string) (map[string]*crossdomain.ProjectPrMetric, errors.Error) { + var rows []*crossdomain.ProjectPrMetric + err := db.All(&rows, + dal.From(&crossdomain.ProjectPrMetric{}), + dal.Where("project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading project_pr_metrics") + } + byPr := make(map[string]*crossdomain.ProjectPrMetric, len(rows)) + for _, r := range rows { + byPr[r.Id] = r + } + return byPr, nil +} + +// loadSubProjectDeployments returns the successful production deployments of each +// sub-project, sorted by finish time so they can be searched by merge date. +func loadSubProjectDeployments(db dal.Dal, projectName string) (map[string][]deployedAt, errors.Error) { + var rows []models.SubProjectDeployment + err := db.All(&rows, + dal.From(&models.SubProjectDeployment{}), + dal.Where( + "project_name = ? AND result = ? AND environment = ? AND finished_date IS NOT NULL", + projectName, devops.RESULT_SUCCESS, devops.PRODUCTION, + ), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading monorepo_subproject_deployments") + } + bySubProject := make(map[string][]deployedAt) + for _, r := range rows { + bySubProject[r.SubProject] = append(bySubProject[r.SubProject], deployedAt{ + Id: r.CicdDeploymentId, + FinishedDate: *r.FinishedDate, + }) + } + for name := range bySubProject { + list := bySubProject[name] + sort.Slice(list, func(i, j int) bool { + return list[i].FinishedDate.Before(list[j].FinishedDate) + }) + bySubProject[name] = list + } + return bySubProject, nil +} + +// computeTimeSpan returns the whole minutes between start and end, or nil when either is +// missing or the span is negative. Mirrors the identical unexported helper in the DORA +// plugin (plugins/dora/tasks/change_lead_time_calculator.go) so the two produce the same +// numbers; it cannot be imported because it is not exported there. +func computeTimeSpan(start, end *time.Time) *int64 { + if start == nil || end == nil { + return nil + } + span := end.Sub(*start) + minutes := int64(math.Ceil(span.Minutes())) + if minutes < 0 { + return nil + } + return &minutes +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor_test.go b/backend/plugins/monorepo/tasks/pr_attributor_test.go new file mode 100644 index 00000000000..c716d005a56 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor_test.go @@ -0,0 +1,125 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func at(hour int) time.Time { + return time.Date(2026, 8, 9, hour, 0, 0, 0, time.UTC) +} + +func TestFirstDeploymentAfter(t *testing.T) { + // Sorted ascending, as loadSubProjectDeployments guarantees. + deployments := []deployedAt{ + {Id: "deploy-08", FinishedDate: at(8)}, + {Id: "deploy-12", FinishedDate: at(12)}, + {Id: "deploy-18", FinishedDate: at(18)}, + } + + cases := []struct { + name string + deployments []deployedAt + mergedDate *time.Time + expectedId string + }{ + { + name: "picks the earliest deployment after the merge", + deployments: deployments, + mergedDate: ptrTime(at(10)), + expectedId: "deploy-12", + }, + { + name: "merge before every deployment picks the first", + deployments: deployments, + mergedDate: ptrTime(at(1)), + expectedId: "deploy-08", + }, + { + name: "merge after every deployment has none to link", + deployments: deployments, + mergedDate: ptrTime(at(20)), + expectedId: "", + }, + { + name: "a deployment finishing exactly at merge time does not count", + deployments: deployments, + mergedDate: ptrTime(at(12)), + expectedId: "deploy-18", + }, + { + name: "no deployments at all", + deployments: nil, + mergedDate: ptrTime(at(10)), + expectedId: "", + }, + { + name: "unmerged pull request", + deployments: deployments, + mergedDate: nil, + expectedId: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := firstDeploymentAfter(tc.deployments, tc.mergedDate) + if tc.expectedId == "" { + assert.Nil(t, got) + return + } + assert.NotNil(t, got) + assert.Equal(t, tc.expectedId, got.Id) + }) + } +} + +func TestComputeTimeSpan(t *testing.T) { + start := at(10) + end := at(12) + + t.Run("whole minutes between two times", func(t *testing.T) { + got := computeTimeSpan(&start, &end) + assert.NotNil(t, got) + assert.Equal(t, int64(120), *got) + }) + + t.Run("partial minutes round up", func(t *testing.T) { + later := start.Add(90 * time.Second) + got := computeTimeSpan(&start, &later) + assert.NotNil(t, got) + assert.Equal(t, int64(2), *got) + }) + + t.Run("negative spans are discarded", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(&end, &start)) + }) + + t.Run("missing endpoints yield nil", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(nil, &end)) + assert.Nil(t, computeTimeSpan(&start, nil)) + assert.Nil(t, computeTimeSpan(nil, nil)) + }) +} + +func ptrTime(t time.Time) *time.Time { + return &t +} diff --git a/backend/plugins/monorepo/tasks/task_data.go b/backend/plugins/monorepo/tasks/task_data.go new file mode 100644 index 00000000000..26b57b390f6 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data.go @@ -0,0 +1,154 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "regexp" + + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type MonorepoApiParams struct { + ProjectName string +} + +// SubProjectConfig declares one logical project living inside a monorepo. +type SubProjectConfig struct { + // Name identifies the sub-project in the output tables and dashboards. + Name string `json:"name" mapstructure:"name"` + // PrLabels are the pull request labels that mark a PR as belonging to this + // sub-project. Matching is exact and case-sensitive. + PrLabels []string `json:"prLabels" mapstructure:"prLabels"` + // DeployJobPattern is a regular expression matched against cicd_tasks.name to + // recognise this sub-project's deployment jobs, e.g. "^deploy-serviceA$". + DeployJobPattern string `json:"deployJobPattern" mapstructure:"deployJobPattern"` +} + +type MonorepoOptions struct { + ProjectName string `json:"projectName" mapstructure:"projectName"` + // SubProjects is ordered: when a pull request carries the labels of more than one + // sub-project, the earliest entry in this list wins. + SubProjects []SubProjectConfig `json:"subProjects" mapstructure:"subProjects"` +} + +type MonorepoTaskData struct { + Options *MonorepoOptions + Matcher *SubProjectMatcher +} + +// SubProjectMatcher resolves deployments and pull requests to sub-projects. It holds +// the compiled form of the configuration so the regexes are built once per task rather +// than once per row. +type SubProjectMatcher struct { + names []string + prLabels []map[string]struct{} + deployJobRes []*regexp.Regexp +} + +// NewSubProjectMatcher compiles the sub-project configuration, validating it along the way. +func NewSubProjectMatcher(subProjects []SubProjectConfig) (*SubProjectMatcher, errors.Error) { + m := &SubProjectMatcher{ + names: make([]string, 0, len(subProjects)), + prLabels: make([]map[string]struct{}, 0, len(subProjects)), + deployJobRes: make([]*regexp.Regexp, 0, len(subProjects)), + } + seen := make(map[string]struct{}, len(subProjects)) + for i, sp := range subProjects { + if sp.Name == "" { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: name is required", i)) + } + if _, dup := seen[sp.Name]; dup { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: duplicate name %q", i, sp.Name)) + } + seen[sp.Name] = struct{}{} + + var jobRe *regexp.Regexp + if sp.DeployJobPattern != "" { + compiled, err := regexp.Compile(sp.DeployJobPattern) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf( + "subProjects[%d] (%s): invalid deployJobPattern", i, sp.Name)) + } + jobRe = compiled + } + + labels := make(map[string]struct{}, len(sp.PrLabels)) + for _, l := range sp.PrLabels { + if l != "" { + labels[l] = struct{}{} + } + } + + m.names = append(m.names, sp.Name) + m.prLabels = append(m.prLabels, labels) + m.deployJobRes = append(m.deployJobRes, jobRe) + } + return m, nil +} + +// MatchDeployJob returns every sub-project whose DeployJobPattern matches jobName. +// +// More than one match is possible and is reported faithfully: a single pipeline running +// both deploy-serviceA and deploy-serviceB genuinely deploys two sub-projects. If a +// single job name matches two patterns, that indicates overlapping configuration. +func (m *SubProjectMatcher) MatchDeployJob(jobName string) []string { + var matched []string + for i, re := range m.deployJobRes { + if re != nil && re.MatchString(jobName) { + matched = append(matched, m.names[i]) + } + } + return matched +} + +// MatchPrLabels returns the single sub-project a pull request belongs to, or "" when no +// sub-project claims it. When several sub-projects match, the earliest one in the +// configured order wins — labels carry no size signal that could rank them otherwise. +func (m *SubProjectMatcher) MatchPrLabels(labels []string) string { + if len(labels) == 0 { + return "" + } + present := make(map[string]struct{}, len(labels)) + for _, l := range labels { + present[l] = struct{}{} + } + for i, wanted := range m.prLabels { + for l := range wanted { + if _, ok := present[l]; ok { + return m.names[i] + } + } + } + return "" +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*MonorepoOptions, errors.Error) { + var op MonorepoOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "error decoding monorepo task options") + } + if op.ProjectName == "" { + return nil, errors.BadInput.New("projectName is required for the monorepo plugin") + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New("at least one entry in subProjects is required for the monorepo plugin") + } + return &op, nil +} diff --git a/backend/plugins/monorepo/tasks/task_data_test.go b/backend/plugins/monorepo/tasks/task_data_test.go new file mode 100644 index 00000000000..2d6affcca05 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data_test.go @@ -0,0 +1,220 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// twoServices is the canonical monorepo configuration used across these tests: +// serviceA is declared first, so it wins any tie. +func twoServices() []SubProjectConfig { + return []SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB", "svc-b"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } +} + +func TestMatchDeployJob(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + jobName string + expected []string + }{ + {"matches serviceA", "deploy-serviceA", []string{"serviceA"}}, + {"matches serviceB", "deploy-serviceB", []string{"serviceB"}}, + {"build job is not a deployment", "build-serviceA", nil}, + {"unrelated job matches nothing", "run-tests", nil}, + {"anchored pattern rejects a superstring", "deploy-serviceAB", nil}, + {"empty job name matches nothing", "", nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchDeployJob(tc.jobName)) + }) + } +} + +// A pipeline that runs both services' deploy jobs produces a row for each. The two jobs +// arrive as separate rows, so each resolves to exactly one sub-project. +func TestMatchDeployJob_PipelineDeployingBothServices(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA"}, matcher.MatchDeployJob("deploy-serviceA")) + assert.Equal(t, []string{"serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +// Overlapping patterns are reported faithfully rather than silently resolved, so a +// misconfiguration is visible in the data instead of hidden. +func TestMatchDeployJob_OverlappingPatterns(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "deploy"}, + {Name: "serviceB", DeployJobPattern: "^deploy-serviceB$"}, + }) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA", "serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +func TestMatchDeployJob_NoPatternNeverMatches(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "labelsOnly", PrLabels: []string{"labelsOnly"}}, + }) + assert.Nil(t, err) + + assert.Nil(t, matcher.MatchDeployJob("deploy-labelsOnly")) +} + +func TestMatchPrLabels(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + labels []string + expected string + }{ + {"single matching label", []string{"serviceA"}, "serviceA"}, + {"alias label resolves to its sub-project", []string{"svc-b"}, "serviceB"}, + {"matching label among unrelated ones", []string{"bug", "serviceB", "urgent"}, "serviceB"}, + {"no matching label", []string{"bug", "urgent"}, ""}, + {"no labels at all", nil, ""}, + {"empty label slice", []string{}, ""}, + {"matching is case sensitive", []string{"servicea"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchPrLabels(tc.labels)) + }) + } +} + +// A PR labelled for several sub-projects is assigned to exactly one: the earliest in +// configuration order. Labels carry no size signal, so declaration order is the tie-break. +func TestMatchPrLabels_TieBreakIsConfigOrder(t *testing.T) { + both := []string{"serviceB", "serviceA"} + + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + assert.Equal(t, "serviceA", matcher.MatchPrLabels(both)) + + // Reversing the configuration reverses the winner, proving order drives the result + // rather than the order of labels on the PR. + reversed := []SubProjectConfig{twoServices()[1], twoServices()[0]} + reversedMatcher, err := NewSubProjectMatcher(reversed) + assert.Nil(t, err) + assert.Equal(t, "serviceB", reversedMatcher.MatchPrLabels(both)) +} + +func TestNewSubProjectMatcher_Validation(t *testing.T) { + cases := []struct { + name string + subProjects []SubProjectConfig + expectErr bool + }{ + { + name: "valid configuration", + subProjects: twoServices(), + }, + { + name: "empty configuration is allowed here, rejected by option decoding", + subProjects: nil, + }, + { + name: "missing name", + subProjects: []SubProjectConfig{{PrLabels: []string{"x"}}}, + expectErr: true, + }, + { + name: "duplicate names", + subProjects: []SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "^a$"}, + {Name: "serviceA", DeployJobPattern: "^b$"}, + }, + expectErr: true, + }, + { + name: "invalid deploy job regex", + subProjects: []SubProjectConfig{{Name: "serviceA", DeployJobPattern: "^deploy-(unclosed"}}, + expectErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + matcher, err := NewSubProjectMatcher(tc.subProjects) + if tc.expectErr { + assert.NotNil(t, err) + assert.Nil(t, matcher) + return + } + assert.Nil(t, err) + assert.NotNil(t, matcher) + }) + } +} + +func TestDecodeAndValidateTaskOptions(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + op, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + "subProjects": []interface{}{ + map[string]interface{}{ + "name": "serviceA", + "prLabels": []interface{}{"serviceA"}, + "deployJobPattern": "^deploy-serviceA$", + }, + }, + }) + assert.Nil(t, err) + assert.Equal(t, "monorepo", op.ProjectName) + assert.Len(t, op.SubProjects, 1) + assert.Equal(t, "serviceA", op.SubProjects[0].Name) + assert.Equal(t, []string{"serviceA"}, op.SubProjects[0].PrLabels) + assert.Equal(t, "^deploy-serviceA$", op.SubProjects[0].DeployJobPattern) + }) + + t.Run("missing projectName is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "subProjects": []interface{}{ + map[string]interface{}{"name": "serviceA"}, + }, + }) + assert.NotNil(t, err) + }) + + t.Run("missing subProjects is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + }) + assert.NotNil(t, err) + }) +} diff --git a/config-ui/src/routes/project/detail/settings-panel.tsx b/config-ui/src/routes/project/detail/settings-panel.tsx index b7a78946466..4302f226430 100644 --- a/config-ui/src/routes/project/detail/settings-panel.tsx +++ b/config-ui/src/routes/project/detail/settings-panel.tsx @@ -18,6 +18,7 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; +import { CloseOutlined, PlusOutlined } from '@ant-design/icons'; import { Flex, Space, Card, Modal, Input, Checkbox, Button } from 'antd'; import API from '@/api'; @@ -30,6 +31,15 @@ import * as S from './styled'; const RegexPrIssueDefaultValue = '(?mi)(Closes)[\\s]*.*(((and )?#\\d+[ ]*)+)'; +interface ISubProject { + name: string; + // Comma-separated in the UI; split into an array on save. + prLabels: string; + deployJobPattern: string; +} + +const emptySubProject: ISubProject = { name: '', prLabels: '', deployJobPattern: '' }; + interface Props { project: IProject; onRefresh: () => void; @@ -47,6 +57,10 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const [issueTrace, setIssueTrace] = useState({ enable: false, }); + const [monorepo, setMonorepo] = useState<{ enable: boolean; subProjects: ISubProject[] }>({ + enable: false, + subProjects: [emptySubProject], + }); const [operating, setOperating] = useState(false); const [open, setOpen] = useState(false); @@ -56,6 +70,7 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const dora = project.metrics.find((ms) => ms.pluginName === 'dora'); const linker = project.metrics.find((ms) => ms.pluginName === 'linker'); const issueTrace = project.metrics.find((ms) => ms.pluginName === 'issue_trace'); + const monorepo = project.metrics.find((ms) => ms.pluginName === 'monorepo'); setName(project.name); setDora({ @@ -68,8 +83,35 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { setIssueTrace({ enable: issueTrace?.enable ?? false, }); + const subProjects = monorepo?.pluginOption?.subProjects; + setMonorepo({ + enable: monorepo?.enable ?? false, + subProjects: + Array.isArray(subProjects) && subProjects.length + ? subProjects.map((sp: any) => ({ + name: sp.name ?? '', + prLabels: Array.isArray(sp.prLabels) ? sp.prLabels.join(',') : '', + deployJobPattern: sp.deployJobPattern ?? '', + })) + : [emptySubProject], + }); }, [project]); + const handleAddSubProject = () => { + setMonorepo({ ...monorepo, subProjects: [...monorepo.subProjects, { ...emptySubProject }] }); + }; + + const handleDeleteSubProject = (index: number) => { + setMonorepo({ ...monorepo, subProjects: monorepo.subProjects.filter((_, i) => i !== index) }); + }; + + const handleUpdateSubProject = (index: number, field: keyof ISubProject, value: string) => { + setMonorepo({ + ...monorepo, + subProjects: monorepo.subProjects.map((sp, i) => (i === index ? { ...sp, [field]: value } : sp)), + }); + }; + const handleUpdate = async () => { const [success] = await operator( () => @@ -94,6 +136,22 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { pluginOption: {}, enable: issueTrace.enable, }, + { + pluginName: 'monorepo', + pluginOption: { + subProjects: monorepo.subProjects + .filter((sp) => sp.name.trim()) + .map((sp) => ({ + name: sp.name.trim(), + prLabels: sp.prLabels + .split(',') + .map((l) => l.trim()) + .filter((l) => l), + deployJobPattern: sp.deployJobPattern.trim(), + })), + }, + enable: monorepo.enable, + }, ], }), { @@ -191,6 +249,57 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { } description="Parse the issue status and assignee history from issue changelogs. Currently, only Jira issues are supported." /> + setMonorepo({ ...monorepo, enable: e.target.checked })} + > + Enable Monorepo Sub-Projects + + } + description={ + + Split a single repository into several logical sub-projects for DORA-style metrics. Deployments are + matched by CI job name, pull requests by label. When a pull request carries more than one + sub-project's label, the first matching sub-project in the list below wins. + + + } + > + {monorepo.enable && ( + + {monorepo.subProjects.map((sp, i) => ( +
+ handleUpdateSubProject(i, 'name', e.target.value)} + /> + handleUpdateSubProject(i, 'prLabels', e.target.value)} + /> + handleUpdateSubProject(i, 'deployJobPattern', e.target.value)} + /> + {monorepo.subProjects.length > 1 && ( +
+ ))} + + + )} +