diff --git a/app/data_client.go b/app/data_client.go index 6f55d61103f..77120183f0b 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -1349,7 +1349,8 @@ func (d *DataClient) GetDataPipeline(ctx context.Context, id string) (*DataPipel // CreateDataPipeline creates a new data pipeline using the given query and schedule. func (d *DataClient) CreateDataPipeline( - ctx context.Context, organizationID, name string, query []map[string]interface{}, schedule string, opts *CreateDataPipelineOptions, + ctx context.Context, organizationID, name string, query []map[string]interface{}, schedule string, + enableBackfill bool, opts *CreateDataPipelineOptions, ) (string, error) { mqlBinary, err := queryBSONToBinary(query) if err != nil { @@ -1369,6 +1370,7 @@ func (d *DataClient) CreateDataPipeline( MqlBinary: mqlBinary, Schedule: schedule, DataSourceType: &dataSourceType, + EnableBackfill: &enableBackfill, }) if err != nil { return "", err diff --git a/app/data_client_test.go b/app/data_client_test.go index f22c92f7837..b558b581b54 100644 --- a/app/data_client_test.go +++ b/app/data_client_test.go @@ -1144,6 +1144,7 @@ func TestDataPipelineClient(t *testing.T) { test.That(t, in.Name, test.ShouldEqual, name) test.That(t, in.MqlBinary, test.ShouldResemble, mqlBinary) test.That(t, in.Schedule, test.ShouldEqual, "0 9 * * *") + test.That(t, *in.EnableBackfill, test.ShouldBeTrue) test.That(t, *in.DataSourceType, test.ShouldEqual, pb.TabularDataSourceType_TABULAR_DATA_SOURCE_TYPE_STANDARD) return &datapipelinesPb.CreateDataPipelineResponse{ Id: "new-data-pipeline-id", @@ -1152,7 +1153,7 @@ func TestDataPipelineClient(t *testing.T) { options := &CreateDataPipelineOptions{ TabularDataSourceType: TabularDataSourceTypeStandard, } - resp, err := client.CreateDataPipeline(context.Background(), organizationID, name, mqlQueries, "0 9 * * *", options) + resp, err := client.CreateDataPipeline(context.Background(), organizationID, name, mqlQueries, "0 9 * * *", true, options) test.That(t, err, test.ShouldBeNil) test.That(t, resp, test.ShouldEqual, "new-data-pipeline-id") }) diff --git a/cli/app.go b/cli/app.go index 31cbdbaa4b0..bd61baa51f2 100644 --- a/cli/app.go +++ b/cli/app.go @@ -126,6 +126,7 @@ const ( datapipelineFlagMQL = "mql" datapipelineFlagMQLFile = "mql-path" datapipelineFlagDataSourceType = "data-source-type" + datapipelineFlagEnableBackfill = "enable-backfill" packageFlagFramework = "model-framework" @@ -1626,7 +1627,7 @@ var app = &cli.App{ Name: "create", Usage: "create a new data pipeline", UsageText: createUsageText("datapipelines create", - []string{generalFlagOrgID, generalFlagName, datapipelineFlagSchedule}, false, false, + []string{generalFlagOrgID, generalFlagName, datapipelineFlagSchedule, datapipelineFlagEnableBackfill}, false, false, fmt.Sprintf("[--%s=<%s> | --%s=<%s>]", datapipelineFlagMQL, datapipelineFlagMQL, datapipelineFlagMQLFile, datapipelineFlagMQLFile), @@ -1655,6 +1656,11 @@ var app = &cli.App{ Name: datapipelineFlagMQLFile, Usage: "path to JSON file containing MQL query for the new data pipeline", }, + &cli.BoolFlag{ + Name: datapipelineFlagEnableBackfill, + Usage: "enable data pipeline to run over organization's historical data", + Required: true, + }, &cli.StringFlag{ Name: datapipelineFlagDataSourceType, Usage: formatAcceptedValues(