From 81ef9ee35bd4352d8519aa988e899de80a7120cc Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 14 Jan 2026 20:55:33 -0800 Subject: [PATCH 1/7] autopush and dev done --- build/ci/cloudbuild.deploy_mixer.yaml | 2 +- build/ci/cloudbuild.deploy_website_mixer.yaml | 2 +- deploy/helm_charts/dc_website/values.yaml | 25 ++-------- deploy/helm_charts/envs/autopush.yaml | 27 ----------- deploy/helm_charts/envs/dev.yaml | 27 +---------- deploy/helm_charts/envs/prod.yaml | 11 ----- deploy/helm_charts/envs/staging.yaml | 11 ----- .../envs/website_mixer_autopush.yaml | 33 +++++++++++++ .../helm_charts/envs/website_mixer_dev.yaml | 36 +++++++++++++++ .../helm_charts/envs/website_mixer_prod.yaml | 10 ++++ .../envs/website_mixer_staging.yaml | 10 ++++ scripts/deploy_mixer_cloud_deploy.sh | 2 +- scripts/deploy_website_cloud_deploy.sh | 2 +- skaffold.yaml | 46 ++++--------------- 14 files changed, 105 insertions(+), 139 deletions(-) create mode 100644 deploy/helm_charts/envs/website_mixer_autopush.yaml create mode 100644 deploy/helm_charts/envs/website_mixer_dev.yaml create mode 100644 deploy/helm_charts/envs/website_mixer_prod.yaml create mode 100644 deploy/helm_charts/envs/website_mixer_staging.yaml diff --git a/build/ci/cloudbuild.deploy_mixer.yaml b/build/ci/cloudbuild.deploy_mixer.yaml index b65273cc86..ab9b108df8 100644 --- a/build/ci/cloudbuild.deploy_mixer.yaml +++ b/build/ci/cloudbuild.deploy_mixer.yaml @@ -63,7 +63,7 @@ steps: --region=us-central1 \ --skaffold-file=skaffold.yaml \ --images="gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:${mixer_githash}" \ - --deploy-parameters="mixer.githash=${mixer_githash},MIXER_GITHASH=${mixer_githash}" \ + --deploy-parameters="MIXER_GITHASH=${mixer_githash}" \ --project=datcom-ci echo "Successfully created release ${release_name}." \ No newline at end of file diff --git a/build/ci/cloudbuild.deploy_website_mixer.yaml b/build/ci/cloudbuild.deploy_website_mixer.yaml index 7987a7b250..06a0742825 100644 --- a/build/ci/cloudbuild.deploy_website_mixer.yaml +++ b/build/ci/cloudbuild.deploy_website_mixer.yaml @@ -89,7 +89,7 @@ steps: --region=us-central1 \ --skaffold-file=skaffold.yaml \ --images="gcr.io/datcom-ci/datacommons-website=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-website:$${website_githash},gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$${mixer_githash},gcr.io/datcom-ci/datacommons-nodejs=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nodejs:$${website_githash},gcr.io/datcom-ci/datacommons-nl=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nl:$${website_githash}" \ - --deploy-parameters="mixer.githash=$${mixer_githash},MIXER_GITHASH=$${mixer_githash},website.githash=$${website_githash}" \ + --deploy-parameters="MIXER_GITHASH=$${mixer_githash},WEBSITE_GITHASH=$${website_githash}" \ --project=datcom-ci echo "Successfully created release $${release_name}." diff --git a/deploy/helm_charts/dc_website/values.yaml b/deploy/helm_charts/dc_website/values.yaml index 01daec0623..943d736968 100644 --- a/deploy/helm_charts/dc_website/values.yaml +++ b/deploy/helm_charts/dc_website/values.yaml @@ -78,6 +78,9 @@ kgStoreConfig: baseBigtableInfo: customBigtableInfo: "" +# NOTE: These serviceGroups configs are used for ROUTING ONLY (generating ingress-config). +# To configure workload settings (replicas, resources, etc.) for the mixer deployments, +# please modify the respective `website_mixer_{env}.yaml` overrides file. serviceGroups: svg: urlPaths: @@ -86,37 +89,15 @@ serviceGroups: - "/v1/variable/*" - "/v1/bulk/info/variable-group" - "/v1/bulk/info/variable" - replicas: 1 - resources: - memoryRequest: "8G" - memoryLimit: "8G" - cacheSVG: true - nodePool: observation: urlPaths: - "/v2/observation" - replicas: 1 - resources: - memoryRequest: "4G" - memoryLimit: "4G" - nodePool: node: urlPaths: - "/v2/node" - replicas: 1 - resources: - memoryRequest: "8G" - memoryLimit: "8G" - cacheSVG: true - nodePool: default: urlPaths: - "/*" - replicas: 1 - resources: - memoryRequest: "2G" - memoryLimit: "2G" - nodePool: ############################################################################### # Config for NL diff --git a/deploy/helm_charts/envs/autopush.yaml b/deploy/helm_charts/envs/autopush.yaml index 5259ed84f2..2e0a87feb4 100644 --- a/deploy/helm_charts/envs/autopush.yaml +++ b/deploy/helm_charts/envs/autopush.yaml @@ -39,22 +39,6 @@ website: } } -mixer: - serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog" - hostProject: "datcom-website-autopush" - cacheSVFormula: true - redis: - enabled: true - configFile: | - instances: - - region: "us-central1" - host: "10.13.112.19" - port: "6379" - enableOtlp: true - -serviceAccount: - name: website-ksa - ingress: enabled: true @@ -92,17 +76,6 @@ nl: prediction_endpoint_id: "284894457873039360" enable_reranking: true -serviceGroups: - recon: null - svg: - replicas: 5 - node: - replicas: 12 - observation: - replicas: 12 - default: - replicas: 12 - nodejs: enabled: true apiRoot: "http://website-service:8080" diff --git a/deploy/helm_charts/envs/dev.yaml b/deploy/helm_charts/envs/dev.yaml index 3a66b784da..bb9d237cdf 100644 --- a/deploy/helm_charts/envs/dev.yaml +++ b/deploy/helm_charts/envs/dev.yaml @@ -29,35 +29,10 @@ website: redis: enabled: false -mixer: - serviceName: "website-esp.endpoints.datcom-website-dev.cloud.goog" - hostProject: "datcom-website-dev" - redis: - enabled: true - configFile: | - instances: - - region: "us-central1" - host: "10.60.165.235" - port: "6379" - enableOtlp: true - -serviceAccount: - name: website-ksa - +# Enable ingress for website ingress: enabled: true -serviceGroups: - recon: null - svg: - replicas: 1 - observation: - replicas: 3 - node: - replicas: 3 - default: - replicas: 3 - nl: enabled: true image: diff --git a/deploy/helm_charts/envs/prod.yaml b/deploy/helm_charts/envs/prod.yaml index 6483eb0ab1..ec511cba5a 100644 --- a/deploy/helm_charts/envs/prod.yaml +++ b/deploy/helm_charts/envs/prod.yaml @@ -64,16 +64,5 @@ nl: prediction_endpoint_id: "430892009855647744" enable_reranking: false -serviceGroups: - recon: null - svg: - replicas: 10 - observation: - replicas: 100 - node: - replicas: 100 - default: - replicas: 50 - svg: blocklistFile: ["dc/g/Uncategorized", "oecd/g/OECD"] diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml index 8f25bc65ef..0778084dd5 100644 --- a/deploy/helm_charts/envs/staging.yaml +++ b/deploy/helm_charts/envs/staging.yaml @@ -69,17 +69,6 @@ nl: prediction_endpoint_id: "8435477397553283072" enable_reranking: false -serviceGroups: - recon: null - svg: - replicas: 2 - node: - replicas: 4 - observation: - replicas: 4 - default: - replicas: 4 - nodejs: enabled: true apiRoot: "http://website-service:8080" diff --git a/deploy/helm_charts/envs/website_mixer_autopush.yaml b/deploy/helm_charts/envs/website_mixer_autopush.yaml new file mode 100644 index 0000000000..31319d83db --- /dev/null +++ b/deploy/helm_charts/envs/website_mixer_autopush.yaml @@ -0,0 +1,33 @@ + + +mixer: + serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog" + hostProject: "datcom-website-autopush" + cacheSVFormula: true + useRedis: true + redis: + enabled: true + configFile: | + instances: + - region: "us-central1" + host: "10.13.112.19" + port: "6379" + enableOtlp: true + +serviceAccount: + name: website-ksa + +# Disable ingress for mixer in datcom-website-autopush +ingress: + enabled: false + +serviceGroups: + recon: null + svg: + replicas: 5 + node: + replicas: 12 + observation: + replicas: 12 + default: + replicas: 12 \ No newline at end of file diff --git a/deploy/helm_charts/envs/website_mixer_dev.yaml b/deploy/helm_charts/envs/website_mixer_dev.yaml new file mode 100644 index 0000000000..2df22ee6ba --- /dev/null +++ b/deploy/helm_charts/envs/website_mixer_dev.yaml @@ -0,0 +1,36 @@ +# Overrides for datcom-website-dev's mixer deployments +# Default values are defined by +# priority: /mixer/deploy/helm_charts/envs/mixer_dev.yaml +# fallback: /mixer/deploy/helm_charts/values.yaml + +project: "datcom-website-dev" + +mixer: + hostProject: "datcom-website-dev" + serviceName: "website-esp.endpoints.datcom-website-dev.cloud.goog" + redis: + enabled: true + configFile: | + instances: + - region: "us-central1" + host: "10.60.165.235" + port: "6379" + enableOtlp: true + +serviceAccount: + name: website-ksa + +# Disable ingress for mixer in datcom-website-dev +ingress: + enabled: false + +serviceGroups: + recon: null + svg: + replicas: 1 + observation: + replicas: 3 + node: + replicas: 3 + default: + replicas: 3 \ No newline at end of file diff --git a/deploy/helm_charts/envs/website_mixer_prod.yaml b/deploy/helm_charts/envs/website_mixer_prod.yaml new file mode 100644 index 0000000000..29ba14b61d --- /dev/null +++ b/deploy/helm_charts/envs/website_mixer_prod.yaml @@ -0,0 +1,10 @@ +serviceGroups: + recon: null + svg: + replicas: 10 + observation: + replicas: 100 + node: + replicas: 100 + default: + replicas: 50 \ No newline at end of file diff --git a/deploy/helm_charts/envs/website_mixer_staging.yaml b/deploy/helm_charts/envs/website_mixer_staging.yaml new file mode 100644 index 0000000000..94767eda05 --- /dev/null +++ b/deploy/helm_charts/envs/website_mixer_staging.yaml @@ -0,0 +1,10 @@ +serviceGroups: + recon: null + svg: + replicas: 2 + node: + replicas: 4 + observation: + replicas: 4 + default: + replicas: 4 \ No newline at end of file diff --git a/scripts/deploy_mixer_cloud_deploy.sh b/scripts/deploy_mixer_cloud_deploy.sh index 05113daf3c..d3c68b4ca0 100644 --- a/scripts/deploy_mixer_cloud_deploy.sh +++ b/scripts/deploy_mixer_cloud_deploy.sh @@ -30,5 +30,5 @@ gcloud deploy releases create "dev-manual-$TIMESTAMP" \ --region=us-central1 \ --skaffold-file=skaffold.yaml \ --images="gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$MIXER_GITHASH" \ ---deploy-parameters="mixer.githash=$MIXER_GITHASH,MIXER_GITHASH=$MIXER_GITHASH" \ +--deploy-parameters="MIXER_GITHASH=$MIXER_GITHASH" \ --project=datcom-ci diff --git a/scripts/deploy_website_cloud_deploy.sh b/scripts/deploy_website_cloud_deploy.sh index 6a9fb6d891..18ce553a03 100644 --- a/scripts/deploy_website_cloud_deploy.sh +++ b/scripts/deploy_website_cloud_deploy.sh @@ -31,5 +31,5 @@ gcloud deploy releases create "dev-manual-$TIMESTAMP" \ --region=us-central1 \ --skaffold-file=skaffold.yaml \ --images="gcr.io/datcom-ci/datacommons-website=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-website:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$MIXER_GITHASH,gcr.io/datcom-ci/datacommons-nodejs=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nodejs:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-nl=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nl:$WEBSITE_GITHASH" \ ---deploy-parameters="mixer.githash=$MIXER_GITHASH,MIXER_GITHASH=$MIXER_GITHASH,website.githash=$WEBSITE_GITHASH" \ +--deploy-parameters="MIXER_GITHASH=$MIXER_GITHASH,WEBSITE_GITHASH=$WEBSITE_GITHASH" \ --project=datcom-ci \ No newline at end of file diff --git a/skaffold.yaml b/skaffold.yaml index 49a86f26dd..dd30b91e96 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -104,29 +104,20 @@ profiles: valuesFiles: - deploy/helm_charts/envs/dev.yaml setValues: - serviceAccount.name: website-ksa + website.githash: ${WEBSITE_GITHASH} - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: - mixer/deploy/helm_charts/envs/mixer_dev.yaml + - deploy/helm_charts/envs/website_mixer_dev.yaml + setValues: + mixer.githash: ${MIXER_GITHASH} namespace: website setFiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml setValues: - mixer.redis.enabled: true - mixer.redis.configFile: | - instances: - - region: "us-central1" - host: "10.60.165.235" - port: "6379" - mixer.enableOtlp: true - serviceAccount.name: website-ksa - ingress.enabled: "true" - mixer.hostProject: "datcom-website-dev" - project: "datcom-website-dev" - mixer.serviceName: "website-esp.endpoints.datcom-website-dev.cloud.goog" mixer.githash: ${MIXER_GITHASH} - name: mixer-dev customActions: @@ -152,10 +143,6 @@ profiles: kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml setValues: - serviceAccount.name: mixer-ksa - ingress.enabled: "true" - mixer.hostProject: "datcom-mixer-dev-316822" - mixer.serviceName: "dev.api.datacommons.org" mixer.githash: ${MIXER_GITHASH} - name: website-autopush @@ -186,31 +173,20 @@ profiles: valuesFiles: - deploy/helm_charts/envs/autopush.yaml setValues: - serviceAccount.name: website-ksa website.githash: ${WEBSITE_GITHASH} - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: - mixer/deploy/helm_charts/envs/mixer_autopush.yaml + - deploy/helm_charts/envs/website_mixer_autopush.yaml namespace: website setFiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - mixer.redis.enabled: "true" - mixer.redis.configFile: | - instances: - - region: "us-central1" - host: "10.13.112.19" - port: "6379" - mixer.enableOtlp: true - serviceAccount.name: website-ksa - ingress.enabled: "true" - mixer.useRedis: "true" mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-website-autopush" - mixer.serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog" + - name: mixer-autopush customActions: - name: deploy-cloud-esp-mixer-autopush @@ -243,14 +219,6 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - serviceAccount.name: mixer-ksa - ingress.enabled: "true" - mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-mixer-autopush" - project: datcom-mixer-autopush - cluster_prefix: mixer - mixer.serviceName: "autopush.api.datacommons.org" - name: website-staging customActions: @@ -295,6 +263,7 @@ profiles: chartPath: mixer/deploy/helm_charts/mixer valuesFiles: - mixer/deploy/helm_charts/envs/mixer_staging.yaml + - deploy/helm_charts/envs/website_mixer_staging.yaml namespace: website setFiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml @@ -428,6 +397,7 @@ profiles: chartPath: mixer/deploy/helm_charts/mixer valuesFiles: - mixer/deploy/helm_charts/envs/mixer_prod.yaml + - deploy/helm_charts/envs/website_mixer_prod.yaml namespace: website setFiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml From c4314f8c85fbafd24c6fc61eb7ee6f006536d4f7 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 14 Jan 2026 23:31:09 -0800 Subject: [PATCH 2/7] staging and prod --- deploy/helm_charts/envs/prod.yaml | 13 +++--- deploy/helm_charts/envs/staging.yaml | 16 ++------ .../helm_charts/envs/website_mixer_prod.yaml | 34 +++++++++++----- .../envs/website_mixer_staging.yaml | 24 ++++++++++- skaffold.yaml | 40 +------------------ 5 files changed, 58 insertions(+), 69 deletions(-) diff --git a/deploy/helm_charts/envs/prod.yaml b/deploy/helm_charts/envs/prod.yaml index ec511cba5a..a5e260b960 100644 --- a/deploy/helm_charts/envs/prod.yaml +++ b/deploy/helm_charts/envs/prod.yaml @@ -40,15 +40,14 @@ website: } } -mixer: - serviceName: "website-esp.endpoints.datcom-website-prod.cloud.goog" - hostProject: "datcom-website-prod" -serviceAccount: - name: website-ksa -nl: - enabled: true +# Ingress is disabled because we use Multi Cluster Ingress (MCI) and Multi Cluster Service (MCS) +# defined in gke/ which manage networking independently of this Helm chart. +# Enabling ingress here would mean using templates/single_cluster.yaml to set up a standard +# Ingress and Service, which is not what we want for this multi-cluster environment. +ingress: + enabled: false env: default_indexes: - base_uae_mem diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml index 0778084dd5..49838a9223 100644 --- a/deploy/helm_charts/envs/staging.yaml +++ b/deploy/helm_charts/envs/staging.yaml @@ -37,21 +37,13 @@ website: } } -mixer: - serviceName: "website-esp.endpoints.datcom-website-staging.cloud.goog" - hostProject: "datcom-website-staging" - redis: - enabled: true - enableOtlp: true +# Ingress is disabled because we use Multi Cluster Ingress (MCI) and Multi Cluster Service (MCS) +# defined in gke/ which manage networking independently of this Helm chart. +# Enabling ingress here would mean using templates/single_cluster.yaml to set up a standard +# Ingress and Service, which is not what we want for this environment. ingress: enabled: false - -serviceAccount: - name: website-ksa - -nl: - enabled: true image: tag: "" env: diff --git a/deploy/helm_charts/envs/website_mixer_prod.yaml b/deploy/helm_charts/envs/website_mixer_prod.yaml index 29ba14b61d..ac9aa0fa6e 100644 --- a/deploy/helm_charts/envs/website_mixer_prod.yaml +++ b/deploy/helm_charts/envs/website_mixer_prod.yaml @@ -1,10 +1,24 @@ -serviceGroups: - recon: null - svg: - replicas: 10 - observation: - replicas: 100 - node: - replicas: 100 - default: - replicas: 50 \ No newline at end of file + +project: "datcom-website-prod" +mixer: + hostProject: "datcom-website-prod" + serviceName: "website-esp.endpoints.datcom-website-prod.cloud.goog" + useRedis: false + redis: + enabled: true + configFile: | + instances: + - region: "us-central1" + host: "10.167.58.139" + port: "6379" + - region: "us-west1" + host: "10.158.101.59" + port: "6379" + enableOtlp: true + +serviceAccount: + name: website-ksa + +# Disable ingress for mixer in datcom-website-prod (Sidecar only) +ingress: + enabled: false \ No newline at end of file diff --git a/deploy/helm_charts/envs/website_mixer_staging.yaml b/deploy/helm_charts/envs/website_mixer_staging.yaml index 94767eda05..8a72cb58ee 100644 --- a/deploy/helm_charts/envs/website_mixer_staging.yaml +++ b/deploy/helm_charts/envs/website_mixer_staging.yaml @@ -1,10 +1,32 @@ + +project: "datcom-website-staging" +mixer: + hostProject: "datcom-website-staging" + serviceName: "website-esp.endpoints.datcom-website-staging.cloud.goog" + useRedis: true + redis: + enabled: true + configFile: | + instances: + - region: "us-central1" + host: "10.38.211.19" + port: "6379" + enableOtlp: true + +serviceAccount: + name: website-ksa +ingress: + enabled: false + serviceGroups: recon: null svg: replicas: 2 + nodePool: pool-1 node: replicas: 4 observation: replicas: 4 default: - replicas: 4 \ No newline at end of file + replicas: 4 + diff --git a/skaffold.yaml b/skaffold.yaml index dd30b91e96..d2c2e9adb2 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -257,7 +257,6 @@ profiles: valuesFiles: - deploy/helm_charts/envs/staging.yaml setValues: - serviceAccount.name: website-ksa website.githash: ${WEBSITE_GITHASH} - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer @@ -270,21 +269,7 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - serviceGroups.svg.nodePool: pool-1 - mixer.redis.enabled: "true" - mixer.redis.configFile: | - instances: - - region: "us-central1" - host: "10.38.211.19" - port: "6379" - mixer.enableOtlp: true - serviceAccount.name: website-ksa - ingress.enabled: "true" - mixer.useRedis: "true" mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-website-staging" - project: "datcom-website-staging" - mixer.serviceName: "website-esp.endpoints.datcom-website-staging.cloud.goog" - name: mixer-staging customActions: - name: deploy-cloud-esp-mixer-staging @@ -318,12 +303,7 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - serviceGroups.svg.nodePool: pool-2 - serviceAccount.name: mixer-ksa - ingress.enabled: "true" mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-mixer-staging" - mixer.serviceName: "staging.api.datacommons.org" - name: mixer-prod customActions: - name: deploy-cloud-esp-mixer-prod @@ -348,12 +328,8 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - serviceAccount.name: mixer-ksa - ingress.enabled: "true" mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-mixer" - mixer.serviceName: "api.datacommons.org" - + - name: website-prod customActions: - name: deploy-cloud-esp-prod @@ -391,7 +367,6 @@ profiles: valuesFiles: - deploy/helm_charts/envs/prod.yaml setValues: - serviceAccount.name: website-ksa website.githash: ${WEBSITE_GITHASH} - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer @@ -404,17 +379,4 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - mixer.redis.enabled: true - mixer.redis.configFile: | - instances: - - region: "us-central1" - host: "10.212.232.163" - port: "6379" - mixer.enableOtlp: true - serviceAccount.name: website-ksa - ingress.enabled: "true" - mixer.useRedis: "false" mixer.githash: ${MIXER_GITHASH} - mixer.hostProject: "datcom-website-prod" - project: "datcom-website-prod" - mixer.serviceName: "website-esp.endpoints.datcom-website-prod.cloud.goog" From 49fff173d8d84b26e9384dd5b5fa6acf4e9705ef Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 14 Jan 2026 23:46:40 -0800 Subject: [PATCH 3/7] fix nl enabled for staging and prod, add back servicegroup configs for website_mixer_prod overrides, other nits --- deploy/helm_charts/envs/prod.yaml | 3 +++ deploy/helm_charts/envs/staging.yaml | 3 +++ .../envs/website_mixer_autopush.yaml | 2 +- deploy/helm_charts/envs/website_mixer_dev.yaml | 2 +- .../helm_charts/envs/website_mixer_prod.yaml | 18 +++++++++++++----- scripts/deploy_website_cloud_deploy.sh | 2 +- skaffold.yaml | 7 +++---- 7 files changed, 25 insertions(+), 12 deletions(-) diff --git a/deploy/helm_charts/envs/prod.yaml b/deploy/helm_charts/envs/prod.yaml index a5e260b960..c160d4da37 100644 --- a/deploy/helm_charts/envs/prod.yaml +++ b/deploy/helm_charts/envs/prod.yaml @@ -48,6 +48,9 @@ website: # Ingress and Service, which is not what we want for this multi-cluster environment. ingress: enabled: false + +nl: + enabled: true env: default_indexes: - base_uae_mem diff --git a/deploy/helm_charts/envs/staging.yaml b/deploy/helm_charts/envs/staging.yaml index 49838a9223..1c5e934002 100644 --- a/deploy/helm_charts/envs/staging.yaml +++ b/deploy/helm_charts/envs/staging.yaml @@ -44,6 +44,9 @@ website: # Ingress and Service, which is not what we want for this environment. ingress: enabled: false + +nl: + enabled: true image: tag: "" env: diff --git a/deploy/helm_charts/envs/website_mixer_autopush.yaml b/deploy/helm_charts/envs/website_mixer_autopush.yaml index 31319d83db..6cc7ec8b5e 100644 --- a/deploy/helm_charts/envs/website_mixer_autopush.yaml +++ b/deploy/helm_charts/envs/website_mixer_autopush.yaml @@ -30,4 +30,4 @@ serviceGroups: observation: replicas: 12 default: - replicas: 12 \ No newline at end of file + replicas: 12 diff --git a/deploy/helm_charts/envs/website_mixer_dev.yaml b/deploy/helm_charts/envs/website_mixer_dev.yaml index 2df22ee6ba..4ae7b15e29 100644 --- a/deploy/helm_charts/envs/website_mixer_dev.yaml +++ b/deploy/helm_charts/envs/website_mixer_dev.yaml @@ -33,4 +33,4 @@ serviceGroups: node: replicas: 3 default: - replicas: 3 \ No newline at end of file + replicas: 3 diff --git a/deploy/helm_charts/envs/website_mixer_prod.yaml b/deploy/helm_charts/envs/website_mixer_prod.yaml index ac9aa0fa6e..c049c6081c 100644 --- a/deploy/helm_charts/envs/website_mixer_prod.yaml +++ b/deploy/helm_charts/envs/website_mixer_prod.yaml @@ -9,10 +9,7 @@ mixer: configFile: | instances: - region: "us-central1" - host: "10.167.58.139" - port: "6379" - - region: "us-west1" - host: "10.158.101.59" + host: "10.212.232.163" port: "6379" enableOtlp: true @@ -21,4 +18,15 @@ serviceAccount: # Disable ingress for mixer in datcom-website-prod (Sidecar only) ingress: - enabled: false \ No newline at end of file + enabled: false + +serviceGroups: + recon: null + svg: + replicas: 10 + observation: + replicas: 100 + node: + replicas: 100 + default: + replicas: 50 diff --git a/scripts/deploy_website_cloud_deploy.sh b/scripts/deploy_website_cloud_deploy.sh index 18ce553a03..2fa16b5d28 100644 --- a/scripts/deploy_website_cloud_deploy.sh +++ b/scripts/deploy_website_cloud_deploy.sh @@ -32,4 +32,4 @@ gcloud deploy releases create "dev-manual-$TIMESTAMP" \ --skaffold-file=skaffold.yaml \ --images="gcr.io/datcom-ci/datacommons-website=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-website:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-mixer=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-mixer:$MIXER_GITHASH,gcr.io/datcom-ci/datacommons-nodejs=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nodejs:$WEBSITE_GITHASH,gcr.io/datcom-ci/datacommons-nl=us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-nl:$WEBSITE_GITHASH" \ --deploy-parameters="MIXER_GITHASH=$MIXER_GITHASH,WEBSITE_GITHASH=$WEBSITE_GITHASH" \ ---project=datcom-ci \ No newline at end of file +--project=datcom-ci diff --git a/skaffold.yaml b/skaffold.yaml index d2c2e9adb2..5be5d447eb 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -186,7 +186,6 @@ profiles: kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: mixer.githash: ${MIXER_GITHASH} - - name: mixer-autopush customActions: - name: deploy-cloud-esp-mixer-autopush @@ -219,7 +218,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - + setValues: + mixer.githash: ${MIXER_GITHASH} - name: website-staging customActions: - name: deploy-cloud-esp-staging @@ -328,8 +328,7 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - mixer.githash: ${MIXER_GITHASH} - + mixer.githash: ${MIXER_GITHASH} - name: website-prod customActions: - name: deploy-cloud-esp-prod From 3441ee2a757893aa0382ba66779f86f24acb19dc Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 14 Jan 2026 23:51:20 -0800 Subject: [PATCH 4/7] dedupe one setvalues --- skaffold.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/skaffold.yaml b/skaffold.yaml index 5be5d447eb..212ae410b4 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -110,8 +110,6 @@ profiles: valuesFiles: - mixer/deploy/helm_charts/envs/mixer_dev.yaml - deploy/helm_charts/envs/website_mixer_dev.yaml - setValues: - mixer.githash: ${MIXER_GITHASH} namespace: website setFiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml From 51e7671b171aa5708871f58e51ceed71a48a64e1 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 15 Jan 2026 00:03:04 -0800 Subject: [PATCH 5/7] fix whitespace --- skaffold.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skaffold.yaml b/skaffold.yaml index 212ae410b4..7ae3c7dbd4 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -217,7 +217,7 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - mixer.githash: ${MIXER_GITHASH} + mixer.githash: ${MIXER_GITHASH} - name: website-staging customActions: - name: deploy-cloud-esp-staging @@ -326,7 +326,7 @@ profiles: kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version setValues: - mixer.githash: ${MIXER_GITHASH} + mixer.githash: ${MIXER_GITHASH} - name: website-prod customActions: - name: deploy-cloud-esp-prod From 09f94bd53b7eaa0eece94a1c29c1f90e7d8eb669 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 15 Jan 2026 00:41:45 -0800 Subject: [PATCH 6/7] add comment headers --- deploy/helm_charts/envs/website_mixer_autopush.yaml | 5 ++++- deploy/helm_charts/envs/website_mixer_prod.yaml | 4 ++++ deploy/helm_charts/envs/website_mixer_staging.yaml | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/deploy/helm_charts/envs/website_mixer_autopush.yaml b/deploy/helm_charts/envs/website_mixer_autopush.yaml index 6cc7ec8b5e..a0cb2c9a6b 100644 --- a/deploy/helm_charts/envs/website_mixer_autopush.yaml +++ b/deploy/helm_charts/envs/website_mixer_autopush.yaml @@ -1,4 +1,7 @@ - +# Overrides for datcom-website-autopush's mixer deployments +# Default values are defined by +# priority: /mixer/deploy/helm_charts/envs/mixer_autopush.yaml +# fallback: /mixer/deploy/helm_charts/values.yaml mixer: serviceName: "website-esp.endpoints.datcom-website-autopush.cloud.goog" diff --git a/deploy/helm_charts/envs/website_mixer_prod.yaml b/deploy/helm_charts/envs/website_mixer_prod.yaml index c049c6081c..385fdf22bc 100644 --- a/deploy/helm_charts/envs/website_mixer_prod.yaml +++ b/deploy/helm_charts/envs/website_mixer_prod.yaml @@ -1,3 +1,7 @@ +# Overrides for datcom-website-prod's mixer deployments +# Default values are defined by +# priority: /mixer/deploy/helm_charts/envs/mixer_prod.yaml +# fallback: /mixer/deploy/helm_charts/values.yaml project: "datcom-website-prod" mixer: diff --git a/deploy/helm_charts/envs/website_mixer_staging.yaml b/deploy/helm_charts/envs/website_mixer_staging.yaml index 8a72cb58ee..81d54e8479 100644 --- a/deploy/helm_charts/envs/website_mixer_staging.yaml +++ b/deploy/helm_charts/envs/website_mixer_staging.yaml @@ -1,3 +1,7 @@ +# Overrides for datcom-website-staging's mixer deployments +# Default values are defined by +# priority: /mixer/deploy/helm_charts/envs/mixer_staging.yaml +# fallback: /mixer/deploy/helm_charts/values.yaml project: "datcom-website-staging" mixer: From a78181191b0f1db545a8843cefcdad5b47ef866b Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 15 Jan 2026 08:40:45 -0800 Subject: [PATCH 7/7] convert githashes in skaffold to setValueTemplates --- skaffold.yaml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/skaffold.yaml b/skaffold.yaml index 7ae3c7dbd4..c531476cf9 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -103,8 +103,8 @@ profiles: chartPath: deploy/helm_charts/dc_website valuesFiles: - deploy/helm_charts/envs/dev.yaml - setValues: - website.githash: ${WEBSITE_GITHASH} + setValueTemplates: + website.githash: "{{ .WEBSITE_GITHASH }}" - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: @@ -115,8 +115,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: mixer-dev customActions: - name: deploy-cloud-esp-mixer-dev @@ -140,8 +140,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: website-autopush customActions: @@ -170,8 +170,8 @@ profiles: chartPath: deploy/helm_charts/dc_website valuesFiles: - deploy/helm_charts/envs/autopush.yaml - setValues: - website.githash: ${WEBSITE_GITHASH} + setValueTemplates: + website.githash: "{{ .WEBSITE_GITHASH }}" - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: @@ -182,8 +182,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: mixer-autopush customActions: - name: deploy-cloud-esp-mixer-autopush @@ -216,8 +216,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: website-staging customActions: - name: deploy-cloud-esp-staging @@ -254,8 +254,8 @@ profiles: chartPath: deploy/helm_charts/dc_website valuesFiles: - deploy/helm_charts/envs/staging.yaml - setValues: - website.githash: ${WEBSITE_GITHASH} + setValueTemplates: + website.githash: "{{ .WEBSITE_GITHASH }}" - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: @@ -266,8 +266,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: mixer-staging customActions: - name: deploy-cloud-esp-mixer-staging @@ -300,8 +300,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: mixer-prod customActions: - name: deploy-cloud-esp-mixer-prod @@ -325,8 +325,8 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}" - name: website-prod customActions: - name: deploy-cloud-esp-prod @@ -363,8 +363,8 @@ profiles: chartPath: deploy/helm_charts/dc_website valuesFiles: - deploy/helm_charts/envs/prod.yaml - setValues: - website.githash: ${WEBSITE_GITHASH} + setValueTemplates: + website.githash: "{{ .WEBSITE_GITHASH }}" - name: dc-mixer chartPath: mixer/deploy/helm_charts/mixer valuesFiles: @@ -375,5 +375,5 @@ profiles: kgStoreConfig.baseBigtableInfo: mixer/deploy/storage/base_bigtable_info.yaml kgStoreConfig.spannerGraphInfo: mixer/deploy/storage/spanner_graph_info.yaml kgStoreConfig.bigqueryVersion: mixer/deploy/storage/bigquery.version - setValues: - mixer.githash: ${MIXER_GITHASH} + setValueTemplates: + mixer.githash: "{{ .MIXER_GITHASH }}"