Merge pull request #613 from fultonj/dz_storage_post_deploy_az

Update Glance configuration for dz-storage

Update Nova on EDPM nodes for each rack to use a unique Glance URL per AZ by switching the standard nova-extra-config to a custom nova-extra-config-az for each AZ 0,1,2.
Increase timeout for GlanceAPI and HAProxy since tempest tests had timeouts and add Glance enabled_import_methods.
Update automation dz-storage.yaml to post_stage_run ci-framework playbook dz_storage_post_deploy_az.yaml which creates Cinder volume types for Glance and run playbook dz_storage_pre_test_images.yaml which creates images in multiple stores so that Tempest has what in needs in order to test Glance with distributed zones.
Depends-On: openstack-k8s-operators/ci-framework#3250

Reviewed-by: Andrew Bays <[email protected]>

Author: softwarefactory-project-zuul[bot]

automation/vars/dz-storage.yaml

@@ -222,8 +222,16 @@ vas:
             src_file: values.yaml
         build_output: edpm-deployment.yaml
         post_stage_run:
-          - name: Wait until computes are ready
+          - name: "1 - Wait until computes are ready"
             type: playbook
             source: "../../playbooks/bgp-l3-computes-ready.yml"
             extra_vars:
               num_computes: 6
+          - name: "2 - Setup Cinder volume types and Nova aggregates for dz-storage"
+            type: playbook
+            source: "../../hooks/playbooks/dz_storage_post_deploy_az.yaml"
+            inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"
+          - name: "3 - Create and import cirros image to all Glance stores"
+            type: playbook
+            source: "../../hooks/playbooks/dz_storage_pre_test_images.yaml"
+            inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"

examples/dt/dz-storage/control-plane/kustomization.yaml

@@ -5,6 +5,16 @@ kind: Kustomization
 components:
   - ../../../../dt/bgp/
 
+patches:
+  # Override the base layer's glance apiOverrides with empty object
+  - target:
+      kind: OpenStackControlPlane
+      name: controlplane
+    patch: |-
+      - op: replace
+        path: /spec/glance/apiOverrides
+        value: {}
+
 resources:
   - networking/nncp/values.yaml
   - service-values.yaml
@@ -155,6 +165,19 @@ replacements:
         options:
           create: true
 
+  # Set glance apiTimeout for timeout configuration
+  - source:
+      kind: ConfigMap
+      name: service-values
+      fieldPath: data.glance.apiTimeout
+    targets:
+      - select:
+          kind: OpenStackControlPlane
+        fieldPaths:
+          - spec.glance.template.apiTimeout
+        options:
+          create: true
+
   # Set all four glanceAPIs (default, az{0,1,2})
   - source:
       kind: ConfigMap

examples/dt/dz-storage/control-plane/service-values.yaml

@@ -107,12 +107,15 @@ data:
       replicas: 0
     # Ensure keystoneEndpoint is set
     keystoneEndpoint: default
+    # Set API timeout to 600 seconds (this controls both HAProxy and Apache timeouts)
+    apiTimeout: 600
     # There are 4 glanceAPIs keys below (default, az{0,1,2})
     glanceAPIs:
       az0:
         customServiceConfig: |
           [DEFAULT]
           enabled_backends = az0:cinder
+          enabled_import_methods = [web-download,copy-image,glance-direct]
           debug = true
           [glance_store]
           default_backend = az0
@@ -146,6 +149,7 @@ data:
         customServiceConfig: |
           [DEFAULT]
           enabled_backends = az0:cinder,az1:cinder
+          enabled_import_methods = [web-download,copy-image,glance-direct]
           [glance_store]
           default_backend = az1
           [az1]
@@ -188,6 +192,7 @@ data:
         customServiceConfig: |
           [DEFAULT]
           enabled_backends = az0:cinder,az2:cinder
+          enabled_import_methods = [web-download,copy-image,glance-direct]
           [glance_store]
           default_backend = az2
           [az2]
@@ -230,6 +235,7 @@ data:
         customServiceConfig: |
           [DEFAULT]
           enabled_backends = az0:cinder,az1:cinder,az2:cinder
+          enabled_import_methods = [web-download,copy-image,glance-direct]
           [glance_store]
           default_backend = az0
           [az0]

examples/dt/dz-storage/data-plane.md

@@ -41,26 +41,50 @@ vi values.yaml
 
 ## Storage Related Compute Node Overrides
 
-Edit the following file to contain any desired override to the Nova configuration:
+Due to the distributed zones architecture, each availability zone requires its own Nova configuration to use the local Glance endpoint. Instead of using a single global `nova-extra-config`, we use **AZ-specific EDPM custom services**.
 
-- [edpm/computes/nova-extra-config.yaml](edpm/computes/nova-extra-config.yaml)
+### AZ-specific ConfigMaps
 
-The file is prepopulated with the following:
+Each AZ has its own Nova configuration file:
+
+- [edpm/computes/r0/nova-extra-config-az0.yaml](edpm/computes/r0/nova-extra-config-az0.yaml) - points to `glance-az0-internal.openstack.svc:9292`
+- [edpm/computes/r1/nova-extra-config-az1.yaml](edpm/computes/r1/nova-extra-config-az1.yaml) - points to `glance-az1-internal.openstack.svc:9292`
+- [edpm/computes/r2/nova-extra-config-az2.yaml](edpm/computes/r2/nova-extra-config-az2.yaml) - points to `glance-az2-internal.openstack.svc:9292`
+
+Each file contains:
 ```ini
 [DEFAULT]
 # Triple the default of the following
 reimage_timeout_per_gb = 60
+[glance]
+endpoint_override = https://glance-azX-internal.openstack.svc:9292
+valid_interfaces = internal
+[cinder]
+cross_az_attach = False
+catalog_info = volumev3:cinderv3:internalURL
 ```
-The configmap can be genereated with the following command.
-```
-kustomize build edpm/computes/
-```
 
-As per [the docs](https://docs.redhat.com/en/documentation/red_hat_openstack_services_on_openshift/18.0/html-single/customizing_the_red_hat_openstack_services_on_openshift_deployment/index#proc_configuring-a-node-set-for-a-feature-or-workload_custom_dataplane),
+### EDPM Custom Services
+
+Each AZ uses its own EDPM custom service that references the appropriate ConfigMap:
+
+- [edpm/computes/r0/nova-custom-az0.yaml](edpm/computes/r0/nova-custom-az0.yaml)
+- [edpm/computes/r1/nova-custom-az1.yaml](edpm/computes/r1/nova-custom-az1.yaml)
+- [edpm/computes/r2/nova-custom-az2.yaml](edpm/computes/r2/nova-custom-az2.yaml)
+
+### Deployment Process
+
+The ConfigMaps and custom services are automatically included when generating the nodeset CRs from each rack's directory. Each r0, r1, r2 directory contains its own AZ-specific configurations and generates the complete CRs including ConfigMaps and custom services.
+
+### Nodeset Configuration
+
+Each compute nodeset references its AZ-specific custom service:
 
-> The Compute service (nova) provides a default ConfigMap CR named nova-extra-config, where you can add generic configuration that applies to all the node sets that use the default nova service. If you use this default nova-extra-config ConfigMap to add generic configuration to be applied to all the node sets, then you do not need to create a custom service.
+- r0 nodesets use `nova-custom-az0` service
+- r1 nodesets use `nova-custom-az1` service  
+- r2 nodesets use `nova-custom-az2` service
 
-The EDPM `nova` service created in the next step will apply the above configuration to all Compute nodes.
+This ensures that Nova compute services in each AZ connect to their local Glance endpoints for optimal performance and proper distributed zones functionality.
 
 ## Create Networker and Compute Nodeset CRs
 

examples/dt/dz-storage/edpm/computes/kustomization.yaml

@@ -9,6 +9,8 @@ components:
 
 resources:
   - values.yaml
+  - nova-extra-config-az0.yaml
+  - nova-custom-az0.yaml
 
 patches:
   - target:

examples/dt/dz-storage/edpm/computes/nova-extra-config.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: dataplane.openstack.org/v1beta1
+kind: OpenStackDataPlaneService
+metadata:
+  name: nova-custom-az0
+spec:
+  addCertMounts: false
+  caCerts: combined-ca-bundle
+  edpmServiceType: nova
+  playbook: osp.edpm.nova
+  tlsCerts:
+    default:
+      contents:
+        - dnsnames
+        - ips
+      edpmRoleServiceName: nova
+      issuer: osp-rootca-issuer-internal
+      networks:
+        - ctlplane
+      dataSources:
+        - configMapRef:
+            name: nova-extra-config-az0
+        - secretRef:
+            name: nova-migration-ssh-key
+        - secretRef:
+            name: nova-cell1-compute-config

examples/dt/dz-storage/edpm/computes/r0/kustomization.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nova-extra-config-az0
+data:
+  25-nova-extra.conf: |
+    [DEFAULT]
+    # Triple the default of the following
+    reimage_timeout_per_gb = 60
+    [glance]
+    endpoint_override = https://glance-az0-internal.openstack.svc:9292
+    valid_interfaces = internal
+    [cinder]
+    cross_az_attach = False
+    catalog_info = volumev3:cinderv3:internalURL

examples/dt/dz-storage/edpm/computes/r0/nova-custom-az0.yaml

@@ -203,7 +203,7 @@ data:
       - neutron-metadata
       - ovn-bgp-agent
       - libvirt
-      - nova
+      - nova-custom-az0
   nova:
     migration:
       ssh_keys: