diff --git a/jobs/cc_deployment_updater/spec b/jobs/cc_deployment_updater/spec index 8ea05b6300..1318eee4fa 100644 --- a/jobs/cc_deployment_updater/spec +++ b/jobs/cc_deployment_updater/spec @@ -27,6 +27,10 @@ templates: resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem + storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json + storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json + storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json + storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json packages: - capi_utils diff --git a/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb b/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb index da5e6a3edf..b7b2e11b3e 100644 --- a/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb +++ b/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb @@ -152,6 +152,11 @@ maximum_health_check_timeout: <%= link("cloud_controller_internal").p("cc.maximu stacks_file: /var/vcap/jobs/cloud_controller_ng/config/stacks.yml +storage_cli_config_file_droplets: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cc_deployment_updater/config/storage_cli_config_resource_pool.json + resource_pool: blobstore_type: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_type") %> webdav_config: @@ -166,6 +171,8 @@ resource_pool: minimum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.minimum_size") %> maximum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.maximum_size") %> resource_directory_key: <%= link("cloud_controller_internal").p("cc.resource_pool.resource_directory_key") %> + blobstore_provider: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_provider") %> + connection_config: <%= link("cloud_controller_internal").p("cc.resource_pool.connection_config", {}).to_json %> <% link("cloud_controller_internal").if_p("cc.resource_pool.cdn") do %> cdn: uri: <%= link("cloud_controller_internal").p("cc.resource_pool.cdn.uri") %> @@ -188,6 +195,8 @@ packages: ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/packages_ca_cert.pem" <% end %> app_package_directory_key: <%= link("cloud_controller_internal").p("cc.packages.app_package_directory_key") %> + blobstore_provider: <%= link("cloud_controller_internal").p("cc.packages.blobstore_provider") %> + connection_config: <%= link("cloud_controller_internal").p("cc.packages.connection_config", {}).to_json %> max_package_size: <%= link("cloud_controller_internal").p("cc.packages.max_package_size") %> <% link("cloud_controller_internal").if_p("cc.packages.cdn") do %> cdn: @@ -212,6 +221,8 @@ droplets: ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/droplets_ca_cert.pem" <% end %> droplet_directory_key: <%= link("cloud_controller_internal").p("cc.droplets.droplet_directory_key") %> + blobstore_provider: <%= link("cloud_controller_internal").p("cc.droplets.blobstore_provider") %> + connection_config: <%= link("cloud_controller_internal").p("cc.droplets.connection_config", {}).to_json %> <% link("cloud_controller_internal").if_p("cc.droplets.cdn") do %> cdn: uri: <%= link("cloud_controller_internal").p("cc.droplets.cdn.uri") %> @@ -234,6 +245,8 @@ buildpacks: ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/buildpacks_ca_cert.pem" <% end %> buildpack_directory_key: <%= link("cloud_controller_internal").p("cc.buildpacks.buildpack_directory_key") %> + blobstore_provider: <%= link("cloud_controller_internal").p("cc.buildpacks.blobstore_provider") %> + connection_config: <%= link("cloud_controller_internal").p("cc.buildpacks.connection_config", {}).to_json %> <% link("cloud_controller_internal").if_p("cc.buildpacks.cdn") do %> cdn: uri: <%= link("cloud_controller_internal").p("cc.buildpacks.cdn.uri") %> diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb new file mode 100644 index 0000000000..36ba5f755a --- /dev/null +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_buildpacks.json.erb @@ -0,0 +1,50 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +l = link("cloud_controller_internal") + +scope = "cc.buildpacks.connection_config" +provider = l.p("cc.buildpacks.blobstore_provider", nil) + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = l.p("#{scope}.azure_storage_account_name") + options["container_name"] = l.p("#{scope}.container_name") + add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) + add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = l.p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb new file mode 100644 index 0000000000..363a7011f6 --- /dev/null +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_droplets.json.erb @@ -0,0 +1,50 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +l = link("cloud_controller_internal") + +scope = "cc.droplets.connection_config" +provider = l.p("cc.droplets.blobstore_provider", nil) + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = l.p("#{scope}.azure_storage_account_name") + options["container_name"] = l.p("#{scope}.container_name") + add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) + add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = l.p("cc.droplets.connection_config.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb new file mode 100644 index 0000000000..4de210731c --- /dev/null +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_packages.json.erb @@ -0,0 +1,50 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +l = link("cloud_controller_internal") + +scope = "cc.packages.connection_config" +provider = l.p("cc.packages.blobstore_provider", nil) + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = l.p("#{scope}.azure_storage_account_name") + options["container_name"] = l.p("#{scope}.container_name") + add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) + add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = l.p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb new file mode 100644 index 0000000000..bbf336fcfb --- /dev/null +++ b/jobs/cc_deployment_updater/templates/storage_cli_config_resource_pool.json.erb @@ -0,0 +1,50 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +l = link("cloud_controller_internal") + +scope = "cc.resource_pool.connection_config" +provider = l.p("cc.resource_pool.blobstore_provider", nil) + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = l.p("#{scope}.azure_storage_account_name") + options["container_name"] = l.p("#{scope}.container_name") + add(options, "account_key", l.p("#{scope}.azure_storage_access_key")) + add(options, "environment", l.p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", l.p("#{scope}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = l.p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb new file mode 100644 index 0000000000..96bbe225e9 --- /dev/null +++ b/spec/cc_deployment_updater/storage_cli_config_jsons_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'rspec' +require 'yaml' +require 'bosh/template/test' + +TEMPLATES = { + droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]], + buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]], + packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]], + resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]] +}.freeze + +module Bosh + module Template + module Test + RSpec.describe 'storage-cli JSON templates (cc_deployment_updater)' do + let(:release_path) { File.join(File.dirname(__FILE__), '../..') } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('cc_deployment_updater') } + + let(:link_props) do + { + 'cc' => { + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' } + } + } + end + + let(:cc_link) do + Bosh::Template::Test::Link.new( + name: 'cloud_controller_internal', + properties: link_props + ) + end + + let(:links) { [cc_link] } + let(:props) { {} } + + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end + + TEMPLATES.each do |scope, (template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + context "when provider is AzureRM for #{scope}" do + before do + link_props['cc'][scope.to_s]['blobstore_provider'] = 'AzureRM' + end + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(link_props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(link_props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end + + context "when provider is non-Azure for #{scope}" do + it 'renders {}' do + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end + end + end + end +end