From 02a879eca7b0632b117fa2a452eb186839f62e56 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 3 May 2022 14:30:08 +0200 Subject: [PATCH 01/35] Generate stack files --- go.mod | 3 + ...istry => Dockerfile.package-registry.tmpl} | 2 +- .../profile/_static/elastic-agent.env.tmpl | 7 + internal/profile/_static/elastic-agent_8x.env | 4 - .../profile/_static/elastic-agent_default.env | 3 - ...fig_default.yml => elasticsearch.yml.tmpl} | 7 +- .../_static/elasticsearch_config_8x.yml | 11 - .../{kibana_config_8x.yml => kibana.yml.tmpl} | 18 +- internal/profile/_static/kibana_config_80.yml | 46 --- .../profile/_static/kibana_config_default.yml | 16 - internal/profile/config_profile_test.go | 82 ---- internal/profile/files.go | 58 --- internal/profile/install_profile.go | 229 ----------- internal/profile/options.go | 13 - internal/profile/profile.go | 363 ++++-------------- internal/profile/profile_json.go | 55 --- internal/profile/profile_test.go | 13 + internal/profile/static.go | 184 --------- 18 files changed, 120 insertions(+), 994 deletions(-) rename internal/profile/_static/{Dockerfile.package-registry => Dockerfile.package-registry.tmpl} (79%) create mode 100644 internal/profile/_static/elastic-agent.env.tmpl delete mode 100644 internal/profile/_static/elastic-agent_8x.env delete mode 100644 internal/profile/_static/elastic-agent_default.env rename internal/profile/_static/{elasticsearch_config_default.yml => elasticsearch.yml.tmpl} (77%) delete mode 100644 internal/profile/_static/elasticsearch_config_8x.yml rename internal/profile/_static/{kibana_config_8x.yml => kibana.yml.tmpl} (74%) delete mode 100644 internal/profile/_static/kibana_config_80.yml delete mode 100644 internal/profile/_static/kibana_config_default.yml delete mode 100644 internal/profile/config_profile_test.go delete mode 100644 internal/profile/files.go delete mode 100644 internal/profile/install_profile.go delete mode 100644 internal/profile/options.go delete mode 100644 internal/profile/profile_json.go create mode 100644 internal/profile/profile_test.go delete mode 100644 internal/profile/static.go diff --git a/go.mod b/go.mod index 6ece954c0f..54e30d462a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 github.com/elastic/go-elasticsearch/v7 v7.17.1 github.com/elastic/go-licenser v0.4.0 + github.com/elastic/go-resource v0.0.0-00010101000000-000000000000 github.com/elastic/go-ucfg v0.8.4 github.com/elastic/package-spec v1.7.0 github.com/fatih/color v1.13.0 @@ -151,3 +152,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/elastic/go-resource => /home/jaime/gocode/src/github.com/elastic/go-resource diff --git a/internal/profile/_static/Dockerfile.package-registry b/internal/profile/_static/Dockerfile.package-registry.tmpl similarity index 79% rename from internal/profile/_static/Dockerfile.package-registry rename to internal/profile/_static/Dockerfile.package-registry.tmpl index 2e4ebe4904..3275cb3c2a 100644 --- a/internal/profile/_static/Dockerfile.package-registry +++ b/internal/profile/_static/Dockerfile.package-registry.tmpl @@ -1,4 +1,4 @@ -FROM __BASE_IMAGE__ +FROM {{ fact "registry_base_image" }} ARG PROFILE COPY profiles/${PROFILE}/stack/package-registry.config.yml /package-registry/config.yml diff --git a/internal/profile/_static/elastic-agent.env.tmpl b/internal/profile/_static/elastic-agent.env.tmpl new file mode 100644 index 0000000000..c815f1134c --- /dev/null +++ b/internal/profile/_static/elastic-agent.env.tmpl @@ -0,0 +1,7 @@ +{{ $version := fact "agent_version" }} +FLEET_ENROLL=1 +FLEET_INSECURE=1 +FLEET_URL=http://fleet-server:8220 +{{ if not (semverLessThan $version "8.0.0") }} +FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) +{{ end }} diff --git a/internal/profile/_static/elastic-agent_8x.env b/internal/profile/_static/elastic-agent_8x.env deleted file mode 100644 index 2d35422b79..0000000000 --- a/internal/profile/_static/elastic-agent_8x.env +++ /dev/null @@ -1,4 +0,0 @@ -FLEET_ENROLL=1 -FLEET_INSECURE=1 -FLEET_URL=http://fleet-server:8220 -FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package) \ No newline at end of file diff --git a/internal/profile/_static/elastic-agent_default.env b/internal/profile/_static/elastic-agent_default.env deleted file mode 100644 index bb341c3952..0000000000 --- a/internal/profile/_static/elastic-agent_default.env +++ /dev/null @@ -1,3 +0,0 @@ -FLEET_ENROLL=1 -FLEET_INSECURE=1 -FLEET_URL=http://fleet-server:8220 \ No newline at end of file diff --git a/internal/profile/_static/elasticsearch_config_default.yml b/internal/profile/_static/elasticsearch.yml.tmpl similarity index 77% rename from internal/profile/_static/elasticsearch_config_default.yml rename to internal/profile/_static/elasticsearch.yml.tmpl index 8dc75fdda3..7943d8928a 100644 --- a/internal/profile/_static/elasticsearch_config_default.yml +++ b/internal/profile/_static/elasticsearch.yml.tmpl @@ -8,10 +8,13 @@ xpack.license.self_generated.type: "trial" xpack.security.enabled: true xpack.security.authc.api_key.enabled: true +ingest.geoip.downloader.enabled: false + +{{ $version := fact "elasticsearch_version" }} +{{ if semverLessThan $version "8.0.0" }} script.max_compilations_rate: "use-context" script.context.template.max_compilations_rate: "unlimited" script.context.ingest.cache_max_size: 2000 script.context.processor_conditional.cache_max_size: 2000 script.context.template.cache_max_size: 2000 - -ingest.geoip.downloader.enabled: false \ No newline at end of file +{{ end }} diff --git a/internal/profile/_static/elasticsearch_config_8x.yml b/internal/profile/_static/elasticsearch_config_8x.yml deleted file mode 100644 index c4fb16b745..0000000000 --- a/internal/profile/_static/elasticsearch_config_8x.yml +++ /dev/null @@ -1,11 +0,0 @@ -network.host: "" -transport.host: "127.0.0.1" -http.host: "0.0.0.0" - -indices.id_field_data.enabled: true - -xpack.license.self_generated.type: "trial" -xpack.security.enabled: true -xpack.security.authc.api_key.enabled: true - -ingest.geoip.downloader.enabled: false \ No newline at end of file diff --git a/internal/profile/_static/kibana_config_8x.yml b/internal/profile/_static/kibana.yml.tmpl similarity index 74% rename from internal/profile/_static/kibana_config_8x.yml rename to internal/profile/_static/kibana.yml.tmpl index 44fbb7929e..653a4cd9e1 100644 --- a/internal/profile/_static/kibana_config_8x.yml +++ b/internal/profile/_static/kibana.yml.tmpl @@ -1,20 +1,33 @@ +{{ $version := fact "kibana_version" }} server.name: kibana server.host: "0.0.0.0" elasticsearch.hosts: [ "http://elasticsearch:9200" ] +{{ if semverLessThan $version "8.0.0" }} +elasticsearch.username: elastic +elasticsearch.password: changeme + +xpack.monitoring.ui.container.elasticsearch.enabled: true +xpack.fleet.enabled: true +xpack.fleet.agents.elasticsearch.host: "http://elasticsearch:9200" +{{ else }} elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" monitoring.ui.container.elasticsearch.enabled: true +xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"] +{{ end }} xpack.fleet.registryUrl: "http://package-registry:8080" xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"] xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" +{{ if not (semverLessThan $version "8.1.0") }} xpack.cloudSecurityPosture.enabled: true +{{ end }} +{{ if not (semverLessThan $version "8.0.0") }} xpack.fleet.packages: - name: system version: latest @@ -45,4 +58,5 @@ xpack.fleet.agentPolicies: - name: fleet_server-1 id: default-fleet-server package: - name: fleet_server \ No newline at end of file + name: fleet_server +{{ end }} diff --git a/internal/profile/_static/kibana_config_80.yml b/internal/profile/_static/kibana_config_80.yml deleted file mode 100644 index db754f4ed8..0000000000 --- a/internal/profile/_static/kibana_config_80.yml +++ /dev/null @@ -1,46 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" - -elasticsearch.hosts: [ "http://elasticsearch:9200" ] -elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" - -monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.registryUrl: "http://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["http://elasticsearch:9200"] -xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" - -xpack.fleet.packages: - - name: system - version: latest - - name: elastic_agent - version: latest - - name: fleet_server - version: latest -xpack.fleet.agentPolicies: - - name: Elastic-Agent (elastic-package) - id: elastic-agent-managed-ep - is_default: true - is_managed: false - namespace: default - monitoring_enabled: - - logs - - metrics - package_policies: - - name: system-1 - id: default-system - package: - name: system - - name: Fleet Server (elastic-package) - id: fleet-server-policy - is_default_fleet_server: true - is_managed: false - namespace: default - package_policies: - - name: fleet_server-1 - id: default-fleet-server - package: - name: fleet_server \ No newline at end of file diff --git a/internal/profile/_static/kibana_config_default.yml b/internal/profile/_static/kibana_config_default.yml deleted file mode 100644 index d0bec7c25d..0000000000 --- a/internal/profile/_static/kibana_config_default.yml +++ /dev/null @@ -1,16 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" - -elasticsearch.hosts: [ "http://elasticsearch:9200" ] -elasticsearch.username: elastic -elasticsearch.password: changeme - -xpack.monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.enabled: true -xpack.fleet.registryUrl: "http://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.host: "http://elasticsearch:9200" -xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" diff --git a/internal/profile/config_profile_test.go b/internal/profile/config_profile_test.go deleted file mode 100644 index c1f00d967a..0000000000 --- a/internal/profile/config_profile_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. -package profile - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - profileName = "test_profile" -) - -func TestNewProfile(t *testing.T) { - elasticPackageDir, err := os.MkdirTemp("", "package") - defer cleanupProfile(t, elasticPackageDir) - assert.NoError(t, err, "error creating tempdir") - t.Logf("writing to directory %s", elasticPackageDir) - - options := Options{ - PackagePath: elasticPackageDir, - Name: profileName, - OverwriteExisting: false, - } - err = createProfile(options) - assert.NoErrorf(t, err, "error creating profile %s", err) - -} - -func TestNewProfileFrom(t *testing.T) { - elasticPackageDir, err := os.MkdirTemp("", "package") - defer cleanupProfile(t, elasticPackageDir) - assert.NoError(t, err, "error creating tempdir") - t.Logf("writing to directory %s", elasticPackageDir) - - options := Options{ - PackagePath: elasticPackageDir, - Name: profileName, - OverwriteExisting: false, - } - err = createProfile(options) - assert.NoErrorf(t, err, "error creating profile %s", err) - - //update the profile to make sure we're properly copying everything - - testProfile, err := NewConfigProfile(elasticPackageDir, profileName) - assert.NoErrorf(t, err, "error creating profile %s", err) - - pkgRegUpdated := &simpleFile{ - path: filepath.Join(testProfile.ProfilePath, string(PackageRegistryConfigFile)), - body: `package_paths: - - /packages/testing - - /packages/development - - /packages/production - - /packages/staging - - /packages/snapshot - `, - } - t.Logf("updating profile %s", testProfile.ProfilePath) - testProfile.configFiles[PackageRegistryConfigFile] = pkgRegUpdated - err = testProfile.writeProfileResources() - assert.NoErrorf(t, err, "error updating profile %s", err) - - // actually create & check the new profile - option := Options{ - PackagePath: elasticPackageDir, - Name: "test_from", - FromProfile: profileName, - } - err = createProfileFrom(option) - assert.NoErrorf(t, err, "error copying updating profile %s", err) - -} - -func cleanupProfile(t *testing.T, dir string) { - err := os.RemoveAll(dir) - assert.NoErrorf(t, err, "Error cleaning up tempdir %s", dir) -} diff --git a/internal/profile/files.go b/internal/profile/files.go deleted file mode 100644 index 9a4dc445d9..0000000000 --- a/internal/profile/files.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - "os" - - "github.com/pkg/errors" -) - -// NewConfig is a generic function type to return a new Managed config -type NewConfig = func(profileName string, profilePath string) (*simpleFile, error) - -// simpleFile defines a file that's managed by the profile system -// and doesn't require any rendering -type simpleFile struct { - name string - path string - body string -} - -const profileStackPath = "stack" - -// configFilesDiffer checks to see if a local configItem differs from the one it knows. -func (cfg simpleFile) configFilesDiffer() (bool, error) { - changes, err := os.ReadFile(cfg.path) - if err != nil && errors.Is(err, os.ErrNotExist) { - return false, nil - } - if err != nil { - return false, errors.Wrapf(err, "error reading %s", cfg.path) - } - if string(changes) != cfg.body { - return true, nil - } - return false, nil -} - -// writeConfig writes the config item -func (cfg simpleFile) writeConfig() error { - err := os.WriteFile(cfg.path, []byte(cfg.body), 0644) - if err != nil { - return errors.Wrapf(err, "writing file failed (path: %s)", cfg.path) - } - return nil -} - -// readConfig reads the config item, overwriting whatever exists in the fileBody. -func (cfg *simpleFile) readConfig() error { - body, err := os.ReadFile(cfg.path) - if err != nil { - return errors.Wrapf(err, "reading filed failed (path: %s)", cfg.path) - } - cfg.body = string(body) - return nil -} diff --git a/internal/profile/install_profile.go b/internal/profile/install_profile.go deleted file mode 100644 index eb13dfc161..0000000000 --- a/internal/profile/install_profile.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/logger" -) - -// CreateProfile creates an existing profile from the default elastic-package config dir -// if option.Package has a value it'll be used, if not, the default location will be used -// if option.From does not have a supplied value, it'll create a default profile. -func CreateProfile(options Options) error { - if options.PackagePath == "" { - loc, err := locations.NewLocationManager() - if err != nil { - return errors.Wrap(err, "error finding stack dir location") - } - options.PackagePath = loc.ProfileDir() - } - - // If they're creating from Default, assume they want the actual default, and - // not whatever is currently inside default. - if options.FromProfile == "" || options.FromProfile == DefaultProfile { - return createProfile(options) - } - - return createProfileFrom(options) -} - -// MigrateProfileFiles creates a new profile based on existing filepaths -// that are stored elsewhere outside the profile system. -func MigrateProfileFiles(options Options, files []string) error { - profile, err := newProfileFromExistingFiles(options.PackagePath, options.Name, files, true) - if err != nil { - return errors.Wrap(err, "error creating new profile from files") - } - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = profile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing out new profile config") - } - return nil -} - -// LoadProfile loads an existing profile from the default elastic-package config dir -func LoadProfile(profileName string) (*Profile, error) { - loc, err := locations.NewLocationManager() - if err != nil { - return nil, errors.Wrap(err, "error finding stack dir location") - } - - return loadProfile(loc.ProfileDir(), profileName) -} - -// DeleteProfile deletes a profile from the default elastic-package config dir -func DeleteProfile(profileName string) error { - loc, err := locations.NewLocationManager() - if err != nil { - return errors.Wrap(err, "error finding stack dir location") - } - return deleteProfile(loc.ProfileDir(), profileName) -} - -// FetchAllProfiles returns a list of profile values -func FetchAllProfiles(elasticPackagePath string) ([]Metadata, error) { - dirList, err := os.ReadDir(elasticPackagePath) - if err != nil { - return []Metadata{}, errors.Wrapf(err, "error reading from directory %s", elasticPackagePath) - } - - var profiles []Metadata - // TODO: this should read a profile.json file or something like that - for _, item := range dirList { - if !item.IsDir() { - continue - } - profile, err := loadProfile(elasticPackagePath, item.Name()) - if errors.Is(err, ErrNotAProfile) { - continue - } - if err != nil { - return profiles, errors.Wrapf(err, "error loading profile %s", item.Name()) - } - metadata, err := profile.metadata() - if err != nil { - return profiles, errors.Wrap(err, "error reading profile metadata") - } - profiles = append(profiles, metadata) - } - return profiles, nil -} - -// createProfile installs a new profile at the given package path location. -// overwriteExisting determines the behavior if a profile with the given name already exists. -// On true, it'll overwrite the profile, on false, it'll backup the existing profile to profilename_VERSION-DATE-CREATED -func createProfile(options Options) error { - profile, err := createAndCheckProfile(options.PackagePath, options.Name, options.OverwriteExisting) - if err != nil { - return errors.Wrap(err, "error creating new profile") - } - // write the resources - err = profile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing profile file") - } - return nil -} - -// createProfileFrom creates a new profile by copying over an existing profile -func createProfileFrom(options Options) error { - fromProfile, err := loadProfile(options.PackagePath, options.FromProfile) - if err != nil { - return errors.Wrapf(err, "error loading %s profile", options.FromProfile) - } - - newProfile, err := createAndCheckProfile(options.PackagePath, options.Name, options.OverwriteExisting) - if err != nil { - return errors.Wrap(err, "error creating new profile") - } - - newProfile.overwrite(fromProfile.configFiles) - err = newProfile.writeProfileResources() - if err != nil { - return errors.Wrap(err, "error writing new profile") - } - return nil -} - -// createAndCheckProfile does most of the heavy lifting for initializing a new profile, -// including dealing with profile overwrites -func createAndCheckProfile(packagePath, packageName string, overwriteExisting bool) (*Profile, error) { - profile, err := NewConfigProfile(packagePath, packageName) - if err != nil { - return nil, errors.Wrap(err, "error creating profile") - } - - // check to see if we have an existing profile at that location. - exists, err := profile.alreadyExists() - if err != nil { - return nil, errors.Wrap(err, "error checking for existing profile") - } - if exists { - localChanges, err := profile.localFilesChanged() - if err != nil { - return nil, errors.Wrapf(err, "error checking for changes in %s", profile.ProfilePath) - } - // If there are changes and we've selected CreateNew, move the old path - if localChanges && !overwriteExisting { - if localChanges && packageName == DefaultProfile { - logger.Warn("Default profile has been changed by user or updated by elastic-package. The current profile will be moved.") - } - // Migrate the existing profile - err = updateExistingDefaultProfile(packagePath) - if err != nil { - return nil, errors.Wrap(err, "error moving old profile") - } - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = os.Mkdir(profile.ProfileStackPath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - } - } else { - err = os.Mkdir(profile.ProfilePath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - err = os.Mkdir(profile.ProfileStackPath, 0755) - if err != nil { - return nil, errors.Wrapf(err, "error crating profile directory %s", profile.ProfilePath) - } - } - - return profile, nil -} - -// updateExistingDefaultProfile migrates the old default profile to profile_VERSION_DATE-CREATED -func updateExistingDefaultProfile(path string) error { - profile, err := NewConfigProfile(path, DefaultProfile) - if err != nil { - return errors.Wrap(err, "error creating profile") - } - meta, err := profile.metadata() - if err != nil { - return errors.Wrap(err, "error updating metadata") - } - newName := fmt.Sprintf("default_%s_%d", meta.Version, meta.DateCreated.Unix()) - newFilePath := filepath.Join(filepath.Dir(profile.ProfilePath), newName) - meta.Name = newName - meta.Path = newFilePath - - err = profile.updateMetadata(meta) - if err != nil { - return errors.Wrap(err, "error updating metadata") - } - - err = os.Rename(profile.ProfilePath, newFilePath) - if err != nil { - return errors.Wrap(err, "error moving default profile") - } - return nil -} - -// deleteProfile deletes a given config profile. -func deleteProfile(elasticPackagePath string, profileName string) error { - if profileName == DefaultProfile { - return errors.New("cannot remove default profile") - } - - pathToDelete := filepath.Join(elasticPackagePath, profileName) - - return os.RemoveAll(pathToDelete) - -} diff --git a/internal/profile/options.go b/internal/profile/options.go deleted file mode 100644 index 7e3fca4a08..0000000000 --- a/internal/profile/options.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -// Options defines available stack management options -type Options struct { - PackagePath string - Name string - FromProfile string - OverwriteExisting bool -} diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 82efb775c9..a1589a8309 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -1,315 +1,102 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - package profile import ( - "encoding/json" + "embed" "fmt" "os" "path/filepath" + "strings" + "text/template" - "github.com/pkg/errors" -) - -// Profile manages a a given user config profile -type Profile struct { - // ProfilePath is the absolute path to the profile - ProfilePath string - ProfileStackPath string - profileName string - configFiles map[configFile]*simpleFile -} - -const ( - // DefaultProfile is the name of the default profile - DefaultProfile = "default" + "github.com/Masterminds/semver" + "github.com/elastic/go-resource" ) -// ErrNotAProfile is returned in cases where we don't have a valid profile directory -var ErrNotAProfile = errors.New("not a profile") - -// configFile is a type for for the config file names in a managed profile config -type configFile string - -// managedProfileFiles is the list of all files managed in a profile -// If you create a new file that's managed by a profile, it needs to go in this list -var managedProfileFiles = map[configFile]NewConfig{ - ElasticAgentDefaultEnvFile: newElasticAgentDefaultEnv, - ElasticAgent8xEnvFile: newElasticAgent8xEnv, - ElasticAgent80EnvFile: newElasticAgent80Env, - ElasticsearchConfigDefaultFile: newElasticsearchConfigDefault, - ElasticsearchConfig8xFile: newElasticsearchConfig8x, - ElasticsearchConfig80File: newElasticsearchConfig80, - KibanaConfigDefaultFile: newKibanaConfigDefault, - KibanaConfig8xFile: newKibanaConfig8x, - KibanaConfig80File: newKibanaConfig80, - PackageRegistryDockerfileFile: newPackageRegistryDockerfile, - PackageRegistryConfigFile: newPackageRegistryConfig, - SnapshotFile: newSnapshotFile, - PackageProfileMetaFile: createProfileMetadata, -} - -// NewConfigProfile creates a new config profile manager -func NewConfigProfile(elasticPackagePath string, profileName string) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) - - var configMap = map[configFile]*simpleFile{} - for fileItem, configInit := range managedProfileFiles { - cfg, err := configInit(profileName, profilePath) - if err != nil { - return nil, errors.Wrapf(err, "error initializing config %s", cfg) - } - configMap[fileItem] = cfg - } - - newProfile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - ProfileStackPath: filepath.Join(profilePath, profileStackPath), - configFiles: configMap, - } - return newProfile, nil -} - -// newProfileFromExistingFiles creates a profile from a list of absolute filepaths -// This can be used when migrating a config from a non-profiles-managed config set -// ignoreMissing will treat non-existant files as soft errors -func newProfileFromExistingFiles(elasticPackagePath string, profileName string, files []string, ignoreMissing bool) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) - var configMap = map[configFile]*simpleFile{} - for _, file := range files { - if ignoreMissing { - // if we're treating missing files as soft errors, - // just continue on ErrNotExist - // If it's another kind of error, we'll pick it up in ReadFile - if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { - continue - } - } - - byteFile, err := os.ReadFile(file) - if err != nil { - return nil, errors.Wrapf(err, "error reading file %s", file) - } - //format this in the way configFile expects - name := filepath.Base(file) - configMap[configFile(name)] = &simpleFile{ - name: name, - path: filepath.Join(profilePath, name), - body: string(byteFile), - } - } - - //add metadata file - metadata, err := createProfileMetadata(profileName, profilePath) - if err != nil { - return nil, errors.Wrap(err, "error creating profile metadata") - } - configMap[PackageProfileMetaFile] = metadata - - newProfile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - configFiles: configMap, - } - return newProfile, nil -} - -// loadProfile loads an existing profile -func loadProfile(elasticPackagePath string, profileName string) (*Profile, error) { - profilePath := filepath.Join(elasticPackagePath, profileName) - - isValid, err := isProfileDir(profilePath) - if err != nil { - return nil, errors.Wrapf(err, "error checking profile %s", profileName) - } - - if !isValid { - return nil, ErrNotAProfile - } - - var configMap = map[configFile]*simpleFile{} - for fileItem, configInit := range managedProfileFiles { - cfg, err := configInit(profileName, profilePath) - if err != nil { - return nil, errors.Wrapf(err, "error initializing config %s", cfg) - } - configMap[fileItem] = cfg - } - - profile := &Profile{ - profileName: profileName, - ProfilePath: profilePath, - ProfileStackPath: filepath.Join(profilePath, profileStackPath), - configFiles: configMap, - } - - exists, err := profile.alreadyExists() - if err != nil { - return nil, errors.Wrapf(err, "error checking if profile %s exists", profileName) - } - - if !exists { - return nil, fmt.Errorf("profile %s does not exist", profile.ProfilePath) - } - - err = profile.readProfileResources() - if err != nil { - return nil, errors.Wrapf(err, "error reading in profile %s", profileName) - } - - return profile, nil - -} - -// FetchPath returns an absolute path to the given file -func (profile Profile) FetchPath(file configFile) string { - return profile.configFiles[file].path -} - -// ComposeEnvVars returns a list of environment variables that can be passed -// to docker-compose for the sake of filling out paths and names in the snapshot.yml file. -func (profile Profile) ComposeEnvVars() []string { - return []string{ - fmt.Sprintf("PROFILE_NAME=%s", profile.profileName), - fmt.Sprintf("STACK_PATH=%s", profile.ProfileStackPath), - } -} - -// writeProfileResources writes the config files -func (profile Profile) writeProfileResources() error { - for _, cfgFiles := range profile.configFiles { - err := cfgFiles.writeConfig() - if err != nil { - return errors.Wrap(err, "error writing config file") - } +// PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. +const PackageRegistryBaseImage = "docker.elastic.co/package-registry/distribution:snapshot" + +//go:embed _static +var static embed.FS + +var ( + templateFuncs = template.FuncMap{ + "semverLessThan": semverLessThan, + } + staticSource = resource.NewSourceFS(static).WithTemplateFuncs(templateFuncs) + profileResources = []resource.Resource{ + &resource.File{ + Provider: "stack-file", + Path: "Dockerfile.package-registry", + Content: staticSource.Template("_static/Dockerfile.package-registry.tmpl"), + }, + &resource.File{ + Provider: "stack-file", + Path: "snapshot.yml", + Content: staticSource.File("_static/docker-compose-stack.yml"), + }, + &resource.File{ + Provider: "stack-file", + Path: "elasticsearch.yml", + Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), + }, + &resource.File{ + Provider: "stack-file", + Path: "kibana.yml", + Content: staticSource.Template("_static/kibana.yml.tmpl"), + }, + &resource.File{ + Provider: "stack-file", + Path: "package_registry.yml", + Content: staticSource.File("_static/package_registry.yml"), + }, + &resource.File{ + Provider: "stack-file", + Path: "elastic-agent.env", + Content: staticSource.Template("_static/elastic-agent.env.tmpl"), + }, } - return nil -} +) -// overwrite updates the string contents of the config files -func (profile *Profile) overwrite(newBody map[configFile]*simpleFile) { - for key := range profile.configFiles { - // skip metadata - if key == PackageProfileMetaFile { - continue - } - toReplace, ok := newBody[key] - if ok { - updatedProfile := profile.configFiles[key] - updatedProfile.body = toReplace.body - profile.configFiles[key] = updatedProfile - } - } +func CreateProfile(path string) error { + stackVersion := "8.1.0" // TODO: Parameterize this. -} + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "registry_base_image": PackageRegistryBaseImage, -// alreadyExists checks to see if a profile with this name already exists -func (profile Profile) alreadyExists() (bool, error) { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - // We do this in stages to make sure we return the right error. - _, err := os.Stat(profile.ProfilePath) - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - if err != nil { - return false, errors.Wrapf(err, "error checking root directory: %s", packageMetadata.path) - } + "elasticsearch_version": stackVersion, + "kibana_version": stackVersion, + "agent_version": stackVersion, + }) - // If the folder exists, check to make sure it's a profile folder - _, err = os.Stat(packageMetadata.path) - if errors.Is(err, os.ErrNotExist) { - return false, ErrNotAProfile - } - if err != nil { - return false, errors.Wrapf(err, "error checking metadata: %s", packageMetadata.path) - } + stackDir := filepath.Join(path, "stack") + os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("stack-file", &resource.FileProvider{ + Prefix: stackDir, + }) - //if it is, see if it has the same profile name - profileInfo, err := profile.metadata() + results, err := resourceManager.Apply(profileResources) if err != nil { - return false, errors.Wrap(err, "error reading metadata") - } - - //TODO: this will break default_old, as we don't update the json - if profileInfo.Name != profile.profileName { - return false, nil - } - - return true, nil -} - -func (profile Profile) localFilesChanged() (bool, error) { - for cfgName, cfgFile := range profile.configFiles { - // skip checking the metadata file - // TODO: in the future, we might want to check version to see if the default profile needs to be updated - if cfgName == PackageProfileMetaFile { - continue - } - changes, err := cfgFile.configFilesDiffer() - if err != nil { - return false, errors.Wrap(err, "error checking config file") - } - if changes { - return true, nil - } - } - return false, nil -} - -// readProfileResources reads the associated files into the config, as opposed to writing them out. -func (profile Profile) readProfileResources() error { - for _, cfgFile := range profile.configFiles { - err := cfgFile.readConfig() - if err != nil { - return errors.Wrap(err, "error reading in profile") + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) } return nil } -// metadata returns the metadata struct for the profile -func (profile Profile) metadata() (Metadata, error) { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - rawPackageMetadata, err := os.ReadFile(packageMetadata.path) - if err != nil { - return Metadata{}, errors.Wrap(err, "error reading metadata file") - } - - profileInfo := Metadata{} - - err = json.Unmarshal(rawPackageMetadata, &profileInfo) - if err != nil { - return Metadata{}, errors.Wrap(err, "error unmarshalling JSON") - } - return profileInfo, nil -} - -// updateMetadata updates the metadata json file -func (profile *Profile) updateMetadata(meta Metadata) error { - packageMetadata := profile.configFiles[PackageProfileMetaFile] - metaString, err := json.Marshal(meta) +func semverLessThan(a, b string) (bool, error) { + sa, err := semver.NewVersion(a) if err != nil { - return errors.Wrap(err, "error marshalling metadata json") + return false, err } - err = os.WriteFile(packageMetadata.path, metaString, 0664) + sb, err := semver.NewVersion(b) if err != nil { - return errors.Wrap(err, "error writing metadata file") + return false, err } - return nil -} -// isProfileDir checks to see if the given path points to a valid profile -func isProfileDir(path string) (bool, error) { - metaPath := filepath.Join(path, string(PackageProfileMetaFile)) - _, err := os.Stat(metaPath) - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - if err != nil { - return false, errors.Wrapf(err, "error stat: %s", metaPath) - } - return true, nil + return sa.LessThan(sb), nil } diff --git a/internal/profile/profile_json.go b/internal/profile/profile_json.go deleted file mode 100644 index 449f051ca5..0000000000 --- a/internal/profile/profile_json.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - "encoding/json" - "os/user" - "path/filepath" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/version" -) - -// Metadata stores the data associated with a given profile -type Metadata struct { - Name string `json:"name"` - DateCreated time.Time `json:"date_created"` - User string `json:"user"` - Version string `json:"version"` - Path string `json:"path"` -} - -// PackageProfileMetaFile is the filename of the profile metadata file -const PackageProfileMetaFile configFile = "profile.json" - -// createProfileMetadata creates the body of the profile.json file -func createProfileMetadata(profileName string, profilePath string) (*simpleFile, error) { - currentUser, err := user.Current() - if err != nil { - return nil, errors.Wrap(err, "error fetching current user") - } - - profileData := Metadata{ - profileName, - time.Now(), - currentUser.Username, - version.CommitHash, - profilePath, - } - - jsonRaw, err := json.MarshalIndent(profileData, "", " ") - if err != nil { - return nil, errors.Wrap(err, "error marshalling json") - } - - return &simpleFile{ - name: string(PackageProfileMetaFile), - path: filepath.Join(profilePath, string(PackageProfileMetaFile)), - body: string(jsonRaw), - }, nil -} diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go new file mode 100644 index 0000000000..bf02c54f1c --- /dev/null +++ b/internal/profile/profile_test.go @@ -0,0 +1,13 @@ +package profile + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateProfile(t *testing.T) { + err := CreateProfile(t.TempDir()) + require.NoError(t, err) + +} diff --git a/internal/profile/static.go b/internal/profile/static.go deleted file mode 100644 index 7a96499e64..0000000000 --- a/internal/profile/static.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package profile - -import ( - _ "embed" - "path/filepath" - "strings" -) - -// SnapshotFile is the docker-compose snapshot.yml file name -const SnapshotFile configFile = "snapshot.yml" - -//go:embed _static/docker-compose-stack.yml -var snapshotYml string - -// newSnapshotFile returns a Managed Config -func newSnapshotFile(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(SnapshotFile), - path: filepath.Join(profilePath, profileStackPath, string(SnapshotFile)), - body: snapshotYml, - }, nil -} - -// KibanaConfigDefaultFile is the default kibana config file -const KibanaConfigDefaultFile configFile = "kibana.config.default.yml" - -//go:embed _static/kibana_config_default.yml -var kibanaConfigDefaultYml string - -func newKibanaConfigDefault(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfigDefaultFile), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfigDefaultFile)), - body: kibanaConfigDefaultYml, - }, nil -} - -// KibanaConfig8xFile is the Kibana config file for 8.x stack family -const KibanaConfig8xFile configFile = "kibana.config.8x.yml" - -//go:embed _static/kibana_config_8x.yml -var kibanaConfig8xYml string - -func newKibanaConfig8x(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfig8xFile), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig8xFile)), - body: kibanaConfig8xYml, - }, nil -} - -// KibanaConfig80File is the Kibana config file for 8.0 stack family (8.0 to 8.1) -const KibanaConfig80File configFile = "kibana.config.80.yml" - -//go:embed _static/kibana_config_80.yml -var kibanaConfig80Yml string - -func newKibanaConfig80(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(KibanaConfig80File), - path: filepath.Join(profilePath, profileStackPath, string(KibanaConfig80File)), - body: kibanaConfig80Yml, - }, nil -} - -// ElasticsearchConfigDefaultFile is the default Elasticsearch config file -const ElasticsearchConfigDefaultFile configFile = "elasticsearch.config.default.yml" - -//go:embed _static/elasticsearch_config_default.yml -var elasticsearchConfigDefaultYml string - -func newElasticsearchConfigDefault(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfigDefaultFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfigDefaultFile)), - body: elasticsearchConfigDefaultYml, - }, nil -} - -// ElasticsearchConfig8xFile is the Elasticsearch config file for 8.x stack family -const ElasticsearchConfig8xFile configFile = "elasticsearch.config.8x.yml" - -//go:embed _static/elasticsearch_config_8x.yml -var elasticsearchConfig8xYml string - -func newElasticsearchConfig8x(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfig8xFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig8xFile)), - body: elasticsearchConfig8xYml, - }, nil -} - -// ElasticsearchConfig80File is the Elasticsearch virtual config file name for 8.0 stack family (8.0 to 8.1) -// This file does not exist in the source code, since it's identical to the 8x config file. -const ElasticsearchConfig80File configFile = "elasticsearch.config.80.yml" - -func newElasticsearchConfig80(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticsearchConfig80File), - path: filepath.Join(profilePath, profileStackPath, string(ElasticsearchConfig80File)), - body: elasticsearchConfig8xYml, - }, nil -} - -// PackageRegistryConfigFile is the config file for the Elastic Package registry -const PackageRegistryConfigFile configFile = "package-registry.config.yml" - -//go:embed _static/package_registry.yml -var packageRegistryConfigYml string - -// newPackageRegistryConfig returns a Managed Config -func newPackageRegistryConfig(_ string, profilePath string) (*simpleFile, error) { - - return &simpleFile{ - name: string(PackageRegistryConfigFile), - path: filepath.Join(profilePath, profileStackPath, string(PackageRegistryConfigFile)), - body: packageRegistryConfigYml, - }, nil -} - -// PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. -const PackageRegistryBaseImage = "docker.elastic.co/package-registry/distribution:snapshot" - -// PackageRegistryDockerfileFile is the dockerfile for the Elastic package registry -const PackageRegistryDockerfileFile configFile = "Dockerfile.package-registry" - -//go:embed _static/Dockerfile.package-registry -var packageRegistryDockerfileTmpl string -var packageRegistryDockerfile = strings.Replace(packageRegistryDockerfileTmpl, - "__BASE_IMAGE__", PackageRegistryBaseImage, -1) - -// newPackageRegistryDockerfile returns a new config for the package-registry -func newPackageRegistryDockerfile(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(PackageRegistryDockerfileFile), - path: filepath.Join(profilePath, profileStackPath, string(PackageRegistryDockerfileFile)), - body: packageRegistryDockerfile, - }, nil -} - -// ElasticAgent80EnvFile is the .env for the 8.0 stack. -// This file does not exist in the source code, since it's identical to the 8x env file. -const ElasticAgent80EnvFile configFile = "elastic-agent.80.env" - -func newElasticAgent80Env(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgent80EnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent80EnvFile)), - body: elasticAgent8xEnv, - }, nil -} - -// ElasticAgent8xEnvFile is the .env for the 8x stack. -const ElasticAgent8xEnvFile configFile = "elastic-agent.8x.env" - -//go:embed _static/elastic-agent_8x.env -var elasticAgent8xEnv string - -func newElasticAgent8xEnv(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgent8xEnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgent8xEnvFile)), - body: elasticAgent8xEnv, - }, nil -} - -// ElasticAgentDefaultEnvFile is the default .env file. -const ElasticAgentDefaultEnvFile configFile = "elastic-agent.default.env" - -//go:embed _static/elastic-agent_default.env -var elasticAgentDefaultEnv string - -func newElasticAgentDefaultEnv(_ string, profilePath string) (*simpleFile, error) { - return &simpleFile{ - name: string(ElasticAgentDefaultEnvFile), - path: filepath.Join(profilePath, profileStackPath, string(ElasticAgentDefaultEnvFile)), - body: elasticAgentDefaultEnv, - }, nil -} From 97a3ce7ad3b63f9e83fa56d9c47540a7ce776727 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 3 May 2022 17:12:11 +0200 Subject: [PATCH 02/35] It builds --- cmd/profiles.go | 4 +- internal/install/install.go | 48 ----- ...kage_registry.yml => package-registry.yml} | 0 internal/profile/profile.go | 196 +++++++++++++++++- internal/profile/profile_json.go | 76 +++++++ internal/profile/profile_test.go | 5 +- internal/stack/shellinit.go | 4 +- 7 files changed, 269 insertions(+), 64 deletions(-) rename internal/profile/_static/{package_registry.yml => package-registry.yml} (100%) create mode 100644 internal/profile/profile_json.go diff --git a/cmd/profiles.go b/cmd/profiles.go index e6e14036b9..bda131c566 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -56,8 +56,8 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely return cobraext.FlagParsingError(err, cobraext.ProfileFromFlagName) } options := profile.Options{ - Name: newProfileName, - FromProfile: fromName, + Name: newProfileName, + // TODO: FromProfile: fromName, } err = profile.CreateProfile(options) if err != nil { diff --git a/internal/install/install.go b/internal/install/install.go index 4078ee9d69..b5e32c6b43 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -8,12 +8,10 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" ) @@ -34,11 +32,6 @@ func EnsureInstalled() error { return nil } - err = migrateIfNeeded(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "error migrating old install") - } - // Create the root .elastic-package path err = createElasticPackageDirectory(elasticPackagePath) if err != nil { @@ -86,47 +79,6 @@ func checkIfAlreadyInstalled(elasticPackagePath *locations.LocationManager) (boo return checkIfLatestVersionInstalled(elasticPackagePath) } -// checkIfUnmigrated checks to see if we have a pre-profile config that needs to be migrated -func migrateIfNeeded(elasticPackagePath *locations.LocationManager) error { - // use the snapshot.yml file as a canary to see if we have a pre-profile install - _, err := os.Stat(filepath.Join(elasticPackagePath.StackDir(), string(profile.SnapshotFile))) - if errors.Is(err, os.ErrNotExist) { - return nil - } - if err != nil { - return errors.Wrapf(err, "stat file failed (path: %s)", elasticPackagePath) - } - - profileName := fmt.Sprintf("default_migrated_%d", time.Now().Unix()) - logger.Warnf("Pre-profiles elastic-package detected. Existing config will be migrated to %s", profileName) - // Depending on how old the install is, not all the files will be available to migrate, - // So treat any errors from missing files as "soft" - oldFiles := []string{ - filepath.Join(elasticPackagePath.StackDir(), string(profile.SnapshotFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryDockerfileFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.KibanaConfigDefaultFile)), - filepath.Join(elasticPackagePath.StackDir(), string(profile.PackageRegistryConfigFile)), - } - - opts := profile.Options{ - PackagePath: elasticPackagePath.StackDir(), - Name: profileName, - } - err = profile.MigrateProfileFiles(opts, oldFiles) - if err != nil { - return errors.Wrap(err, "error migrating profile config") - } - - // delete the old files - for _, file := range oldFiles { - err = os.Remove(file) - if err != nil { - return errors.Wrapf(err, "error removing config file %s", file) - } - } - return nil -} - func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager) error { //remove unmanaged subdirectories err := os.RemoveAll(elasticPackagePath.TempDir()) // remove in case of potential upgrade diff --git a/internal/profile/_static/package_registry.yml b/internal/profile/_static/package-registry.yml similarity index 100% rename from internal/profile/_static/package_registry.yml rename to internal/profile/_static/package-registry.yml diff --git a/internal/profile/profile.go b/internal/profile/profile.go index a1589a8309..a78366aac2 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -2,6 +2,7 @@ package profile import ( "embed" + "errors" "fmt" "os" "path/filepath" @@ -9,11 +10,37 @@ import ( "text/template" "github.com/Masterminds/semver" + "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/go-resource" ) -// PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. -const PackageRegistryBaseImage = "docker.elastic.co/package-registry/distribution:snapshot" +const ( + // PackageProfileMetaFile is the filename of the profile metadata file + PackageProfileMetaFile = "profile.json" + + // SnapshotFile is the docker-compose snapshot.yml file name. + SnapshotFile = "snapshot.yml" + + // ElasticsearchConfigFile is the elasticsearch config file. + ElasticsearchConfigFile = "elasticsearch.yml" + + // KibanaConfigFile is the kibana config file. + KibanaConfigFile = "kibana.yml" + + // PackageRegistryConfigFile is the config file for the Elastic Package registry + PackageRegistryConfigFile = "package-registry.yml" + + // PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. + PackageRegistryBaseImage = "docker.elastic.co/package-registry/distribution:snapshot" + + // ElasticAgentEnvFile is the elastic agent environment variables file. + ElasticAgentEnvFile = "elastic-agent.env" + + // DefaultProfile is the name of the default profile. + DefaultProfile = "default" + + profileStackPath = "stack" +) //go:embed _static var static embed.FS @@ -24,6 +51,11 @@ var ( } staticSource = resource.NewSourceFS(static).WithTemplateFuncs(templateFuncs) profileResources = []resource.Resource{ + &resource.File{ + Provider: "profile-file", + Path: PackageProfileMetaFile, + Content: profileMetadataContent, + }, &resource.File{ Provider: "stack-file", Path: "Dockerfile.package-registry", @@ -31,37 +63,54 @@ var ( }, &resource.File{ Provider: "stack-file", - Path: "snapshot.yml", + Path: SnapshotFile, Content: staticSource.File("_static/docker-compose-stack.yml"), }, &resource.File{ Provider: "stack-file", - Path: "elasticsearch.yml", + Path: ElasticsearchConfigFile, Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), }, &resource.File{ Provider: "stack-file", - Path: "kibana.yml", + Path: KibanaConfigFile, Content: staticSource.Template("_static/kibana.yml.tmpl"), }, &resource.File{ Provider: "stack-file", - Path: "package_registry.yml", - Content: staticSource.File("_static/package_registry.yml"), + Path: PackageRegistryConfigFile, + Content: staticSource.File("_static/package-registry.yml"), }, &resource.File{ Provider: "stack-file", - Path: "elastic-agent.env", + Path: ElasticAgentEnvFile, Content: staticSource.Template("_static/elastic-agent.env.tmpl"), }, } ) -func CreateProfile(path string) error { +type Options struct { + PackagePath string + Name string + //FromProfile string + OverwriteExisting bool +} + +func CreateProfile(options Options) error { stackVersion := "8.1.0" // TODO: Parameterize this. + profileName := options.Name + if profileName == "" { + profileName = DefaultProfile + } + profileDir := filepath.Join(options.PackagePath, profileName) + stackDir := filepath.Join(options.PackagePath, profileName, profileStackPath) + resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ + "profile_name": profileName, + "profile_path": profileDir, + "registry_base_image": PackageRegistryBaseImage, "elasticsearch_version": stackVersion, @@ -69,8 +118,10 @@ func CreateProfile(path string) error { "agent_version": stackVersion, }) - stackDir := filepath.Join(path, "stack") os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("profile-file", &resource.FileProvider{ + Prefix: profileDir, + }) resourceManager.RegisterProvider("stack-file", &resource.FileProvider{ Prefix: stackDir, }) @@ -88,6 +139,131 @@ func CreateProfile(path string) error { return nil } +// Profile manages a a given user config profile +type Profile struct { + // ProfilePath is the absolute path to the profile + ProfilePath string + ProfileStackPath string + ProfileName string +} + +// ErrNotAProfile is returned in cases where we don't have a valid profile directory +var ErrNotAProfile = errors.New("not a profile") + +// FetchPath returns an absolute path to the given file +func (profile Profile) FetchPath(name string) string { + for _, r := range profileResources { + file, ok := r.(*resource.File) + if !ok { + continue + } + + if file.Path != name { + continue + } + + return filepath.Join(profile.ProfileStackPath, file.Path) + } + panic(fmt.Sprintf("%q profile file is not defined", name)) +} + +// ComposeEnvVars returns a list of environment variables that can be passed +// to docker-compose for the sake of filling out paths and names in the snapshot.yml file. +func (profile Profile) ComposeEnvVars() []string { + return []string{ + fmt.Sprintf("PROFILE_NAME=%s", profile.ProfileName), + fmt.Sprintf("STACK_PATH=%s", profile.ProfileStackPath), + } +} + +// DeleteProfile deletes a profile from the default elastic-package config dir +func DeleteProfile(profileName string) error { + if profileName == DefaultProfile { + return errors.New("cannot remove default profile") + } + + loc, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("error finding stack dir location: %w", err) + } + + pathToDelete := filepath.Join(loc.ProfileDir(), profileName) + return os.RemoveAll(pathToDelete) +} + +// FetchAllProfiles returns a list of profile values +func FetchAllProfiles(elasticPackagePath string) ([]Metadata, error) { + dirList, err := os.ReadDir(elasticPackagePath) + if err != nil { + return []Metadata{}, fmt.Errorf("error reading from directory %s: %w", elasticPackagePath, err) + } + + var profiles []Metadata + // TODO: this should read a profile.json file or something like that + for _, item := range dirList { + if !item.IsDir() { + continue + } + profile, err := loadProfile(elasticPackagePath, item.Name()) + if errors.Is(err, ErrNotAProfile) { + continue + } + if err != nil { + return profiles, fmt.Errorf("error loading profile %s: %w", item.Name(), err) + } + metadata, err := loadProfileMetadata(filepath.Join(profile.ProfilePath, PackageProfileMetaFile)) + if err != nil { + return profiles, fmt.Errorf("error reading profile metadata: %w", err) + } + profiles = append(profiles, metadata) + } + return profiles, nil +} + +// LoadProfile loads an existing profile from the default elastic-package config dir. +func LoadProfile(profileName string) (*Profile, error) { + loc, err := locations.NewLocationManager() + if err != nil { + return nil, fmt.Errorf("error finding stack dir location: %w", err) + } + + return loadProfile(loc.ProfileDir(), profileName) +} + +// loadProfile loads an existing profile +func loadProfile(elasticPackagePath string, profileName string) (*Profile, error) { + profilePath := filepath.Join(elasticPackagePath, profileName) + + isValid, err := isProfileDir(profilePath) + if err != nil { + return nil, fmt.Errorf("error checking profile %q: %w", profileName, err) + } + if !isValid { + return nil, ErrNotAProfile + } + + profile := Profile{ + ProfileName: profileName, + ProfilePath: profilePath, + ProfileStackPath: filepath.Join(profilePath, profileStackPath), + } + + return &profile, nil +} + +// isProfileDir checks to see if the given path points to a valid profile +func isProfileDir(path string) (bool, error) { + metaPath := filepath.Join(path, string(PackageProfileMetaFile)) + _, err := os.Stat(metaPath) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("error stat: %s: %w", metaPath, err) + } + return true, nil +} + func semverLessThan(a, b string) (bool, error) { sa, err := semver.NewVersion(a) if err != nil { diff --git a/internal/profile/profile_json.go b/internal/profile/profile_json.go new file mode 100644 index 0000000000..b640bec650 --- /dev/null +++ b/internal/profile/profile_json.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package profile + +import ( + "encoding/json" + "io" + "os" + "os/user" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/version" + "github.com/elastic/go-resource" +) + +// Metadata stores the data associated with a given profile +type Metadata struct { + Name string `json:"name"` + DateCreated time.Time `json:"date_created"` + User string `json:"user"` + Version string `json:"version"` + Path string `json:"path"` +} + +// profileMetadataContent generates the content of the profile.json file. +func profileMetadataContent(applyCtx resource.Context, w io.Writer) error { + currentUser, err := user.Current() + if err != nil { + return errors.Wrap(err, "error fetching current user") + } + + profileName, found := applyCtx.Fact("profile_name") + if !found { + return errors.New("unknown profile name") + } + + profilePath, found := applyCtx.Fact("profile_path") + if !found { + return errors.New("unknown profile path") + } + + profileData := Metadata{ + profileName, + time.Now(), + currentUser.Username, + version.CommitHash, + profilePath, + } + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + err = enc.Encode(profileData) + if err != nil { + return errors.Wrap(err, "error marshalling json") + } + + return nil +} + +func loadProfileMetadata(path string) (Metadata, error) { + d, err := os.ReadFile(path) + if err != nil { + return Metadata{}, errors.Wrap(err, "error reading metadata file") + } + + metadata := Metadata{} + err = json.Unmarshal(d, &metadata) + if err != nil { + return Metadata{}, errors.Wrapf(err, "error checking profile metadata file %q", path) + } + return metadata, nil +} diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index bf02c54f1c..248828ae10 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -7,7 +7,10 @@ import ( ) func TestCreateProfile(t *testing.T) { - err := CreateProfile(t.TempDir()) + options := Options{ + PackagePath: t.TempDir(), + } + err := CreateProfile(options) require.NoError(t, err) } diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index 5166c588cd..e793b34ec0 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -38,9 +38,7 @@ type kibanaConfiguration struct { // ShellInit method exposes environment variables that can be used for testing purposes. func ShellInit(elasticStackProfile *profile.Profile) (string, error) { - // Read Elasticsearch username and password from Kibana configuration file. - // FIXME read credentials from correct Kibana config file, not default - body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigDefaultFile)) + body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigFile)) if err != nil { return "", errors.Wrap(err, "error reading Kibana config file") } From 935a90d31036f6e173b80522f5eba702dfaa529f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 3 May 2022 17:16:43 +0200 Subject: [PATCH 03/35] Fix docker compose file --- internal/profile/_static/docker-compose-stack.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 41142a6774..66938f173a 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -10,7 +10,7 @@ services: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "ELASTIC_PASSWORD=changeme" volumes: - - "./elasticsearch.config.${STACK_VERSION_VARIANT}.yml:/usr/share/elasticsearch/config/elasticsearch.yml" + - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - "../../../stack/ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" - "../../../stack/service_tokens:/usr/share/elasticsearch/config/service_tokens" ports: @@ -34,7 +34,7 @@ services: retries: 600 interval: 1s volumes: - - "./kibana.config.${STACK_VERSION_VARIANT}.yml:/usr/share/kibana/config/kibana.yml" + - "./kibana.yml:/usr/share/kibana/config/kibana.yml" - "../../../stack/healthcheck.sh:/usr/share/kibana/healthcheck.sh" ports: - "127.0.0.1:5601:5601" @@ -103,7 +103,7 @@ services: retries: 180 interval: 1s hostname: docker-fleet-agent - env_file: "./elastic-agent.${STACK_VERSION_VARIANT}.env" + env_file: "./elastic-agent.env" volumes: - type: bind source: ../../../tmp/service_logs/ From 36af266b563b141f12ecd27c094eeabdb711afeb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 14:31:18 +0100 Subject: [PATCH 04/35] Fix profile creation --- cmd/profiles.go | 4 +-- internal/profile/profile.go | 66 ++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/cmd/profiles.go b/cmd/profiles.go index dea8ad5e53..957301ecb7 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -57,8 +57,8 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely return cobraext.FlagParsingError(err, cobraext.ProfileFromFlagName) } options := profile.Options{ - Name: newProfileName, - // TODO: FromProfile: fromName, + Name: newProfileName, + FromProfile: fromName, } err = profile.CreateProfile(options) if err != nil { diff --git a/internal/profile/profile.go b/internal/profile/profile.go index e654b68d85..b057ee40ef 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -90,14 +90,57 @@ var ( ) type Options struct { - PackagePath string - Name string - //FromProfile string + PackagePath string + Name string + FromProfile string OverwriteExisting bool } func CreateProfile(options Options) error { + if options.PackagePath == "" { + loc, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("error finding profile dir location: %w", err) + } + options.PackagePath = loc.ProfileDir() + } + + // If they're creating from Default, assume they want the actual default, and + // not whatever is currently inside default. + if from := options.FromProfile; from != "" && from != DefaultProfile { + return createProfileFrom(options) + } + + resources, err := initProfileResources(options) + if err != nil { + return err + } + + return createProfile(options, resources) +} + +func initProfileResources(options Options) ([]resource.Resource, error) { + profileName := options.Name + if profileName == "" { + profileName = DefaultProfile + } + profileDir := filepath.Join(options.PackagePath, profileName) + + resources := append([]resource.Resource{}, profileResources...) + + certResources, err := initTLSCertificates("profile-file", profileDir) + if err != nil { + return nil, fmt.Errorf("failed to create TLS files: %w", err) + } + resources = append(resources, certResources...) + + return resources, nil +} + +func createProfile(options Options, resources []resource.Resource) error { stackVersion := "8.1.0" // TODO: Parameterize this. + fmt.Printf("%+v\n", options) + fmt.Printf("%+v\n", resources) profileName := options.Name if profileName == "" { @@ -126,14 +169,6 @@ func CreateProfile(options Options) error { Prefix: stackDir, }) - resources := append([]resource.Resource{}, profileResources...) - - certResources, err := initTLSCertificates("profile-file", profileDir) - if err != nil { - return fmt.Errorf("failed to create TLS files: %w", err) - } - resources = append(resources, certResources...) - results, err := resourceManager.Apply(resources) if err != nil { var errors []string @@ -148,6 +183,15 @@ func CreateProfile(options Options) error { return nil } +func createProfileFrom(options Options) error { + from, err := LoadProfile(options.FromProfile) + if err != nil { + return fmt.Errorf("failed to load profile to copy %q: %w", options.FromProfile, err) + } + + return createProfile(options, from.resources) +} + // Profile manages a a given user config profile type Profile struct { // ProfilePath is the absolute path to the profile From 6096b2d81b11f822adb021d81d6cef2d254a09d7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 15:17:26 +0100 Subject: [PATCH 05/35] Allow to select the current profile --- cmd/profiles.go | 49 ++++++++++++----- cmd/stack.go | 37 +++++++++---- internal/install/application_configuration.go | 55 ++++++++++++++++++- .../install/application_configuration_yml.go | 33 ----------- internal/install/install.go | 13 +++-- internal/profile/profile.go | 2 - 6 files changed, 124 insertions(+), 65 deletions(-) delete mode 100644 internal/install/application_configuration_yml.go diff --git a/cmd/profiles.go b/cmd/profiles.go index 957301ecb7..8643300ed3 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -16,7 +16,7 @@ import ( "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/environment" + "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" ) @@ -26,9 +26,6 @@ const jsonFormat = "json" // tableFormat is the format for table output const tableFormat = "table" -// profileNameEnvVar is the name of the environment variable to set the default profile -var profileNameEnvVar = environment.WithElasticPackagePrefix("PROFILE") - func setupProfilesCommand() *cobraext.Command { profilesLongDescription := `Use this command to add, remove, and manage multiple config profiles. @@ -122,7 +119,40 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely } profileListCommand.Flags().String(cobraext.ProfileFormatFlagName, tableFormat, cobraext.ProfileFormatFlagDescription) - profileCommand.AddCommand(profileNewCommand, profileDeleteCommand, profileListCommand) + profileUseCommand := &cobra.Command{ + Use: "use", + Short: "Sets the profile to use when no other is specified", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("use requires an argument") + } + profileName := args[0] + + location, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("error fetching profile: %w", err) + } + + config, err := install.Configuration() + if err != nil { + return fmt.Errorf("failed to load current configuration: %w", err) + } + config.SetCurrentProfile(profileName) + + err = install.WriteConfigFile(location, config) + if err != nil { + return fmt.Errorf("failed to store configuration: %w", err) + } + return nil + }, + } + + profileCommand.AddCommand( + profileNewCommand, + profileDeleteCommand, + profileListCommand, + profileUseCommand, + ) return cobraext.NewCommand(profileCommand, cobraext.ContextGlobal) } @@ -175,15 +205,6 @@ func profileToList(profiles []profile.Metadata) [][]string { return profileList } -func lookupEnv() string { - env := os.Getenv(profileNameEnvVar) - if env == "" { - return profile.DefaultProfile - } - return env - -} - func availableProfilesAsAList() ([]string, error) { loc, err := locations.NewLocationManager() if err != nil { diff --git a/cmd/stack.go b/cmd/stack.go index c082606754..2632eefc89 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -78,9 +78,9 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profileName, err := getProfileNameFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } userProfile, err := profile.LoadProfile(profileName) @@ -129,9 +129,9 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Take down the Elastic stack") - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profileName, err := getProfileNameFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } userProfile, err := profile.LoadProfile(profileName) @@ -165,9 +165,9 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Update the Elastic stack") - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profileName, err := getProfileNameFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } profile, err := profile.LoadProfile(profileName) @@ -198,9 +198,9 @@ func setupStackCommand() *cobraext.Command { Use: "shellinit", Short: "Export environment variables", RunE: func(cmd *cobra.Command, args []string) error { - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profileName, err := getProfileNameFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } shellName, err := cmd.Flags().GetString(cobraext.ShellInitShellFlagName) @@ -241,9 +241,9 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackDumpOutputFlagName) } - profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + profileName, err := getProfileNameFlag(cmd) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return err } profile, err := profile.LoadProfile(profileName) @@ -287,7 +287,7 @@ func setupStackCommand() *cobraext.Command { Short: "Manage the Elastic stack", Long: stackLongDescription, } - cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", lookupEnv(), fmt.Sprintf(cobraext.ProfileFlagDescription, profileNameEnvVar)) + cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) cmd.AddCommand( upCommand, downCommand, @@ -367,6 +367,21 @@ func getParentInfo(ppid int) (types.ProcessInfo, error) { return parentInfo, nil } +func getProfileNameFlag(cmd *cobra.Command) (string, error) { + profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) + if err != nil { + return "", cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + } + if profileName == "" { + config, err := install.Configuration() + if err != nil { + return "", fmt.Errorf("cannot read configuration: %w", err) + } + profileName = config.CurrentProfile() + } + return profileName, nil +} + func getShellName(exe string) string { shell := filepath.Base(exe) // NOTE: remove .exe extension from executable names present in Windows diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index f54837ecf0..155c54423f 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -14,26 +14,61 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/profile" ) const ( stackVersion715 = "7.15.0-SNAPSHOT" stackVersion820 = "8.2.0-SNAPSHOT" + + elasticAgentImageName = "docker.elastic.co/beats/elastic-agent" + elasticAgentCompleteLegacyImageName = "docker.elastic.co/beats/elastic-agent-complete" + elasticAgentCompleteImageName = "docker.elastic.co/elastic-agent/elastic-agent-complete" + elasticsearchImageName = "docker.elastic.co/elasticsearch/elasticsearch" + kibanaImageName = "docker.elastic.co/kibana/kibana" + + applicationConfigurationYmlFile = "config.yml" ) var ( elasticAgentCompleteFirstSupportedVersion = semver.MustParse(stackVersion715) elasticAgentCompleteOwnNamespaceVersion = semver.MustParse(stackVersion820) + + // ProfileNameEnvVar is the name of the environment variable to set the default profile + ProfileNameEnvVar = environment.WithElasticPackagePrefix("PROFILE") ) +func DefaultConfiguration() *ApplicationConfiguration { + config := ApplicationConfiguration{} + config.c.Profile.Current = profile.DefaultProfile + + // Uncomment and use the commented definition of "stack" in case of emergency + // to define Docker image overrides (stack.image_ref_overrides). + // The following sample defines overrides for the Elastic stack ver. 7.13.0-SNAPSHOT. + // It's advised to use latest stable snapshots for the stack snapshot. + // + // config.c.Stack.ImageRefOverrides = map[string]ImageRefs{ + // "7.13.0-SNAPSHOT": ImageRefs{ + // ElasticAgent: elasticAgentImageName + `@sha256:76c294cf55654bc28dde72ce936032f34ad5f40c345f3df964924778b249e581`, + // Kibana: kibanaImageName + `@sha256:78ae3b1ca09efee242d2c77597dfab18670e984adb96c2407ec03fe07ceca4f6`, + // }, + // } + + return &config +} + // ApplicationConfiguration represents the configuration of the elastic-package. type ApplicationConfiguration struct { c configFile } type configFile struct { - Stack stack `yaml:"stack"` + Stack stack `yaml:"stack"` + Profile struct { + Current string `yaml:"current"` + } `yaml:"profile"` } type stack struct { @@ -79,6 +114,24 @@ func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs { return refs } +// CurrentProfile returns the current profile, or the default one if not set. +func (ac *ApplicationConfiguration) CurrentProfile() string { + fromEnv := os.Getenv(ProfileNameEnvVar) + if fromEnv != "" { + return fromEnv + } + current := ac.c.Profile.Current + if current == "" { + return profile.DefaultProfile + } + return current +} + +// SetCurrentProfile sets the current profile. +func (ac *ApplicationConfiguration) SetCurrentProfile(name string) { + ac.c.Profile.Current = name +} + // selectElasticAgentImageName function returns the appropriate image name for Elastic-Agent depending on the stack version. // This is mandatory as "elastic-agent-complete" is available since 7.15.0-SNAPSHOT. func selectElasticAgentImageName(version string) string { diff --git a/internal/install/application_configuration_yml.go b/internal/install/application_configuration_yml.go deleted file mode 100644 index dbc6811acf..0000000000 --- a/internal/install/application_configuration_yml.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package install - -const ( - elasticAgentImageName = "docker.elastic.co/beats/elastic-agent" - elasticAgentCompleteLegacyImageName = "docker.elastic.co/beats/elastic-agent-complete" - elasticAgentCompleteImageName = "docker.elastic.co/elastic-agent/elastic-agent-complete" - elasticsearchImageName = "docker.elastic.co/elasticsearch/elasticsearch" - kibanaImageName = "docker.elastic.co/kibana/kibana" -) - -const applicationConfigurationYmlFile = "config.yml" - -/* - -Uncomment and use the commented definition of "stack" in case of emergency to define Docker image overrides -(stack.image_ref_overrides). The following sample defines overrides for the Elastic stack ver. 7.13.0-SNAPSHOT. -It's advised to use latest stable snapshots for the stack snapshot. - -const applicationConfigurationYml = `stack: - image_ref_overrides: - 7.13.0-SNAPSHOT: - # Use stable image versions for Agent and Kibana - elastic-agent: ` + elasticAgentImageName + `@sha256:76c294cf55654bc28dde72ce936032f34ad5f40c345f3df964924778b249e581 - kibana: ` + kibanaImageName + `@sha256:78ae3b1ca09efee242d2c77597dfab18670e984adb96c2407ec03fe07ceca4f6` -*/ - -const applicationConfigurationYml = `stack: - image_ref_overrides: -` diff --git a/internal/install/install.go b/internal/install/install.go index d6867700f0..111799ccb5 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -10,6 +10,7 @@ import ( "path/filepath" "github.com/pkg/errors" + "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/profile" @@ -39,7 +40,7 @@ func EnsureInstalled() error { } // write the root config.yml file - err = writeConfigFile(elasticPackagePath) + err = WriteConfigFile(elasticPackagePath, DefaultConfiguration()) if err != nil { return errors.Wrap(err, "writing configuration file failed") } @@ -186,9 +187,13 @@ func writeDockerCustomAgentResources(elasticPackagePath *locations.LocationManag return nil } -func writeConfigFile(elasticPackagePath *locations.LocationManager) error { - var err error - err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), applicationConfigurationYml) +func WriteConfigFile(elasticPackagePath *locations.LocationManager, configuration *ApplicationConfiguration) error { + d, err := yaml.Marshal(configuration.c) + if err != nil { + return errors.Wrap(err, "failed to encode configuration") + } + + err = writeStaticResource(err, filepath.Join(elasticPackagePath.RootDir(), applicationConfigurationYmlFile), string(d)) if err != nil { return errors.Wrap(err, "writing static resource failed") } diff --git a/internal/profile/profile.go b/internal/profile/profile.go index b057ee40ef..a1be2e28be 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -139,8 +139,6 @@ func initProfileResources(options Options) ([]resource.Resource, error) { func createProfile(options Options, resources []resource.Resource) error { stackVersion := "8.1.0" // TODO: Parameterize this. - fmt.Printf("%+v\n", options) - fmt.Printf("%+v\n", resources) profileName := options.Name if profileName == "" { From 987028d76ed690fb7dc452971d6bea6956cc1c0b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 18:00:52 +0100 Subject: [PATCH 06/35] Move kibana healthcheck to stack --- internal/install/install.go | 6 ------ internal/install/static.go | 3 --- internal/profile/_static/docker-compose-stack.yml | 2 +- .../_static/kibana_healthcheck.sh | 0 internal/profile/profile.go | 14 +++++++++++++- internal/profile/profile_test.go | 5 ++++- 6 files changed, 18 insertions(+), 12 deletions(-) rename internal/{install => profile}/_static/kibana_healthcheck.sh (100%) diff --git a/internal/install/install.go b/internal/install/install.go index 111799ccb5..6b323b7568 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -115,12 +115,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) } - kibanaHealthcheckPath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh") - err = writeStaticResource(err, kibanaHealthcheckPath, kibanaHealthcheckSh) - if err != nil { - return errors.Wrapf(err, "copying healthcheck script failed (%s)", kibanaHealthcheckPath) - } - // Install GeoIP database ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip") err = os.MkdirAll(ingestGeoIPDir, 0755) diff --git a/internal/install/static.go b/internal/install/static.go index fb6b6d40c1..5a9c8e57d1 100644 --- a/internal/install/static.go +++ b/internal/install/static.go @@ -6,9 +6,6 @@ package install import _ "embed" -//go:embed _static/kibana_healthcheck.sh -var kibanaHealthcheckSh string - //go:embed _static/Dockerfile.terraform_deployer var terraformDeployerDockerfile string diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index ab5c0aacde..e5274fe615 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -40,7 +40,7 @@ services: volumes: - "./kibana.yml:/usr/share/kibana/config/kibana.yml" - "../certs/kibana:/usr/share/kibana/config/certs" - - "../../../stack/healthcheck.sh:/usr/share/kibana/healthcheck.sh" + - "./kibana_healthcheck.sh:/usr/share/kibana/healthcheck.sh" ports: - "127.0.0.1:5601:5601" diff --git a/internal/install/_static/kibana_healthcheck.sh b/internal/profile/_static/kibana_healthcheck.sh similarity index 100% rename from internal/install/_static/kibana_healthcheck.sh rename to internal/profile/_static/kibana_healthcheck.sh diff --git a/internal/profile/profile.go b/internal/profile/profile.go index a1be2e28be..f9edde443e 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package profile import ( @@ -27,11 +31,14 @@ const ( // KibanaConfigFile is the kibana config file. KibanaConfigFile = "kibana.yml" + // KibanaHealthcheckFile is the kibana healthcheck. + KibanaHealthcheckFile = "kibana_healthcheck.sh" + // PackageRegistryConfigFile is the config file for the Elastic Package registry PackageRegistryConfigFile = "package-registry.yml" // PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. - PackageRegistryBaseImage = "docker.elastic.co/package-registry/distribution:snapshot" + PackageRegistryBaseImage = "docker.elastic.co/package-registry/package-registry:v1.15.0" // ElasticAgentEnvFile is the elastic agent environment variables file. ElasticAgentEnvFile = "elastic-agent.env" @@ -76,6 +83,11 @@ var ( Path: KibanaConfigFile, Content: staticSource.Template("_static/kibana.yml.tmpl"), }, + &resource.File{ + Provider: "stack-file", + Path: KibanaHealthcheckFile, + Content: staticSource.File("_static/kibana_healthcheck.sh"), + }, &resource.File{ Provider: "stack-file", Path: PackageRegistryConfigFile, diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index 248828ae10..5fc03d5f44 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package profile import ( @@ -12,5 +16,4 @@ func TestCreateProfile(t *testing.T) { } err := CreateProfile(options) require.NoError(t, err) - } From 8a8f8dac4147d7e3350a2c6b87c48d442fcd1509 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 19:57:00 +0100 Subject: [PATCH 07/35] Remove code for service deployers from installation --- cmd/profiles.go | 15 ++- internal/install/application_configuration.go | 3 + internal/install/install.go | 123 ++++-------------- 3 files changed, 38 insertions(+), 103 deletions(-) diff --git a/cmd/profiles.go b/cmd/profiles.go index 8643300ed3..73f918e167 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -62,12 +62,16 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely return errors.Wrapf(err, "error creating profile %s from profile %s", newProfileName, fromName) } - fmt.Printf("Created profile %s from %s.\n", newProfileName, fromName) + if fromName == "" { + fmt.Printf("Created profile %s.\n", newProfileName) + } else { + fmt.Printf("Created profile %s from %s.\n", newProfileName, fromName) + } return nil }, } - profileNewCommand.Flags().String(cobraext.ProfileFromFlagName, "default", cobraext.ProfileFromFlagDescription) + profileNewCommand.Flags().String(cobraext.ProfileFromFlagName, "", cobraext.ProfileFromFlagDescription) profileDeleteCommand := &cobra.Command{ Use: "delete", @@ -104,7 +108,7 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely format, err := cmd.Flags().GetString(cobraext.ProfileFormatFlagName) if err != nil { - return cobraext.FlagParsingError(err, cobraext.ProfileFromFlagName) + return cobraext.FlagParsingError(err, cobraext.ProfileFormatFlagName) } switch format { @@ -128,6 +132,11 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely } profileName := args[0] + _, err := profile.LoadProfile(profileName) + if err != nil { + return fmt.Errorf("cannot use profile %q: %v", profileName, err) + } + location, err := locations.NewLocationManager() if err != nil { return fmt.Errorf("error fetching profile: %w", err) diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index 155c54423f..e7ccd02773 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -161,6 +161,9 @@ func Configuration() (*ApplicationConfiguration, error) { } cfg, err := os.ReadFile(filepath.Join(configPath.RootDir(), applicationConfigurationYmlFile)) + if errors.Is(err, os.ErrNotExist) { + return DefaultConfiguration(), nil + } if err != nil { return nil, errors.Wrap(err, "can't read configuration file") } diff --git a/internal/install/install.go b/internal/install/install.go index 6b323b7568..2175b32d26 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -13,12 +13,11 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" - "github.com/elastic/elastic-package/internal/profile" ) const versionFilename = "version" -// EnsureInstalled method installs once static resources for the testing Docker stack. +// EnsureInstalled method installs once the required configuration files. func EnsureInstalled() error { elasticPackagePath, err := locations.NewLocationManager() if err != nil { @@ -30,42 +29,34 @@ func EnsureInstalled() error { return errors.Wrap(err, "failed to check if there is an elastic-package installation") } if installed { - return nil + latestInstalled, err := checkIfLatestVersionInstalled(elasticPackagePath) + if err != nil { + return errors.Wrap(err, "failed to check if latest version is installed") + } + if latestInstalled { + return nil + } + return migrateConfigDirectory(elasticPackagePath) } - // Create the root .elastic-package path + // Create the root .elastic-package path. err = createElasticPackageDirectory(elasticPackagePath) if err != nil { return errors.Wrap(err, "creating elastic package directory failed") } - // write the root config.yml file + // Write the root config.yml file. err = WriteConfigFile(elasticPackagePath, DefaultConfiguration()) if err != nil { return errors.Wrap(err, "writing configuration file failed") } - // write root version file + // Write root version file. err = writeVersionFile(elasticPackagePath) if err != nil { return errors.Wrap(err, "writing version file failed") } - err = writeStackResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing stack resources failed") - } - - err = writeTerraformDeployerResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing Terraform deployer resources failed") - } - - err = writeDockerCustomAgentResources(elasticPackagePath) - if err != nil { - return errors.Wrap(err, "writing Terraform deployer resources failed") - } - if err := createServiceLogsDir(elasticPackagePath); err != nil { return errors.Wrap(err, "creating service logs directory failed") } @@ -75,14 +66,14 @@ func EnsureInstalled() error { } func checkIfAlreadyInstalled(elasticPackagePath *locations.LocationManager) (bool, error) { - _, err := os.Stat(elasticPackagePath.StackDir()) + _, err := os.Stat(elasticPackagePath.RootDir()) if errors.Is(err, os.ErrNotExist) { return false, nil } if err != nil { return false, errors.Wrapf(err, "stat file failed (path: %s)", elasticPackagePath) } - return checkIfLatestVersionInstalled(elasticPackagePath) + return true, nil } func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager) error { @@ -104,83 +95,6 @@ func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager return nil } -func writeStackResources(elasticPackagePath *locations.LocationManager) error { - err := os.MkdirAll(elasticPackagePath.PackagesDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) - } - - err = os.MkdirAll(elasticPackagePath.ProfileDir(), 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) - } - - // Install GeoIP database - ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip") - err = os.MkdirAll(ingestGeoIPDir, 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIPDir) - } - - geoIpAsnMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-ASN.mmdb") - err = writeStaticResource(err, geoIpAsnMmdbPath, geoIpAsnMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP ASN database failed (%s)", geoIpAsnMmdbPath) - } - - geoIpCityMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-City.mmdb") - err = writeStaticResource(err, geoIpCityMmdbPath, geoIpCityMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP city database failed (%s)", geoIpCityMmdbPath) - } - - geoIpCountryMmdbPath := filepath.Join(ingestGeoIPDir, "GeoLite2-Country.mmdb") - err = writeStaticResource(err, geoIpCountryMmdbPath, geoIpCountryMmdb) - if err != nil { - return errors.Wrapf(err, "copying GeoIP country database failed (%s)", geoIpCountryMmdbPath) - } - - serviceTokensPath := filepath.Join(elasticPackagePath.StackDir(), "service_tokens") - err = writeStaticResource(err, serviceTokensPath, serviceTokens) - if err != nil { - return errors.Wrapf(err, "copying service_tokens failed (%s)", serviceTokensPath) - } - - options := profile.Options{ - PackagePath: elasticPackagePath.ProfileDir(), - Name: profile.DefaultProfile, - OverwriteExisting: false, - } - return profile.CreateProfile(options) -} - -func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error { - terraformDeployer := elasticPackagePath.TerraformDeployerDir() - err := os.MkdirAll(terraformDeployer, 0755) - if err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", terraformDeployer) - } - - err = writeStaticResource(err, elasticPackagePath.TerraformDeployerYml(), terraformDeployerYml) - err = writeStaticResource(err, filepath.Join(terraformDeployer, "Dockerfile"), terraformDeployerDockerfile) - err = writeStaticResource(err, filepath.Join(terraformDeployer, "run.sh"), terraformDeployerRun) - if err != nil { - return errors.Wrap(err, "writing static resource failed") - } - return nil -} - -func writeDockerCustomAgentResources(elasticPackagePath *locations.LocationManager) error { - dir := elasticPackagePath.DockerCustomAgentDeployerDir() - if err := os.MkdirAll(dir, 0755); err != nil { - return errors.Wrapf(err, "creating directory failed (path: %s)", dir) - } - if err := writeStaticResource(nil, elasticPackagePath.DockerCustomAgentDeployerYml(), dockerCustomAgentBaseYml); err != nil { - return errors.Wrap(err, "writing static resource failed") - } - return nil -} - func WriteConfigFile(elasticPackagePath *locations.LocationManager, configuration *ApplicationConfiguration) error { d, err := yaml.Marshal(configuration.c) if err != nil { @@ -206,6 +120,15 @@ func writeStaticResource(err error, path, content string) error { return nil } +func migrateConfigDirectory(elasticPackagePath *locations.LocationManager) error { + err := writeVersionFile(elasticPackagePath) + if err != nil { + return errors.Wrap(err, "writing version file failed") + } + + return nil +} + func createServiceLogsDir(elasticPackagePath *locations.LocationManager) error { dirPath := elasticPackagePath.ServiceLogDir() err := os.MkdirAll(dirPath, 0755) From 5fefa3c9ec340c7ec9809155e8b433c3f1604954 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 20:38:31 +0100 Subject: [PATCH 08/35] Move service tokens and geoip to profile --- cmd/profiles.go | 4 +++ cmd/stack.go | 2 +- internal/install/install.go | 12 ++++++++ internal/install/static.go | 12 -------- .../_static/GeoLite2-ASN.mmdb | Bin .../_static/GeoLite2-City.mmdb | Bin .../_static/GeoLite2-Country.mmdb | Bin .../profile/_static/docker-compose-stack.yml | 4 +-- .../_static/service_tokens | 0 internal/profile/profile.go | 26 ++++++++++++++++++ 10 files changed, 45 insertions(+), 15 deletions(-) rename internal/{install => profile}/_static/GeoLite2-ASN.mmdb (100%) rename internal/{install => profile}/_static/GeoLite2-City.mmdb (100%) rename internal/{install => profile}/_static/GeoLite2-Country.mmdb (100%) rename internal/{install => profile}/_static/service_tokens (100%) diff --git a/cmd/profiles.go b/cmd/profiles.go index 73f918e167..82b4174b6f 100644 --- a/cmd/profiles.go +++ b/cmd/profiles.go @@ -105,6 +105,10 @@ User profiles are not overwritten on upgrade of elastic-stack, and can be freely if err != nil { return errors.Wrap(err, "error listing all profiles") } + if len(profileList) == 0 { + fmt.Println("There are no profiles yet.") + return nil + } format, err := cmd.Flags().GetString(cobraext.ProfileFormatFlagName) if err != nil { diff --git a/cmd/stack.go b/cmd/stack.go index 2632eefc89..94bc36c520 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -89,7 +89,7 @@ func setupStackCommand() *cobraext.Command { if err != nil { return errors.Wrap(err, "error listing known profiles") } - return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, pList) + return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, strings.Join(pList, ", ")) } if err != nil { return errors.Wrap(err, "error loading profile") diff --git a/internal/install/install.go b/internal/install/install.go index 2175b32d26..814ba072b4 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -13,6 +13,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/profile" ) const versionFilename = "version" @@ -57,6 +58,17 @@ func EnsureInstalled() error { return errors.Wrap(err, "writing version file failed") } + // Create initial profile: + options := profile.Options{ + PackagePath: elasticPackagePath.ProfileDir(), + Name: profile.DefaultProfile, + OverwriteExisting: false, + } + err = profile.CreateProfile(options) + if err != nil { + return errors.Wrap(err, "creation of initial profile failed") + } + if err := createServiceLogsDir(elasticPackagePath); err != nil { return errors.Wrap(err, "creating service logs directory failed") } diff --git a/internal/install/static.go b/internal/install/static.go index 5a9c8e57d1..db8a32556b 100644 --- a/internal/install/static.go +++ b/internal/install/static.go @@ -15,17 +15,5 @@ var terraformDeployerYml string //go:embed _static/terraform_deployer_run.sh var terraformDeployerRun string -//go:embed _static/GeoLite2-ASN.mmdb -var geoIpAsnMmdb string - -//go:embed _static/GeoLite2-City.mmdb -var geoIpCityMmdb string - -//go:embed _static/GeoLite2-Country.mmdb -var geoIpCountryMmdb string - -//go:embed _static/service_tokens -var serviceTokens string - //go:embed _static/docker-custom-agent-base.yml var dockerCustomAgentBaseYml string diff --git a/internal/install/_static/GeoLite2-ASN.mmdb b/internal/profile/_static/GeoLite2-ASN.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-ASN.mmdb rename to internal/profile/_static/GeoLite2-ASN.mmdb diff --git a/internal/install/_static/GeoLite2-City.mmdb b/internal/profile/_static/GeoLite2-City.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-City.mmdb rename to internal/profile/_static/GeoLite2-City.mmdb diff --git a/internal/install/_static/GeoLite2-Country.mmdb b/internal/profile/_static/GeoLite2-Country.mmdb similarity index 100% rename from internal/install/_static/GeoLite2-Country.mmdb rename to internal/profile/_static/GeoLite2-Country.mmdb diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index e5274fe615..71dd2184b4 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -12,8 +12,8 @@ services: volumes: - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - "../certs/elasticsearch:/usr/share/elasticsearch/config/certs" - - "../../../stack/ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" - - "../../../stack/service_tokens:/usr/share/elasticsearch/config/service_tokens" + - "./ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" + - "./service_tokens:/usr/share/elasticsearch/config/service_tokens" ports: - "127.0.0.1:9200:9200" diff --git a/internal/install/_static/service_tokens b/internal/profile/_static/service_tokens similarity index 100% rename from internal/install/_static/service_tokens rename to internal/profile/_static/service_tokens diff --git a/internal/profile/profile.go b/internal/profile/profile.go index f9edde443e..4eb3732808 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -78,6 +78,29 @@ var ( Path: ElasticsearchConfigFile, Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), }, + &resource.File{ + Provider: "stack-file", + Path: "service_tokens", + Content: staticSource.File("_static/service_tokens"), + }, + &resource.File{ + Provider: "stack-file", + Path: "ingest-geoip/GeoLite2-ASN.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-ASN.mmdb"), + }, + &resource.File{ + Provider: "stack-file", + Path: "ingest-geoip/GeoLite2-City.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-City.mmdb"), + }, + &resource.File{ + Provider: "stack-file", + Path: "ingest-geoip/GeoLite2-Country.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-Country.mmdb"), + }, &resource.File{ Provider: "stack-file", Path: KibanaConfigFile, @@ -259,6 +282,9 @@ func DeleteProfile(profileName string) error { // FetchAllProfiles returns a list of profile values func FetchAllProfiles(elasticPackagePath string) ([]Metadata, error) { dirList, err := os.ReadDir(elasticPackagePath) + if errors.Is(err, os.ErrNotExist) { + return []Metadata{}, nil + } if err != nil { return []Metadata{}, fmt.Errorf("error reading from directory %s: %w", elasticPackagePath, err) } From 6a89b90119845f9c742e90675eb05737b4de8fee Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Nov 2022 21:04:44 +0100 Subject: [PATCH 09/35] Fix tests --- internal/profile/certs_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/profile/certs_test.go b/internal/profile/certs_test.go index 5d8a9154e9..95a78de66c 100644 --- a/internal/profile/certs_test.go +++ b/internal/profile/certs_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/elastic/go-resource" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -20,11 +21,15 @@ func TestTLSCertsInitialization(t *testing.T) { assert.Error(t, verifyTLSCertificates(caCertFile, caCertFile, caKeyFile, "")) - configMap := make(map[configFile]*simpleFile) - err := initTLSCertificates(profilePath, configMap) + providerName := "test-file" + resources, err := initTLSCertificates(providerName, profilePath) require.NoError(t, err) - err = writeConfigFiles(configMap) + resourceManager := resource.NewManager() + resourceManager.RegisterProvider(providerName, &resource.FileProvider{ + Prefix: profilePath, + }) + _, err = resourceManager.Apply(resources) require.NoError(t, err) assert.NoError(t, verifyTLSCertificates(caCertFile, caCertFile, caKeyFile, "")) @@ -49,12 +54,11 @@ func TestTLSCertsInitialization(t *testing.T) { assert.Error(t, verifyTLSCertificates(caCertFile, serviceCertFile, serviceKeyFile, service)) // Check it is created again and is validated by the same CA. - configMap := make(map[configFile]*simpleFile) - err := initTLSCertificates(profilePath, configMap) + resources, err := initTLSCertificates(providerName, profilePath) require.NoError(t, err) - - err = writeConfigFiles(configMap) + _, err = resourceManager.Apply(resources) require.NoError(t, err) + assert.NoError(t, verifyTLSCertificates(caCertFile, serviceCertFile, serviceKeyFile, service)) }) } From ae240221319c6359a26941daaad22dae5c8f47da Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 30 Nov 2022 17:28:42 +0100 Subject: [PATCH 10/35] Unify code to load profiles in stack subcommands --- cmd/stack.go | 76 ++++++++++++++++++---------------------------------- 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/cmd/stack.go b/cmd/stack.go index 94bc36c520..b4c1044a76 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -78,23 +78,11 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - profileName, err := getProfileNameFlag(cmd) + userProfile, err := getProfileFlag(cmd) if err != nil { return err } - userProfile, err := profile.LoadProfile(profileName) - if errors.Is(err, profile.ErrNotAProfile) { - pList, err := availableProfilesAsAList() - if err != nil { - return errors.Wrap(err, "error listing known profiles") - } - return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, strings.Join(pList, ", ")) - } - if err != nil { - return errors.Wrap(err, "error loading profile") - } - // Print information before starting the stack, for cases where // this is executed in the foreground, without daemon mode. cmd.Printf("Using profile %s.\n", userProfile.ProfilePath) @@ -129,24 +117,11 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Take down the Elastic stack") - profileName, err := getProfileNameFlag(cmd) + userProfile, err := getProfileFlag(cmd) if err != nil { return err } - userProfile, err := profile.LoadProfile(profileName) - if errors.Is(err, profile.ErrNotAProfile) { - pList, err := availableProfilesAsAList() - if err != nil { - return errors.Wrap(err, "error listing known profiles") - } - return fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, pList) - } - - if err != nil { - return errors.Wrap(err, "error loading profile") - } - err = stack.TearDown(stack.Options{ Profile: userProfile, }) @@ -165,16 +140,11 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Update the Elastic stack") - profileName, err := getProfileNameFlag(cmd) + profile, err := getProfileFlag(cmd) if err != nil { return err } - profile, err := profile.LoadProfile(profileName) - if err != nil { - return errors.Wrap(err, "error loading profile") - } - stackVersion, err := cmd.Flags().GetString(cobraext.StackVersionFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) @@ -198,11 +168,6 @@ func setupStackCommand() *cobraext.Command { Use: "shellinit", Short: "Export environment variables", RunE: func(cmd *cobra.Command, args []string) error { - profileName, err := getProfileNameFlag(cmd) - if err != nil { - return err - } - shellName, err := cmd.Flags().GetString(cobraext.ShellInitShellFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.ShellInitShellFlagName) @@ -216,9 +181,9 @@ func setupStackCommand() *cobraext.Command { fmt.Fprintf(cmd.OutOrStderr(), "Detected shell: %s\n", shellName) } - profile, err := profile.LoadProfile(profileName) + profile, err := getProfileFlag(cmd) if err != nil { - return errors.Wrap(err, "error loading profile") + return err } shellCode, err := stack.ShellInit(profile, shellName) @@ -241,16 +206,11 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackDumpOutputFlagName) } - profileName, err := getProfileNameFlag(cmd) + profile, err := getProfileFlag(cmd) if err != nil { return err } - profile, err := profile.LoadProfile(profileName) - if err != nil { - return errors.Wrap(err, "error loading profile") - } - target, err := stack.Dump(stack.DumpOptions{ Output: output, Profile: profile, @@ -367,19 +327,35 @@ func getParentInfo(ppid int) (types.ProcessInfo, error) { return parentInfo, nil } -func getProfileNameFlag(cmd *cobra.Command) (string, error) { +func getProfileFlag(cmd *cobra.Command) (*profile.Profile, error) { profileName, err := cmd.Flags().GetString(cobraext.ProfileFlagName) if err != nil { - return "", cobraext.FlagParsingError(err, cobraext.ProfileFlagName) + return nil, cobraext.FlagParsingError(err, cobraext.ProfileFlagName) } if profileName == "" { config, err := install.Configuration() if err != nil { - return "", fmt.Errorf("cannot read configuration: %w", err) + return nil, fmt.Errorf("cannot read configuration: %w", err) } profileName = config.CurrentProfile() } - return profileName, nil + + userProfile, err := profile.LoadProfile(profileName) + if errors.Is(err, profile.ErrNotAProfile) { + list, err := availableProfilesAsAList() + if err != nil { + return nil, errors.Wrap(err, "error listing known profiles") + } + if len(list) == 0 { + return nil, fmt.Errorf("%s is not a valid profile", profileName) + } + return nil, fmt.Errorf("%s is not a valid profile, known profiles are: %s", profileName, strings.Join(list, ", ")) + } + if err != nil { + return nil, errors.Wrap(err, "error loading profile") + } + + return userProfile, nil } func getShellName(exe string) string { From 3d73ed71007c294cc51c3c6600ad9b066a89871a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 30 Nov 2022 18:01:59 +0100 Subject: [PATCH 11/35] Move stack files to stack package --- internal/profile/profile.go | 226 ++++-------------- .../_static/Dockerfile.package-registry.tmpl | 0 .../_static/GeoLite2-ASN.mmdb | Bin .../_static/GeoLite2-City.mmdb | Bin .../_static/GeoLite2-Country.mmdb | Bin .../_static/docker-compose-stack.yml | 14 +- .../_static/elastic-agent.env.tmpl | 0 .../_static/elasticsearch.yml.tmpl | 0 .../_static/kibana.yml.tmpl | 0 .../_static/kibana_healthcheck.sh | 0 .../_static/package-registry.yml | 0 .../{profile => stack}/_static/service_tokens | 0 internal/stack/boot.go | 5 + internal/{profile => stack}/certs.go | 2 +- internal/{profile => stack}/certs_test.go | 2 +- internal/stack/compose.go | 9 +- internal/stack/dump.go | 2 +- internal/stack/initconfig.go | 6 +- internal/stack/resources.go | 153 ++++++++++++ internal/stack/update.go | 8 +- 20 files changed, 221 insertions(+), 206 deletions(-) rename internal/{profile => stack}/_static/Dockerfile.package-registry.tmpl (100%) rename internal/{profile => stack}/_static/GeoLite2-ASN.mmdb (100%) rename internal/{profile => stack}/_static/GeoLite2-City.mmdb (100%) rename internal/{profile => stack}/_static/GeoLite2-Country.mmdb (100%) rename internal/{profile => stack}/_static/docker-compose-stack.yml (90%) rename internal/{profile => stack}/_static/elastic-agent.env.tmpl (100%) rename internal/{profile => stack}/_static/elasticsearch.yml.tmpl (100%) rename internal/{profile => stack}/_static/kibana.yml.tmpl (100%) rename internal/{profile => stack}/_static/kibana_healthcheck.sh (100%) rename internal/{profile => stack}/_static/package-registry.yml (100%) rename internal/{profile => stack}/_static/service_tokens (100%) rename internal/{profile => stack}/certs.go (99%) rename internal/{profile => stack}/certs_test.go (99%) create mode 100644 internal/stack/resources.go diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 4eb3732808..57220e74ad 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -5,16 +5,14 @@ package profile import ( - "embed" "errors" "fmt" "os" "path/filepath" "strings" - "text/template" - "github.com/Masterminds/semver" "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/files" "github.com/elastic/go-resource" ) @@ -22,104 +20,15 @@ const ( // PackageProfileMetaFile is the filename of the profile metadata file PackageProfileMetaFile = "profile.json" - // SnapshotFile is the docker-compose snapshot.yml file name. - SnapshotFile = "snapshot.yml" - - // ElasticsearchConfigFile is the elasticsearch config file. - ElasticsearchConfigFile = "elasticsearch.yml" - - // KibanaConfigFile is the kibana config file. - KibanaConfigFile = "kibana.yml" - - // KibanaHealthcheckFile is the kibana healthcheck. - KibanaHealthcheckFile = "kibana_healthcheck.sh" - - // PackageRegistryConfigFile is the config file for the Elastic Package registry - PackageRegistryConfigFile = "package-registry.yml" - - // PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. - PackageRegistryBaseImage = "docker.elastic.co/package-registry/package-registry:v1.15.0" - - // ElasticAgentEnvFile is the elastic agent environment variables file. - ElasticAgentEnvFile = "elastic-agent.env" - // DefaultProfile is the name of the default profile. DefaultProfile = "default" - - profileStackPath = "stack" ) -//go:embed _static -var static embed.FS - var ( - templateFuncs = template.FuncMap{ - "semverLessThan": semverLessThan, - } - staticSource = resource.NewSourceFS(static).WithTemplateFuncs(templateFuncs) profileResources = []resource.Resource{ &resource.File{ - Provider: "profile-file", - Path: PackageProfileMetaFile, - Content: profileMetadataContent, - }, - &resource.File{ - Provider: "stack-file", - Path: "Dockerfile.package-registry", - Content: staticSource.Template("_static/Dockerfile.package-registry.tmpl"), - }, - &resource.File{ - Provider: "stack-file", - Path: SnapshotFile, - Content: staticSource.File("_static/docker-compose-stack.yml"), - }, - &resource.File{ - Provider: "stack-file", - Path: ElasticsearchConfigFile, - Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), - }, - &resource.File{ - Provider: "stack-file", - Path: "service_tokens", - Content: staticSource.File("_static/service_tokens"), - }, - &resource.File{ - Provider: "stack-file", - Path: "ingest-geoip/GeoLite2-ASN.mmdb", - CreateParent: true, - Content: staticSource.File("_static/GeoLite2-ASN.mmdb"), - }, - &resource.File{ - Provider: "stack-file", - Path: "ingest-geoip/GeoLite2-City.mmdb", - CreateParent: true, - Content: staticSource.File("_static/GeoLite2-City.mmdb"), - }, - &resource.File{ - Provider: "stack-file", - Path: "ingest-geoip/GeoLite2-Country.mmdb", - CreateParent: true, - Content: staticSource.File("_static/GeoLite2-Country.mmdb"), - }, - &resource.File{ - Provider: "stack-file", - Path: KibanaConfigFile, - Content: staticSource.Template("_static/kibana.yml.tmpl"), - }, - &resource.File{ - Provider: "stack-file", - Path: KibanaHealthcheckFile, - Content: staticSource.File("_static/kibana_healthcheck.sh"), - }, - &resource.File{ - Provider: "stack-file", - Path: PackageRegistryConfigFile, - Content: staticSource.File("_static/package-registry.yml"), - }, - &resource.File{ - Provider: "stack-file", - Path: ElasticAgentEnvFile, - Content: staticSource.Template("_static/elastic-agent.env.tmpl"), + Path: PackageProfileMetaFile, + Content: profileMetadataContent, }, } ) @@ -140,67 +49,42 @@ func CreateProfile(options Options) error { options.PackagePath = loc.ProfileDir() } - // If they're creating from Default, assume they want the actual default, and - // not whatever is currently inside default. - if from := options.FromProfile; from != "" && from != DefaultProfile { - return createProfileFrom(options) - } - - resources, err := initProfileResources(options) - if err != nil { - return err + if options.Name == "" { + options.Name = DefaultProfile } - return createProfile(options, resources) -} - -func initProfileResources(options Options) ([]resource.Resource, error) { - profileName := options.Name - if profileName == "" { - profileName = DefaultProfile + if !options.OverwriteExisting { + _, err := LoadProfile(options.Name) + if err == nil { + return fmt.Errorf("profile %q already exists", options.Name) + } + if err != nil && err != ErrNotAProfile { + return fmt.Errorf("failed to check if profile %q exists: %w", options.Name, err) + } } - profileDir := filepath.Join(options.PackagePath, profileName) - - resources := append([]resource.Resource{}, profileResources...) - certResources, err := initTLSCertificates("profile-file", profileDir) - if err != nil { - return nil, fmt.Errorf("failed to create TLS files: %w", err) + // If they're creating from Default, assume they want the actual default, and + // not whatever is currently inside default. + if from := options.FromProfile; from != "" && from != DefaultProfile { + return createProfileFrom(options) } - resources = append(resources, certResources...) - return resources, nil + return createProfile(options, profileResources) } func createProfile(options Options, resources []resource.Resource) error { - stackVersion := "8.1.0" // TODO: Parameterize this. - - profileName := options.Name - if profileName == "" { - profileName = DefaultProfile - } - profileDir := filepath.Join(options.PackagePath, profileName) - stackDir := filepath.Join(options.PackagePath, profileName, profileStackPath) + profileDir := filepath.Join(options.PackagePath, options.Name) resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ - "profile_name": profileName, + "profile_name": options.Name, "profile_path": profileDir, - - "registry_base_image": PackageRegistryBaseImage, - - "elasticsearch_version": stackVersion, - "kibana_version": stackVersion, - "agent_version": stackVersion, }) - os.MkdirAll(stackDir, 0755) - resourceManager.RegisterProvider("profile-file", &resource.FileProvider{ + os.MkdirAll(profileDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ Prefix: profileDir, }) - resourceManager.RegisterProvider("stack-file", &resource.FileProvider{ - Prefix: stackDir, - }) results, err := resourceManager.Apply(resources) if err != nil { @@ -222,45 +106,38 @@ func createProfileFrom(options Options) error { return fmt.Errorf("failed to load profile to copy %q: %w", options.FromProfile, err) } - return createProfile(options, from.resources) + profileDir := filepath.Join(options.PackagePath, options.Name) + err = files.CopyAll(from.ProfilePath, profileDir) + if err != nil { + return fmt.Errorf("failed to copy files from profile %q to %q", options.FromProfile, options.Name) + } + + overwriteOptions := options + overwriteOptions.OverwriteExisting = true + return createProfile(overwriteOptions, profileResources) } // Profile manages a a given user config profile type Profile struct { // ProfilePath is the absolute path to the profile - ProfilePath string - ProfileStackPath string - ProfileName string + ProfilePath string + ProfileName string +} - resources []resource.Resource +// Path returns an absolute path to the given file +func (profile Profile) Path(names ...string) string { + elems := append([]string{profile.ProfilePath}, names...) + return filepath.Join(elems...) } // ErrNotAProfile is returned in cases where we don't have a valid profile directory var ErrNotAProfile = errors.New("not a profile") -// FetchPath returns an absolute path to the given file -func (profile Profile) FetchPath(name string) string { - for _, r := range profile.resources { - file, ok := r.(*resource.File) - if !ok { - continue - } - - if file.Path != name { - continue - } - - return filepath.Join(profile.ProfileStackPath, file.Path) - } - panic(fmt.Sprintf("%q profile file is not defined", name)) -} - // ComposeEnvVars returns a list of environment variables that can be passed // to docker-compose for the sake of filling out paths and names in the snapshot.yml file. func (profile Profile) ComposeEnvVars() []string { return []string{ fmt.Sprintf("PROFILE_NAME=%s", profile.ProfileName), - fmt.Sprintf("STACK_PATH=%s", profile.ProfileStackPath), } } @@ -333,19 +210,9 @@ func loadProfile(elasticPackagePath string, profileName string) (*Profile, error return nil, ErrNotAProfile } - resources := append([]resource.Resource{}, profileResources...) - - certResources, err := initTLSCertificates("profile-file", profilePath) - if err != nil { - return nil, fmt.Errorf("failed to create TLS files: %w", err) - } - resources = append(resources, certResources...) - profile := Profile{ - ProfileName: profileName, - ProfilePath: profilePath, - ProfileStackPath: filepath.Join(profilePath, profileStackPath), - resources: resources, + ProfileName: profileName, + ProfilePath: profilePath, } return &profile, nil @@ -363,16 +230,3 @@ func isProfileDir(path string) (bool, error) { } return true, nil } - -func semverLessThan(a, b string) (bool, error) { - sa, err := semver.NewVersion(a) - if err != nil { - return false, err - } - sb, err := semver.NewVersion(b) - if err != nil { - return false, err - } - - return sa.LessThan(sb), nil -} diff --git a/internal/profile/_static/Dockerfile.package-registry.tmpl b/internal/stack/_static/Dockerfile.package-registry.tmpl similarity index 100% rename from internal/profile/_static/Dockerfile.package-registry.tmpl rename to internal/stack/_static/Dockerfile.package-registry.tmpl diff --git a/internal/profile/_static/GeoLite2-ASN.mmdb b/internal/stack/_static/GeoLite2-ASN.mmdb similarity index 100% rename from internal/profile/_static/GeoLite2-ASN.mmdb rename to internal/stack/_static/GeoLite2-ASN.mmdb diff --git a/internal/profile/_static/GeoLite2-City.mmdb b/internal/stack/_static/GeoLite2-City.mmdb similarity index 100% rename from internal/profile/_static/GeoLite2-City.mmdb rename to internal/stack/_static/GeoLite2-City.mmdb diff --git a/internal/profile/_static/GeoLite2-Country.mmdb b/internal/stack/_static/GeoLite2-Country.mmdb similarity index 100% rename from internal/profile/_static/GeoLite2-Country.mmdb rename to internal/stack/_static/GeoLite2-Country.mmdb diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/stack/_static/docker-compose-stack.yml similarity index 90% rename from internal/profile/_static/docker-compose-stack.yml rename to internal/stack/_static/docker-compose-stack.yml index 71dd2184b4..1704029ea1 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/stack/_static/docker-compose-stack.yml @@ -11,7 +11,7 @@ services: - "ELASTIC_PASSWORD=changeme" volumes: - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - - "../certs/elasticsearch:/usr/share/elasticsearch/config/certs" + - "./certs/elasticsearch:/usr/share/elasticsearch/config/certs" - "./ingest-geoip:/usr/share/elasticsearch/config/ingest-geoip" - "./service_tokens:/usr/share/elasticsearch/config/service_tokens" ports: @@ -39,7 +39,7 @@ services: - "NODE_EXTRA_CA_CERTS=/usr/share/kibana/config/certs/ca-cert.pem" volumes: - "./kibana.yml:/usr/share/kibana/config/kibana.yml" - - "../certs/kibana:/usr/share/kibana/config/certs" + - "./certs/kibana:/usr/share/kibana/config/certs" - "./kibana_healthcheck.sh:/usr/share/kibana/healthcheck.sh" ports: - "127.0.0.1:5601:5601" @@ -53,7 +53,7 @@ services: package-registry: build: context: ../../../ - dockerfile: "${STACK_PATH}/Dockerfile.package-registry" + dockerfile: "./profiles/${PROFILE_NAME}/stack/Dockerfile.package-registry" args: PROFILE: "${PROFILE_NAME}" healthcheck: @@ -67,7 +67,7 @@ services: - "EPR_TLS_KEY=/etc/ssl/package-registry/key.pem" - "EPR_TLS_CERT=/etc/ssl/package-registry/cert.pem" volumes: - - "../certs/package-registry:/etc/ssl/package-registry" + - "./certs/package-registry:/etc/ssl/package-registry" ports: - "127.0.0.1:8080:8080" - "127.0.0.1:9000:9000" @@ -104,8 +104,8 @@ services: - "KIBANA_FLEET_SETUP=1" - "KIBANA_HOST=https://kibana:5601" volumes: - - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" - - "../certs/fleet-server:/etc/ssl/elastic-agent" + - "./certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" + - "./certs/fleet-server:/etc/ssl/elastic-agent" ports: - "127.0.0.1:8220:8220" @@ -127,7 +127,7 @@ services: hostname: docker-fleet-agent env_file: "./elastic-agent.env" volumes: - - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" + - "./certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem" - type: bind source: ../../../tmp/service_logs/ target: /tmp/service_logs/ diff --git a/internal/profile/_static/elastic-agent.env.tmpl b/internal/stack/_static/elastic-agent.env.tmpl similarity index 100% rename from internal/profile/_static/elastic-agent.env.tmpl rename to internal/stack/_static/elastic-agent.env.tmpl diff --git a/internal/profile/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl similarity index 100% rename from internal/profile/_static/elasticsearch.yml.tmpl rename to internal/stack/_static/elasticsearch.yml.tmpl diff --git a/internal/profile/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl similarity index 100% rename from internal/profile/_static/kibana.yml.tmpl rename to internal/stack/_static/kibana.yml.tmpl diff --git a/internal/profile/_static/kibana_healthcheck.sh b/internal/stack/_static/kibana_healthcheck.sh similarity index 100% rename from internal/profile/_static/kibana_healthcheck.sh rename to internal/stack/_static/kibana_healthcheck.sh diff --git a/internal/profile/_static/package-registry.yml b/internal/stack/_static/package-registry.yml similarity index 100% rename from internal/profile/_static/package-registry.yml rename to internal/stack/_static/package-registry.yml diff --git a/internal/profile/_static/service_tokens b/internal/stack/_static/service_tokens similarity index 100% rename from internal/profile/_static/service_tokens rename to internal/stack/_static/service_tokens diff --git a/internal/stack/boot.go b/internal/stack/boot.go index 61b1426bb3..4e97731d9f 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -53,6 +53,11 @@ func BootUp(options Options) error { fmt.Printf("- %s\n", buildPackagesPath) } + err = applyResources(options.Profile, options.StackVersion) + if err != nil { + return errors.Wrap(err, "creating stack files failed") + } + err = dockerComposeBuild(options) if err != nil { return errors.Wrap(err, "building docker images failed") diff --git a/internal/profile/certs.go b/internal/stack/certs.go similarity index 99% rename from internal/profile/certs.go rename to internal/stack/certs.go index c4004d8020..2f9e87ecd5 100644 --- a/internal/profile/certs.go +++ b/internal/stack/certs.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package profile +package stack import ( "bytes" diff --git a/internal/profile/certs_test.go b/internal/stack/certs_test.go similarity index 99% rename from internal/profile/certs_test.go rename to internal/stack/certs_test.go index 95a78de66c..2d7daf2eba 100644 --- a/internal/profile/certs_test.go +++ b/internal/stack/certs_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package profile +package stack import ( "os" diff --git a/internal/stack/compose.go b/internal/stack/compose.go index 38b150df3f..11aa90ce29 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-package/internal/docker" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/elastic-package/internal/profile" ) type ServiceStatus struct { @@ -55,7 +54,7 @@ func (eb *envBuilder) build() []string { } func dockerComposeBuild(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -81,7 +80,7 @@ func dockerComposeBuild(options Options) error { } func dockerComposePull(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -107,7 +106,7 @@ func dockerComposePull(options Options) error { } func dockerComposeUp(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } @@ -139,7 +138,7 @@ func dockerComposeUp(options Options) error { } func dockerComposeDown(options Options) error { - c, err := compose.NewProject(DockerComposeProjectName, options.Profile.FetchPath(profile.SnapshotFile)) + c, err := compose.NewProject(DockerComposeProjectName, options.Profile.Path(profileStackPath, SnapshotFile)) if err != nil { return errors.Wrap(err, "could not create docker compose project") } diff --git a/internal/stack/dump.go b/internal/stack/dump.go index 008272afb1..7a80a27077 100644 --- a/internal/stack/dump.go +++ b/internal/stack/dump.go @@ -52,7 +52,7 @@ func dumpStackLogs(options DumpOptions) error { return errors.Wrapf(err, "can't create output location (path: %s)", logsPath) } - snapshotPath := options.Profile.FetchPath(profile.SnapshotFile) + snapshotPath := options.Profile.Path(profileStackPath, SnapshotFile) for _, serviceName := range observedServices { logger.Debugf("Dump stack logs for %s", serviceName) diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index 4fe898e438..6c7d6a3a12 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -26,7 +26,7 @@ type InitConfig struct { func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) { // Read Elasticsearch username and password from Kibana configuration file. - body, err := os.ReadFile(elasticStackProfile.FetchPath(profile.KibanaConfigFile)) + body, err := os.ReadFile(elasticStackProfile.Path(profileStackPath, KibanaConfigFile)) if err != nil { return nil, errors.Wrap(err, "error reading Kibana config file") } @@ -41,7 +41,7 @@ func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) } // Read Elasticsearch and Kibana hostnames from Elastic Stack Docker Compose configuration file. - p, err := compose.NewProject(DockerComposeProjectName, elasticStackProfile.FetchPath(profile.SnapshotFile)) + p, err := compose.NewProject(DockerComposeProjectName, elasticStackProfile.Path(profileStackPath, SnapshotFile)) if err != nil { return nil, errors.Wrap(err, "could not create docker compose project") } @@ -68,7 +68,7 @@ func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) es := serviceComposeConfig.Services["elasticsearch"] esHostPort := fmt.Sprintf("https://%s:%d", es.Ports[0].ExternalIP, es.Ports[0].ExternalPort) - caCert := elasticStackProfile.FetchPath(profile.CACertificateFile) + caCert := elasticStackProfile.Path(profileStackPath, CACertificateFile) return &InitConfig{ ElasticsearchHostPort: esHostPort, diff --git a/internal/stack/resources.go b/internal/stack/resources.go new file mode 100644 index 0000000000..130a0aa704 --- /dev/null +++ b/internal/stack/resources.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "embed" + "fmt" + "html/template" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver" + "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/go-resource" +) + +//go:embed _static +var static embed.FS + +const ( + // SnapshotFile is the docker-compose snapshot.yml file name. + SnapshotFile = "snapshot.yml" + + // ElasticsearchConfigFile is the elasticsearch config file. + ElasticsearchConfigFile = "elasticsearch.yml" + + // KibanaConfigFile is the kibana config file. + KibanaConfigFile = "kibana.yml" + + // KibanaHealthcheckFile is the kibana healthcheck. + KibanaHealthcheckFile = "kibana_healthcheck.sh" + + // PackageRegistryConfigFile is the config file for the Elastic Package registry + PackageRegistryConfigFile = "package-registry.yml" + + // PackageRegistryBaseImage is the base Docker image of the Elastic Package Registry. + PackageRegistryBaseImage = "docker.elastic.co/package-registry/package-registry:v1.15.0" + + // ElasticAgentEnvFile is the elastic agent environment variables file. + ElasticAgentEnvFile = "elastic-agent.env" + + profileStackPath = "stack" +) + +var ( + templateFuncs = template.FuncMap{ + "semverLessThan": semverLessThan, + } + staticSource = resource.NewSourceFS(static).WithTemplateFuncs(templateFuncs) + stackResources = []resource.Resource{ + &resource.File{ + Path: "Dockerfile.package-registry", + Content: staticSource.Template("_static/Dockerfile.package-registry.tmpl"), + }, + &resource.File{ + Path: SnapshotFile, + Content: staticSource.File("_static/docker-compose-stack.yml"), + }, + &resource.File{ + Path: ElasticsearchConfigFile, + Content: staticSource.Template("_static/elasticsearch.yml.tmpl"), + }, + &resource.File{ + Path: "service_tokens", + Content: staticSource.File("_static/service_tokens"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-ASN.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-ASN.mmdb"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-City.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-City.mmdb"), + }, + &resource.File{ + Path: "ingest-geoip/GeoLite2-Country.mmdb", + CreateParent: true, + Content: staticSource.File("_static/GeoLite2-Country.mmdb"), + }, + &resource.File{ + Path: KibanaConfigFile, + Content: staticSource.Template("_static/kibana.yml.tmpl"), + }, + &resource.File{ + Path: KibanaHealthcheckFile, + Content: staticSource.File("_static/kibana_healthcheck.sh"), + }, + &resource.File{ + Path: PackageRegistryConfigFile, + Content: staticSource.File("_static/package-registry.yml"), + }, + &resource.File{ + Path: ElasticAgentEnvFile, + Content: staticSource.Template("_static/elastic-agent.env.tmpl"), + }, + } +) + +func applyResources(profile *profile.Profile, stackVersion string) error { + stackDir := filepath.Join(profile.ProfilePath, profileStackPath) + + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "registry_base_image": PackageRegistryBaseImage, + "elasticsearch_version": stackVersion, + "kibana_version": stackVersion, + "agent_version": stackVersion, + }) + + os.MkdirAll(stackDir, 0755) + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: stackDir, + }) + + resources := append([]resource.Resource{}, stackResources...) + + certResources, err := initTLSCertificates("file", profile.ProfilePath) + if err != nil { + return fmt.Errorf("failed to create TLS files: %w", err) + } + resources = append(resources, certResources...) + + results, err := resourceManager.Apply(resources) + if err != nil { + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) + } + + return nil +} + +func semverLessThan(a, b string) (bool, error) { + sa, err := semver.NewVersion(a) + if err != nil { + return false, err + } + sb, err := semver.NewVersion(b) + if err != nil { + return false, err + } + + return sa.LessThan(sb), nil +} diff --git a/internal/stack/update.go b/internal/stack/update.go index ad52150169..26188d6410 100644 --- a/internal/stack/update.go +++ b/internal/stack/update.go @@ -8,12 +8,16 @@ import ( "github.com/pkg/errors" "github.com/elastic/elastic-package/internal/docker" - "github.com/elastic/elastic-package/internal/profile" ) // Update pulls down the most recent versions of the Docker images. func Update(options Options) error { - err := docker.Pull(profile.PackageRegistryBaseImage) + err := applyResources(options.Profile, options.StackVersion) + if err != nil { + return errors.Wrap(err, "creating stack files failed") + } + + err = docker.Pull(PackageRegistryBaseImage) if err != nil { return errors.Wrap(err, "pulling package-registry docker image failed") } From e164209f47a5ac49337578f2dd8421c64bc1185d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 30 Nov 2022 19:58:38 +0100 Subject: [PATCH 12/35] Add TODO --- internal/stack/compose.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/stack/compose.go b/internal/stack/compose.go index 11aa90ce29..e380cd492c 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -35,6 +35,7 @@ type envBuilder struct { vars []string } +// TODO: Use template variables instead of environment variables to parameterize docker-compose. func newEnvBuilder() *envBuilder { return new(envBuilder) } From 01f754bd53162be5a959a8bfbea812c11daaee8d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 10:03:31 +0100 Subject: [PATCH 13/35] Use public go-resource --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4550cde7ec..7b12e1ec72 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 - github.com/elastic/go-resource v0.0.0-00010101000000-000000000000 + github.com/elastic/go-resource v0.1.0 github.com/elastic/go-sysinfo v1.8.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.2.0 @@ -162,5 +162,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/elastic/go-resource => /home/jaime/gocode/src/github.com/elastic/go-resource diff --git a/go.sum b/go.sum index e1edb2cde6..eae49feaa9 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73El github.com/elastic/go-elasticsearch/v7 v7.17.7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= +github.com/elastic/go-resource v0.1.0 h1:7is06Dsz1EdgjTzSZ1PcR8MYO7sypHTin0BZqHV4h8s= +github.com/elastic/go-resource v0.1.0/go.mod h1:7F1Wjs6eSFX0i/235yAK/x9bvPNd9/ML92AiULa4XYA= github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4= github.com/elastic/go-sysinfo v1.8.1/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= From d22311c96c3c35a4cd2dfa3dc61b24aef6565aa9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 10:14:41 +0100 Subject: [PATCH 14/35] Linting --- internal/install/static.go | 19 ------------------- internal/profile/profile.go | 3 ++- internal/profile/profile_json.go | 3 ++- internal/stack/certs.go | 3 ++- internal/stack/resources.go | 3 ++- 5 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 internal/install/static.go diff --git a/internal/install/static.go b/internal/install/static.go deleted file mode 100644 index db8a32556b..0000000000 --- a/internal/install/static.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package install - -import _ "embed" - -//go:embed _static/Dockerfile.terraform_deployer -var terraformDeployerDockerfile string - -//go:embed _static/terraform_deployer.yml -var terraformDeployerYml string - -//go:embed _static/terraform_deployer_run.sh -var terraformDeployerRun string - -//go:embed _static/docker-custom-agent-base.yml -var dockerCustomAgentBaseYml string diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 57220e74ad..cc3b564d68 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -11,9 +11,10 @@ import ( "path/filepath" "strings" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" - "github.com/elastic/go-resource" ) const ( diff --git a/internal/profile/profile_json.go b/internal/profile/profile_json.go index b640bec650..edd142099d 100644 --- a/internal/profile/profile_json.go +++ b/internal/profile/profile_json.go @@ -13,8 +13,9 @@ import ( "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/version" "github.com/elastic/go-resource" + + "github.com/elastic/elastic-package/internal/version" ) // Metadata stores the data associated with a given profile diff --git a/internal/stack/certs.go b/internal/stack/certs.go index 2f9e87ecd5..e99cf1b3e7 100644 --- a/internal/stack/certs.go +++ b/internal/stack/certs.go @@ -11,8 +11,9 @@ import ( "io" "path/filepath" - "github.com/elastic/elastic-package/internal/certs" "github.com/elastic/go-resource" + + "github.com/elastic/elastic-package/internal/certs" ) // tlsServices is the list of server TLS certificates that will be diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 130a0aa704..96e15446a7 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -13,8 +13,9 @@ import ( "strings" "github.com/Masterminds/semver" - "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/go-resource" + + "github.com/elastic/elastic-package/internal/profile" ) //go:embed _static From 5c319597bc3d2bd27531f62851631cdfb53b85c8 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 17:39:02 +0100 Subject: [PATCH 15/35] Fix username and password --- ...tack.yml => docker-compose-stack.yml.tmpl} | 6 ++++-- internal/stack/_static/kibana.yml.tmpl | 4 ++-- ...lthcheck.sh => kibana_healthcheck.sh.tmpl} | 2 +- internal/stack/initconfig.go | 21 ++----------------- internal/stack/resources.go | 10 +++++++-- 5 files changed, 17 insertions(+), 26 deletions(-) rename internal/stack/_static/{docker-compose-stack.yml => docker-compose-stack.yml.tmpl} (94%) rename internal/stack/_static/{kibana_healthcheck.sh => kibana_healthcheck.sh.tmpl} (64%) diff --git a/internal/stack/_static/docker-compose-stack.yml b/internal/stack/_static/docker-compose-stack.yml.tmpl similarity index 94% rename from internal/stack/_static/docker-compose-stack.yml rename to internal/stack/_static/docker-compose-stack.yml.tmpl index 1704029ea1..4118ddc705 100644 --- a/internal/stack/_static/docker-compose-stack.yml +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -1,14 +1,16 @@ +{{ $username := fact "username" }} +{{ $password := fact "password" }} version: '2.3' services: elasticsearch: image: "${ELASTICSEARCH_IMAGE_REF}" healthcheck: - test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u elastic:changeme https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" + test: "curl -s --cacert /usr/share/elasticsearch/config/certs/ca-cert.pem -f -u {{ $username }}:{{ $password }} https://127.0.0.1:9200/_cat/health | cut -f4 -d' ' | grep -E '(green|yellow)'" retries: 300 interval: 1s environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - - "ELASTIC_PASSWORD=changeme" + - "ELASTIC_PASSWORD={{ $password }}" volumes: - "./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" - "./certs/elasticsearch:/usr/share/elasticsearch/config/certs" diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index c9b838fe8b..e40db946c7 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -10,8 +10,8 @@ elasticsearch.hosts: [ "https://elasticsearch:9200" ] elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" {{ if semverLessThan $version "8.0.0" }} -elasticsearch.username: elastic -elasticsearch.password: changeme +elasticsearch.username: {{ fact "username" }} +elasticsearch.password: {{ fact "password" }} xpack.monitoring.ui.container.elasticsearch.enabled: true xpack.fleet.enabled: true diff --git a/internal/stack/_static/kibana_healthcheck.sh b/internal/stack/_static/kibana_healthcheck.sh.tmpl similarity index 64% rename from internal/stack/_static/kibana_healthcheck.sh rename to internal/stack/_static/kibana_healthcheck.sh.tmpl index 09bc39c355..386b55add4 100644 --- a/internal/stack/_static/kibana_healthcheck.sh +++ b/internal/stack/_static/kibana_healthcheck.sh.tmpl @@ -3,4 +3,4 @@ set -e curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f https://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null -curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f -u elastic:changeme "https://elasticsearch:9200/_cat/indices/.security-*?h=health" | grep -v red +curl -s --cacert /usr/share/kibana/config/certs/ca-cert.pem -f -u {{ fact "username" }}:{{ fact "password" }} "https://elasticsearch:9200/_cat/indices/.security-*?h=health" | grep -v red diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index 6c7d6a3a12..1af8926054 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -6,10 +6,8 @@ package stack import ( "fmt" - "os" "github.com/pkg/errors" - "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/install" @@ -25,21 +23,6 @@ type InitConfig struct { } func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) { - // Read Elasticsearch username and password from Kibana configuration file. - body, err := os.ReadFile(elasticStackProfile.Path(profileStackPath, KibanaConfigFile)) - if err != nil { - return nil, errors.Wrap(err, "error reading Kibana config file") - } - - var kibanaCfg struct { - ElasticsearchUsername string `yaml:"elasticsearch.username"` - ElasticsearchPassword string `yaml:"elasticsearch.password"` - } - err = yaml.Unmarshal(body, &kibanaCfg) - if err != nil { - return nil, errors.Wrap(err, "unmarshalling Kibana configuration failed") - } - // Read Elasticsearch and Kibana hostnames from Elastic Stack Docker Compose configuration file. p, err := compose.NewProject(DockerComposeProjectName, elasticStackProfile.Path(profileStackPath, SnapshotFile)) if err != nil { @@ -72,8 +55,8 @@ func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) return &InitConfig{ ElasticsearchHostPort: esHostPort, - ElasticsearchUsername: kibanaCfg.ElasticsearchUsername, - ElasticsearchPassword: kibanaCfg.ElasticsearchPassword, + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, KibanaHostPort: kibHostPort, CACertificatePath: caCert, }, nil diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 96e15446a7..fc072c61d2 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -44,6 +44,9 @@ const ( ElasticAgentEnvFile = "elastic-agent.env" profileStackPath = "stack" + + elasticsearchUsername = "elastic" + elasticsearchPassword = "changeme" ) var ( @@ -58,7 +61,7 @@ var ( }, &resource.File{ Path: SnapshotFile, - Content: staticSource.File("_static/docker-compose-stack.yml"), + Content: staticSource.Template("_static/docker-compose-stack.yml.tmpl"), }, &resource.File{ Path: ElasticsearchConfigFile, @@ -89,7 +92,7 @@ var ( }, &resource.File{ Path: KibanaHealthcheckFile, - Content: staticSource.File("_static/kibana_healthcheck.sh"), + Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"), }, &resource.File{ Path: PackageRegistryConfigFile, @@ -111,6 +114,9 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "elasticsearch_version": stackVersion, "kibana_version": stackVersion, "agent_version": stackVersion, + + "username": elasticsearchUsername, + "password": elasticsearchPassword, }) os.MkdirAll(stackDir, 0755) From 200b4d71d0614a10d021ce57fe109dd5f6a84031 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 18:50:13 +0100 Subject: [PATCH 16/35] Compose provider --- cmd/stack.go | 69 ++++++++++++++++++++++++++------- internal/cobraext/flags.go | 3 ++ internal/stack/cloud.go | 42 ++++++++++++++++++++ internal/stack/providers.go | 76 +++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 13 deletions(-) create mode 100644 internal/stack/cloud.go create mode 100644 internal/stack/providers.go diff --git a/cmd/stack.go b/cmd/stack.go index b4c1044a76..7eaa428e40 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -78,25 +78,30 @@ func setupStackCommand() *cobraext.Command { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - userProfile, err := getProfileFlag(cmd) + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + + provider, err := getProviderFlag(cmd, profile) if err != nil { return err } // Print information before starting the stack, for cases where // this is executed in the foreground, without daemon mode. - cmd.Printf("Using profile %s.\n", userProfile.ProfilePath) + cmd.Printf("Using profile %s.\n", profile.ProfilePath) cmd.Println(`Remember to load stack environment variables using 'eval "$(elastic-package stack shellinit)"'.`) - err = printInitConfig(cmd, userProfile) + err = printInitConfig(cmd, profile) if err != nil { return err } - err = stack.BootUp(stack.Options{ + err = provider.BootUp(stack.Options{ DaemonMode: daemonMode, StackVersion: stackVersion, Services: services, - Profile: userProfile, + Profile: profile, }) if err != nil { return errors.Wrap(err, "booting up the stack failed") @@ -117,13 +122,18 @@ func setupStackCommand() *cobraext.Command { RunE: func(cmd *cobra.Command, args []string) error { cmd.Println("Take down the Elastic stack") - userProfile, err := getProfileFlag(cmd) + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + + provider, err := getProviderFlag(cmd, profile) if err != nil { return err } - err = stack.TearDown(stack.Options{ - Profile: userProfile, + err = provider.TearDown(stack.Options{ + Profile: profile, }) if err != nil { return errors.Wrap(err, "tearing down the stack failed") @@ -145,12 +155,17 @@ func setupStackCommand() *cobraext.Command { return err } + provider, err := getProviderFlag(cmd, profile) + if err != nil { + return err + } + stackVersion, err := cmd.Flags().GetString(cobraext.StackVersionFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.StackVersionFlagName) } - err = stack.Update(stack.Options{ + err = provider.Update(stack.Options{ StackVersion: stackVersion, Profile: profile, }) @@ -211,7 +226,12 @@ func setupStackCommand() *cobraext.Command { return err } - target, err := stack.Dump(stack.DumpOptions{ + provider, err := getProviderFlag(cmd, profile) + if err != nil { + return err + } + + target, err := provider.Dump(stack.DumpOptions{ Output: output, Profile: profile, }) @@ -231,7 +251,17 @@ func setupStackCommand() *cobraext.Command { Use: "status", Short: "Show status of the stack services", RunE: func(cmd *cobra.Command, args []string) error { - servicesStatus, err := stack.Status() + profile, err := getProfileFlag(cmd) + if err != nil { + return err + } + + provider, err := getProviderFlag(cmd, profile) + if err != nil { + return err + } + + servicesStatus, err := provider.Status() if err != nil { return errors.Wrap(err, "failed getting stack status") } @@ -248,6 +278,7 @@ func setupStackCommand() *cobraext.Command { Long: stackLongDescription, } cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) + cmd.PersistentFlags().String(cobraext.StackProviderFlagName, "compose", fmt.Sprintf(cobraext.StackProviderFlagDescription, strings.Join(stack.SupportedProviders, ", "))) cmd.AddCommand( upCommand, downCommand, @@ -340,7 +371,7 @@ func getProfileFlag(cmd *cobra.Command) (*profile.Profile, error) { profileName = config.CurrentProfile() } - userProfile, err := profile.LoadProfile(profileName) + p, err := profile.LoadProfile(profileName) if errors.Is(err, profile.ErrNotAProfile) { list, err := availableProfilesAsAList() if err != nil { @@ -355,7 +386,19 @@ func getProfileFlag(cmd *cobra.Command) (*profile.Profile, error) { return nil, errors.Wrap(err, "error loading profile") } - return userProfile, nil + return p, nil +} + +func getProviderFlag(cmd *cobra.Command, profile *profile.Profile) (stack.Provider, error) { + providerName, err := cmd.Flags().GetString(cobraext.StackProviderFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.StackProviderFlagName) + } + if providerName == "" { + providerName = stack.DefaultProvider + } + + return stack.BuildProvider(providerName, profile) } func getShellName(exe string) string { diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 22b9103226..e6a505baaf 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -144,6 +144,9 @@ const ( TLSSkipVerifyFlagName = "tls-skip-verify" TLSSkipVerifyFlagDescription = "skip TLS verify" + StackProviderFlagName = "provider" + StackProviderFlagDescription = "service provider to start a stack (%s)" + StackServicesFlagName = "services" StackServicesFlagDescription = "component services (comma-separated values: \"%s\")" diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go new file mode 100644 index 0000000000..820ceec2c6 --- /dev/null +++ b/internal/stack/cloud.go @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + + "github.com/elastic/elastic-package/internal/profile" +) + +type cloudProvider struct { + profile *profile.Profile +} + +func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { + return &cloudProvider{ + profile: profile, + }, nil +} + +func (*cloudProvider) BootUp(options Options) error { + return fmt.Errorf("not implemented") +} + +func (*cloudProvider) TearDown(options Options) error { + return fmt.Errorf("not implemented") +} + +func (*cloudProvider) Update(options Options) error { + fmt.Println("Nothing to do.") + return nil +} + +func (*cloudProvider) Dump(options DumpOptions) (string, error) { + return "", fmt.Errorf("not implemented") +} + +func (*cloudProvider) Status() ([]ServiceStatus, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/internal/stack/providers.go b/internal/stack/providers.go new file mode 100644 index 0000000000..4e5b61baa2 --- /dev/null +++ b/internal/stack/providers.go @@ -0,0 +1,76 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "fmt" + "strings" + + "github.com/elastic/elastic-package/internal/profile" +) + +const ( + ProviderCompose = "compose" + ProviderCloud = "cloud" +) + +var ( + DefaultProvider = ProviderCompose + SupportedProviders = []string{ + ProviderCompose, + ProviderCloud, + } +) + +// Provider is the implementation of a stack provider. +type Provider interface { + // BootUp starts a stack. + BootUp(Options) error + + // TearDown stops and/or removes a stack. + TearDown(Options) error + + // Update updates resources associated to a stack. + Update(Options) error + + // Dump dumps data for debug purpouses. + Dump(DumpOptions) (string, error) + + // Status obtains status information of the stack. + Status() ([]ServiceStatus, error) +} + +// BuildProvider returns the provider for the given name. +func BuildProvider(name string, profile *profile.Profile) (Provider, error) { + switch name { + case "compose": + return &composeProvider{}, nil + case "cloud": + return newCloudProvider(profile) + } + return nil, fmt.Errorf("unknown provider %q, supported providers: %s", name, strings.Join(SupportedProviders, ", ")) +} + +type composeProvider struct{} + +func (*composeProvider) BootUp(options Options) error { + return BootUp(options) +} + +func (*composeProvider) TearDown(options Options) error { + return TearDown(options) +} + +func (*composeProvider) Update(options Options) error { + return Update(options) +} + +func (*composeProvider) Dump(options DumpOptions) (string, error) { + return Dump(options) +} + +func (*composeProvider) Status() ([]ServiceStatus, error) { + return Status() +} From 57c3e972436dd06b4bb23b4630c03da1f0324b03 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 19:18:32 +0100 Subject: [PATCH 17/35] Persist stack configuration --- cmd/stack.go | 4 ++- internal/stack/boot.go | 13 +++++++ internal/stack/cloud.go | 2 +- internal/stack/config.go | 69 ++++++++++++++++++++++++++++++++++++ internal/stack/initconfig.go | 47 +++++------------------- internal/stack/providers.go | 4 +-- internal/stack/shellinit.go | 6 ++-- 7 files changed, 98 insertions(+), 47 deletions(-) create mode 100644 internal/stack/config.go diff --git a/cmd/stack.go b/cmd/stack.go index 7eaa428e40..51d836baac 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -261,7 +261,9 @@ func setupStackCommand() *cobraext.Command { return err } - servicesStatus, err := provider.Status() + servicesStatus, err := provider.Status(stack.Options{ + Profile: profile, + }) if err != nil { return errors.Wrap(err, "failed getting stack status") } diff --git a/internal/stack/boot.go b/internal/stack/boot.go index 4e97731d9f..77d1fc39e3 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -68,6 +68,19 @@ func BootUp(options Options) error { return errors.Wrap(err, "running docker-compose failed") } + config := Config{ + Provider: ProviderCompose, + ElasticsearchHost: "https://127.0.0.1:9200", + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, + KibanaHost: "https://127.0.0.1:5601", + CACertFile: options.Profile.Path(profileStackPath, CACertificateFile), + } + err = storeConfig(options.Profile, config) + if err != nil { + return errors.Wrap(err, "failed to store config") + } + return nil } diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 820ceec2c6..430f5bc720 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -37,6 +37,6 @@ func (*cloudProvider) Dump(options DumpOptions) (string, error) { return "", fmt.Errorf("not implemented") } -func (*cloudProvider) Status() ([]ServiceStatus, error) { +func (*cloudProvider) Status(options Options) ([]ServiceStatus, error) { return nil, fmt.Errorf("not implemented") } diff --git a/internal/stack/config.go b/internal/stack/config.go new file mode 100644 index 0000000000..67c4976443 --- /dev/null +++ b/internal/stack/config.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stack + +import ( + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/elastic/elastic-package/internal/profile" +) + +var configFileName = "config.json" + +type Config struct { + Provider string `json:"provider,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` + + ElasticsearchHost string `json:"elasticsearch_host,omitempty"` + ElasticsearchUsername string `json:"elasticsearch_username,omitempty"` + ElasticsearchPassword string `json:"elasticsearch_password,omitempty"` + KibanaHost string `json:"kibana_host,omitempty"` + CACertFile string `json:"ca_cert_file,omitempty"` +} + +func configPath(profile *profile.Profile) string { + return profile.Path(profileStackPath, configFileName) +} + +func defaultConfig() Config { + return Config{ + Provider: DefaultProvider, + } +} + +func loadConfig(profile *profile.Profile) (Config, error) { + d, err := os.ReadFile(configPath(profile)) + if errors.Is(err, os.ErrNotExist) { + return defaultConfig(), nil + } + if err != nil { + return Config{}, fmt.Errorf("failed to read stack config: %w", err) + } + + var config Config + err = json.Unmarshal(d, &config) + if err != nil { + return Config{}, fmt.Errorf("failed to decode stack config: %w", err) + } + + return config, nil +} + +func storeConfig(profile *profile.Profile, config Config) error { + d, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to encode stack config: %w", err) + } + + err = os.WriteFile(configPath(profile), d, 0644) + if err != nil { + return fmt.Errorf("failed to write stack config: %s", err) + } + + return nil +} diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index 1af8926054..4711883b29 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -5,12 +5,6 @@ package stack import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-package/internal/compose" - "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/profile" ) @@ -22,42 +16,17 @@ type InitConfig struct { CACertificatePath string } -func StackInitConfig(elasticStackProfile *profile.Profile) (*InitConfig, error) { - // Read Elasticsearch and Kibana hostnames from Elastic Stack Docker Compose configuration file. - p, err := compose.NewProject(DockerComposeProjectName, elasticStackProfile.Path(profileStackPath, SnapshotFile)) - if err != nil { - return nil, errors.Wrap(err, "could not create docker compose project") - } - - appConfig, err := install.Configuration() +func StackInitConfig(profile *profile.Profile) (*InitConfig, error) { + config, err := loadConfig(profile) if err != nil { - return nil, errors.Wrap(err, "can't read application configuration") + return nil, err } - serviceComposeConfig, err := p.Config(compose.CommandOptions{ - Env: newEnvBuilder(). - withEnvs(appConfig.StackImageRefs(install.DefaultStackVersion).AsEnv()). - withEnvs(elasticStackProfile.ComposeEnvVars()). - withEnv(stackVariantAsEnv(install.DefaultStackVersion)). - build(), - }) - if err != nil { - return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") - } - - kib := serviceComposeConfig.Services["kibana"] - kibHostPort := fmt.Sprintf("https://%s:%d", kib.Ports[0].ExternalIP, kib.Ports[0].ExternalPort) - - es := serviceComposeConfig.Services["elasticsearch"] - esHostPort := fmt.Sprintf("https://%s:%d", es.Ports[0].ExternalIP, es.Ports[0].ExternalPort) - - caCert := elasticStackProfile.Path(profileStackPath, CACertificateFile) - return &InitConfig{ - ElasticsearchHostPort: esHostPort, - ElasticsearchUsername: elasticsearchUsername, - ElasticsearchPassword: elasticsearchPassword, - KibanaHostPort: kibHostPort, - CACertificatePath: caCert, + ElasticsearchHostPort: config.ElasticsearchHost, + ElasticsearchUsername: config.ElasticsearchUsername, + ElasticsearchPassword: config.ElasticsearchPassword, + KibanaHostPort: config.KibanaHost, + CACertificatePath: config.CACertFile, }, nil } diff --git a/internal/stack/providers.go b/internal/stack/providers.go index 4e5b61baa2..7ad12fbd47 100644 --- a/internal/stack/providers.go +++ b/internal/stack/providers.go @@ -39,7 +39,7 @@ type Provider interface { Dump(DumpOptions) (string, error) // Status obtains status information of the stack. - Status() ([]ServiceStatus, error) + Status(Options) ([]ServiceStatus, error) } // BuildProvider returns the provider for the given name. @@ -71,6 +71,6 @@ func (*composeProvider) Dump(options DumpOptions) (string, error) { return Dump(options) } -func (*composeProvider) Status() ([]ServiceStatus, error) { +func (*composeProvider) Status(options Options) ([]ServiceStatus, error) { return Status() } diff --git a/internal/stack/shellinit.go b/internal/stack/shellinit.go index f8d0240f37..e5c64d63c0 100644 --- a/internal/stack/shellinit.go +++ b/internal/stack/shellinit.go @@ -51,16 +51,14 @@ const ( export %s=%s export %s=%s export %s=%s -export %s=%s -` +export %s=%s` // fish shell init code. // fish shell is similar but not compliant to POSIX. fishTemplate = `set -x %s %s; set -x %s %s; set -x %s %s; set -x %s %s; -set -x %s %s; -` +set -x %s %s;` ) // availableShellTypes list all available values for s in initTemplate From a24368c5e12f02d32bdd3fd288fa1022bbff7ff4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 1 Dec 2022 20:34:05 +0100 Subject: [PATCH 18/35] Create a cluster --- go.mod | 26 ++++-- go.sum | 188 ++++++++++++++++++++++++++++++++++++++-- internal/stack/cloud.go | 77 +++++++++++++++- 3 files changed, 273 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 06e86dae6f..109f0e9beb 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aymerick/raymond v2.0.2+incompatible github.com/boumenot/gocover-cobertura v1.2.0 github.com/cespare/xxhash/v2 v2.1.2 + github.com/elastic/cloud-sdk-go v1.11.0 github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-resource v0.1.0 @@ -55,7 +56,8 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/andybalholm/brotli v1.0.3 // indirect - github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudflare/circl v1.1.0 // indirect github.com/creasty/defaults v1.5.2 // indirect @@ -73,12 +75,17 @@ require ( github.com/go-errors/errors v1.0.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/errors v0.19.8 // indirect + github.com/go-openapi/analysis v0.21.1 // indirect + github.com/go-openapi/errors v0.20.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/strfmt v0.21.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/go-stack/stack v1.8.0 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/loads v0.21.0 // indirect + github.com/go-openapi/runtime v0.23.0 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.21.2 // indirect + github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-openapi/validate v0.20.3 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect @@ -88,6 +95,8 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -100,7 +109,7 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -116,6 +125,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/prometheus/procfs v0.8.0 // indirect @@ -130,7 +140,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect - go.mongodb.org/mongo-driver v1.7.3 // indirect + go.mongodb.org/mongo-driver v1.8.2 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect diff --git a/go.sum b/go.sum index 1f3dcfe253..5c509ab0c0 100644 --- a/go.sum +++ b/go.sum @@ -63,12 +63,15 @@ github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f h1:CGq7OieOz3wy github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4= github.com/ProtonMail/gopenpgp/v2 v2.4.10 h1:EYgkxzwmQvsa6kxxkgP1AwzkFqKHscF2UINxaSn6rdI= github.com/ProtonMail/gopenpgp/v2 v2.4.10/go.mod h1:CTRA7/toc/4DxDy5Du4hPDnIZnJvXSeQ8LsRTOUJoyc= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -77,10 +80,18 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/boumenot/gocover-cobertura v1.2.0 h1:g+VROIASoEHBrEilIyaCmgo7HGm+AV5yKEPLk0qIY+s= github.com/boumenot/gocover-cobertura v1.2.0/go.mod h1:fz7ly8dslE42VRR5ZWLt2OHGDHjkTiA2oNvKgJEjLT0= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -115,10 +126,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/elastic/cloud-sdk-go v1.11.0 h1:rJRL6EScg4uJXRFlvWB6GdEBvJ7hvbZZ3NjMdH7ak/k= +github.com/elastic/cloud-sdk-go v1.11.0/go.mod h1:YJfcOSFF/MS+o9dWZAUs+JYWfoeRWzAKmtUm27cbUzA= github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73Els0nbaWOmYachs= github.com/elastic/go-elasticsearch/v7 v7.17.7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= @@ -161,6 +176,8 @@ github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmV github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= @@ -179,20 +196,112 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/errors v0.19.8 h1:doM+tQdZbUm9gydV9yR+iQNmztbjj7I3sW4sIcAwIzc= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= +github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.21.1 h1:krcNCEvCttpSUFBPOrfvn7nniejvrOkoNYRlZwQFpEs= +github.com/go-openapi/analysis v0.21.1/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2 h1:dxy7PGTqEh94zj2E3h1cUmQQWiM1+aeCROfAr02EmK8= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/strfmt v0.21.0 h1:hX2qEZKmYks+t0hKeb4VTJpUm2UYsdL3+DCid5swxIs= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= +github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= +github.com/go-openapi/loads v0.21.0 h1:jYtUO4wwP7psAweisP/MDoOpdzsYEESdoPcsWjHDR68= +github.com/go-openapi/loads v0.21.0/go.mod h1:rHYve9nZrQ4CJhyeIIFJINGCg1tQpx2yJrrNo8sf1ws= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= +github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.23.0 h1:HX6ET2sHCIvaKeDDQoU01CtO1ekg5EkekHSkLTtWXH0= +github.com/go-openapi/runtime v0.23.0/go.mod h1:aQg+kaIQEn+A2CRSY1TxbM8+sT9g2V3aLc1FbIAnbbs= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= +github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.2 h1:5NDNgadiX1Vhemth/TH4gCGopWSTdDjxl60H3B7f+os= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= +github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= +github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= +github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= +github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -290,6 +399,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= @@ -300,6 +410,11 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -316,6 +431,8 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PH github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -331,9 +448,11 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -348,6 +467,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -355,10 +475,14 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= @@ -382,7 +506,11 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= @@ -408,6 +536,10 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -431,6 +563,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -449,6 +582,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -467,11 +601,14 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -491,8 +628,18 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.2 h1:8ssUXufb90ujcIvR6MyE1SchaNj0SFxsakiZgxIyrMk= +go.mongodb.org/mongo-driver v1.8.2/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -504,12 +651,17 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -558,13 +710,16 @@ golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -580,15 +735,22 @@ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -618,6 +780,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -626,6 +789,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -655,6 +819,7 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -672,6 +837,7 @@ golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -683,6 +849,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -694,7 +861,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -707,6 +876,8 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -866,6 +1037,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/gotestsum v1.8.2 h1:szU3TaSz8wMx/uG+w/A2+4JUPwH903YYaMI9yOOYAyI= diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 430f5bc720..0b98ad4026 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -5,23 +5,96 @@ package stack import ( + "errors" "fmt" + "net/http" + "net/url" + "os" + + "github.com/elastic/cloud-sdk-go/pkg/api" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/depresourceapi" + "github.com/elastic/cloud-sdk-go/pkg/auth" "github.com/elastic/elastic-package/internal/profile" ) +// Docs: https://www.elastic.co/guide/en/cloud/current/ec-api-deployment-crud.html +const cloudAPI = "https://api.elastic-cloud.com/api/v1" + +var cloudDeploymentsAPI string + +func init() { + mustJoinURL := func(base string, elem ...string) string { + joined, err := url.JoinPath(base, elem...) + if err != nil { + panic(err) + } + return joined + } + cloudDeploymentsAPI = mustJoinURL(cloudAPI, "deployments") +} + type cloudProvider struct { profile *profile.Profile } +func (cp *cloudProvider) getAPI() (*api.API, error) { + apiKey := os.Getenv("EC_API_KEY") + if apiKey == "" { + return nil, errors.New("unable to obtain value from EC_API_KEY environment variable") + } + + ec, err := api.NewAPI(api.Config{ + Client: new(http.Client), + AuthWriter: auth.APIKey(apiKey), + }) + if err != nil { + return nil, err + } + + return ec, nil + +} + func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { return &cloudProvider{ profile: profile, }, nil } -func (*cloudProvider) BootUp(options Options) error { - return fmt.Errorf("not implemented") +func (cp *cloudProvider) BootUp(options Options) error { + api, err := cp.getAPI() + if err != nil { + return err + } + + // TODO: Parameterize this. + name := "elastic-package-test" + region := "gcp-europe-west3" + templateID := "gcp-storage-optimized" + + payload, err := depresourceapi.NewPayload(depresourceapi.NewPayloadParams{ + API: api, + Name: name, + Version: options.StackVersion, + Region: region, + DeploymentTemplateID: templateID, + }) + if err != nil { + return fmt.Errorf("failed to initialize payload: %w", err) + } + + res, err := deploymentapi.Create(deploymentapi.CreateParams{ + API: api, + Request: payload, + }) + if err != nil { + return fmt.Errorf("failed to create deployment: %w", err) + } + + fmt.Printf("%+v\n", res) + return nil } func (*cloudProvider) TearDown(options Options) error { From ebf5d9547d503cbe32465f9a741ca18359709d4e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 11:02:11 +0100 Subject: [PATCH 19/35] Cloud deployment creation --- internal/stack/cloud.go | 148 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 11 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 0b98ad4026..9288765aa7 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -5,18 +5,22 @@ package stack import ( + "encoding/json" "errors" "fmt" "net/http" "net/url" "os" + "time" "github.com/elastic/cloud-sdk-go/pkg/api" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" - "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/depresourceapi" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" "github.com/elastic/cloud-sdk-go/pkg/auth" + "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/elastic/elastic-package/internal/profile" + "github.com/elastic/elastic-package/internal/signal" ) // Docs: https://www.elastic.co/guide/en/cloud/current/ec-api-deployment-crud.html @@ -69,34 +73,156 @@ func (cp *cloudProvider) BootUp(options Options) error { return err } + // TODO: Check if the deployment already exists. + // TODO: Parameterize this. name := "elastic-package-test" region := "gcp-europe-west3" - templateID := "gcp-storage-optimized" - - payload, err := depresourceapi.NewPayload(depresourceapi.NewPayloadParams{ - API: api, - Name: name, - Version: options.StackVersion, - Region: region, - DeploymentTemplateID: templateID, + templateID := "gcp-io-optimized" + + template, err := deptemplateapi.Get(deptemplateapi.GetParams{ + API: api, + TemplateID: templateID, + Region: region, + StackVersion: options.StackVersion, }) if err != nil { - return fmt.Errorf("failed to initialize payload: %w", err) + return fmt.Errorf("failed to get deployment template %q: %w", templateID, err) + } + + payload := template.DeploymentTemplate + + // Remove the resources that we don't need. + payload.Resources.Apm = nil + payload.Resources.Appsearch = nil + payload.Resources.EnterpriseSearch = nil + + // Initialize the plan with the id of the template, otherwise the create request fails. + if es := payload.Resources.Elasticsearch; len(es) > 0 { + plan := es[0].Plan + if plan.DeploymentTemplate == nil { + plan.DeploymentTemplate = &models.DeploymentTemplateReference{} + } + plan.DeploymentTemplate.ID = &templateID } res, err := deploymentapi.Create(deploymentapi.CreateParams{ API: api, Request: payload, + Overrides: &deploymentapi.PayloadOverrides{ + Name: name, + }, }) if err != nil { return fmt.Errorf("failed to create deployment: %w", err) } + if created := res.Created; created == nil || !*created { + return fmt.Errorf("request succeeded, but deployment was not created, check in the console UI") + } + + var config Config + config.Parameters = map[string]string{ + "alias": res.Alias, + } + deploymentID := res.ID + if deploymentID == nil { + return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") + } + config.Parameters["id"] = *deploymentID + for _, resource := range res.Resources { + kind := resource.Kind + if kind == nil { + continue + } + if *kind == "elasticsearch" { + if creds := resource.Credentials; creds != nil { + if creds.Username != nil { + config.ElasticsearchUsername = *creds.Username + } + if creds.Password != nil { + config.ElasticsearchPassword = *creds.Password + } + } + } + } + + // Storing once before getting the endpoints, so we have the ID. + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } + + for { + deployment, err := deploymentapi.Get(deploymentapi.GetParams{ + API: api, + DeploymentID: *deploymentID, + }) + if err != nil { + return fmt.Errorf("couldn't check deployment health: %w", err) + } + + if healthy := deployment.Healthy; healthy == nil || !*healthy { + if signal.SIGINT() { + return fmt.Errorf("wait interrupted") + } + time.Sleep(1 * time.Second) + continue + } + + // TODO: Check that resources are healthy too. + + config.ElasticsearchHost, err = cp.getServiceURL(deployment.Resources.Elasticsearch) + if err != nil { + return fmt.Errorf("failed to get elasticsearch host: %w", err) + } + config.KibanaHost, err = cp.getServiceURL(deployment.Resources.Kibana) + if err != nil { + return fmt.Errorf("failed to get kibana host: %w", err) + } + config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) + if err != nil { + return fmt.Errorf("failed to get fleet host: %w", err) + } + + break + } + + // Store the configuration again now with the service urls. + err = storeConfig(cp.profile, config) + if err != nil { + return fmt.Errorf("failed to store config: %w", err) + } - fmt.Printf("%+v\n", res) return nil } +func (*cloudProvider) getServiceURL(resourcesResponse any) (string, error) { + // Converting back and forth for easier access. + var resources []struct { + Info struct { + Metadata struct { + ServiceURL string `json:"service_url"` + } `json:"metadata"` + } `json:"info"` + } + + d, err := json.Marshal(resourcesResponse) + if err != nil { + return "", fmt.Errorf("failed to marshal resources: %w", err) + } + err = json.Unmarshal(d, &resources) + if err != nil { + return "", fmt.Errorf("failed to unmarshal resources back: %w", err) + } + + for _, resource := range resources { + if serviceURL := resource.Info.Metadata.ServiceURL; serviceURL != "" { + return serviceURL, nil + } + } + return "", fmt.Errorf("url not found") +} + func (*cloudProvider) TearDown(options Options) error { return fmt.Errorf("not implemented") } From 592d61094147233687fa5d74b4ffd1c4a59500c0 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 16:01:52 +0100 Subject: [PATCH 20/35] Tear down and status --- internal/stack/cloud.go | 195 ++++++++++++++++++++++++++++++++-------- 1 file changed, 158 insertions(+), 37 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 9288765aa7..cd54493446 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -19,12 +19,23 @@ import ( "github.com/elastic/cloud-sdk-go/pkg/auth" "github.com/elastic/cloud-sdk-go/pkg/models" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" "github.com/elastic/elastic-package/internal/signal" ) -// Docs: https://www.elastic.co/guide/en/cloud/current/ec-api-deployment-crud.html -const cloudAPI = "https://api.elastic-cloud.com/api/v1" +const ( + paramCloudDeploymentID = "cloud_deployment_id" + paramCloudDeploymentAlias = "cloud_deployment_alias" + + // Docs: https://www.elastic.co/guide/en/cloud/current/ec-api-deployment-crud.html + cloudAPI = "https://api.elastic-cloud.com/api/v1" +) + +var ( + deploymentNotExistErr = errors.New("deployment does not exist") + deploymentAlreadyExistErr = errors.New("deployment already exists") +) var cloudDeploymentsAPI string @@ -40,48 +51,46 @@ func init() { } type cloudProvider struct { + api *api.API profile *profile.Profile } -func (cp *cloudProvider) getAPI() (*api.API, error) { +func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { apiKey := os.Getenv("EC_API_KEY") if apiKey == "" { return nil, errors.New("unable to obtain value from EC_API_KEY environment variable") } - - ec, err := api.NewAPI(api.Config{ + api, err := api.NewAPI(api.Config{ Client: new(http.Client), AuthWriter: auth.APIKey(apiKey), }) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to initialize API client: %w", err) } - return ec, nil - -} - -func newCloudProvider(profile *profile.Profile) (*cloudProvider, error) { return &cloudProvider{ + api: api, profile: profile, }, nil } func (cp *cloudProvider) BootUp(options Options) error { - api, err := cp.getAPI() - if err != nil { + _, err := cp.currentDeployment() + if err == nil { + // Do nothing, deployment already exists. + // TODO: Migrate configuration if changed. + return nil + } else if err != nil && err != deploymentNotExistErr { return err } - // TODO: Check if the deployment already exists. - // TODO: Parameterize this. name := "elastic-package-test" region := "gcp-europe-west3" templateID := "gcp-io-optimized" template, err := deptemplateapi.Get(deptemplateapi.GetParams{ - API: api, + API: cp.api, TemplateID: templateID, Region: region, StackVersion: options.StackVersion, @@ -107,7 +116,7 @@ func (cp *cloudProvider) BootUp(options Options) error { } res, err := deploymentapi.Create(deploymentapi.CreateParams{ - API: api, + API: cp.api, Request: payload, Overrides: &deploymentapi.PayloadOverrides{ Name: name, @@ -122,13 +131,13 @@ func (cp *cloudProvider) BootUp(options Options) error { var config Config config.Parameters = map[string]string{ - "alias": res.Alias, + "cloud_deployment_alias": res.Alias, } deploymentID := res.ID if deploymentID == nil { return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") } - config.Parameters["id"] = *deploymentID + config.Parameters["cloud_deployment_id"] = *deploymentID for _, resource := range res.Resources { kind := resource.Kind if kind == nil { @@ -154,7 +163,7 @@ func (cp *cloudProvider) BootUp(options Options) error { for { deployment, err := deploymentapi.Get(deploymentapi.GetParams{ - API: api, + API: cp.api, DeploymentID: *deploymentID, }) if err != nil { @@ -196,6 +205,135 @@ func (cp *cloudProvider) BootUp(options Options) error { return nil } +func (cp *cloudProvider) TearDown(options Options) error { + deployment, err := cp.currentDeployment() + if err != nil { + return fmt.Errorf("failed to find current deployment: %w", err) + } + if deployment.ID == nil { + return fmt.Errorf("deployment doesn't have id?") + } + + logger.Debugf("Deleting deployment %q", *deployment.ID) + + _, err = deploymentapi.Shutdown(deploymentapi.ShutdownParams{ + API: cp.api, + DeploymentID: *deployment.ID, + SkipSnapshot: true, + }) + if err != nil { + return fmt.Errorf("failed to shutdown deployment: %w", err) + } + + /* + _, err = deploymentapi.Delete(deploymentapi.DeleteParams{ + API: cp.api, + DeploymentID: *deployment.ID, + }) + if err != nil { + return fmt.Errorf("failed to remove deployment: %w", err) + } + */ + + return nil +} + +func (*cloudProvider) Update(options Options) error { + fmt.Println("Nothing to do.") + return nil +} + +func (*cloudProvider) Dump(options DumpOptions) (string, error) { + return "", fmt.Errorf("not implemented") +} + +func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { + deployment, err := cp.currentDeployment() + if err != nil { + return nil, err + } + + healthStatus := func(healthy *bool) string { + if healthy != nil && *healthy { + return "healthy" + } + return "unhealthy" + } + + var status []ServiceStatus + for _, resource := range deployment.Resources.Elasticsearch { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("elasticsearch-%d", i) + } else { + name = fmt.Sprintf("elasticsearch-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + for _, resource := range deployment.Resources.Kibana { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("kibana-%d", i) + } else { + name = fmt.Sprintf("kibana-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + for _, resource := range deployment.Resources.IntegrationsServer { + for i, instance := range resource.Info.Topology.Instances { + var name string + if instance.InstanceName == nil { + name = fmt.Sprintf("integrations-server-%d", i) + } else { + name = fmt.Sprintf("integrations-server-%s", *instance.InstanceName) + } + status = append(status, ServiceStatus{ + Name: name, + Version: instance.ServiceVersion, + Status: healthStatus(instance.Healthy), + }) + } + } + return status, nil +} + +func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { + config, err := loadConfig(cp.profile) + if err != nil { + return nil, err + } + deploymentID, found := config.Parameters[paramCloudDeploymentID] + if !found { + return nil, deploymentNotExistErr + } + deployment, err := deploymentapi.Get(deploymentapi.GetParams{ + API: cp.api, + DeploymentID: deploymentID, + }) + if err != nil { + return nil, fmt.Errorf("couldn't check deployment health: %w", err) + } + + // It seems that terminated deployments still exist, but hidden. + if hidden := deployment.Metadata.Hidden; hidden != nil && *hidden { + return nil, deploymentNotExistErr + } + + return deployment, nil +} + func (*cloudProvider) getServiceURL(resourcesResponse any) (string, error) { // Converting back and forth for easier access. var resources []struct { @@ -222,20 +360,3 @@ func (*cloudProvider) getServiceURL(resourcesResponse any) (string, error) { } return "", fmt.Errorf("url not found") } - -func (*cloudProvider) TearDown(options Options) error { - return fmt.Errorf("not implemented") -} - -func (*cloudProvider) Update(options Options) error { - fmt.Println("Nothing to do.") - return nil -} - -func (*cloudProvider) Dump(options DumpOptions) (string, error) { - return "", fmt.Errorf("not implemented") -} - -func (*cloudProvider) Status(options Options) ([]ServiceStatus, error) { - return nil, fmt.Errorf("not implemented") -} From 118362ea964ff34184f35e3af8359d91365cd4ef Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 16:44:48 +0100 Subject: [PATCH 21/35] Wait for cluster to be ready --- internal/stack/cloud.go | 102 ++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index cd54493446..6f75e19c84 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -11,17 +11,17 @@ import ( "net/http" "net/url" "os" - "time" "github.com/elastic/cloud-sdk-go/pkg/api" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" "github.com/elastic/cloud-sdk-go/pkg/auth" "github.com/elastic/cloud-sdk-go/pkg/models" + "github.com/elastic/cloud-sdk-go/pkg/plan" + "github.com/elastic/cloud-sdk-go/pkg/plan/planutil" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/profile" - "github.com/elastic/elastic-package/internal/signal" ) const ( @@ -89,6 +89,7 @@ func (cp *cloudProvider) BootUp(options Options) error { region := "gcp-europe-west3" templateID := "gcp-io-optimized" + logger.Debugf("Getting deployment template %q", templateID) template, err := deptemplateapi.Get(deptemplateapi.GetParams{ API: cp.api, TemplateID: templateID, @@ -115,6 +116,7 @@ func (cp *cloudProvider) BootUp(options Options) error { plan.DeploymentTemplate.ID = &templateID } + logger.Debugf("Creating deployment %q", name) res, err := deploymentapi.Create(deploymentapi.CreateParams{ API: cp.api, Request: payload, @@ -138,6 +140,7 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("deployment created, but couldn't get its ID, check in the console UI") } config.Parameters["cloud_deployment_id"] = *deploymentID + for _, resource := range res.Resources { kind := resource.Kind if kind == nil { @@ -155,53 +158,45 @@ func (cp *cloudProvider) BootUp(options Options) error { } } - // Storing once before getting the endpoints, so we have the ID. - err = storeConfig(cp.profile, config) + deployment, err := deploymentapi.Get(deploymentapi.GetParams{ + API: cp.api, + DeploymentID: *deploymentID, + }) if err != nil { - return fmt.Errorf("failed to store config: %w", err) + return fmt.Errorf("couldn't check deployment health: %w", err) } - for { - deployment, err := deploymentapi.Get(deploymentapi.GetParams{ - API: cp.api, - DeploymentID: *deploymentID, - }) - if err != nil { - return fmt.Errorf("couldn't check deployment health: %w", err) - } - - if healthy := deployment.Healthy; healthy == nil || !*healthy { - if signal.SIGINT() { - return fmt.Errorf("wait interrupted") - } - time.Sleep(1 * time.Second) - continue - } - - // TODO: Check that resources are healthy too. - - config.ElasticsearchHost, err = cp.getServiceURL(deployment.Resources.Elasticsearch) - if err != nil { - return fmt.Errorf("failed to get elasticsearch host: %w", err) - } - config.KibanaHost, err = cp.getServiceURL(deployment.Resources.Kibana) - if err != nil { - return fmt.Errorf("failed to get kibana host: %w", err) - } - config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) - if err != nil { - return fmt.Errorf("failed to get fleet host: %w", err) - } - - break + config.ElasticsearchHost, err = cp.getServiceURL(deployment.Resources.Elasticsearch) + if err != nil { + return fmt.Errorf("failed to get elasticsearch host: %w", err) + } + config.KibanaHost, err = cp.getServiceURL(deployment.Resources.Kibana) + if err != nil { + return fmt.Errorf("failed to get kibana host: %w", err) + } + config.Parameters["fleet_url"], err = cp.getServiceURL(deployment.Resources.IntegrationsServer) + if err != nil { + return fmt.Errorf("failed to get fleet host: %w", err) } - // Store the configuration again now with the service urls. err = storeConfig(cp.profile, config) if err != nil { return fmt.Errorf("failed to store config: %w", err) } + logger.Debug("Waiting for creation plan to be completed") + err = planutil.TrackChange(planutil.TrackChangeParams{ + TrackChangeParams: plan.TrackChangeParams{ + API: cp.api, + DeploymentID: *deploymentID, + }, + Writer: &cloudTrackWriter{}, + Format: "text", + }) + if err != nil { + return fmt.Errorf("failed to track cluster creation", err) + } + return nil } @@ -225,16 +220,6 @@ func (cp *cloudProvider) TearDown(options Options) error { return fmt.Errorf("failed to shutdown deployment: %w", err) } - /* - _, err = deploymentapi.Delete(deploymentapi.DeleteParams{ - API: cp.api, - DeploymentID: *deployment.ID, - }) - if err != nil { - return fmt.Errorf("failed to remove deployment: %w", err) - } - */ - return nil } @@ -253,12 +238,22 @@ func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { return nil, err } + status, _ := cp.deploymentStatus(deployment) + return status, nil +} + +func (*cloudProvider) deploymentStatus(deployment *models.DeploymentGetResponse) ([]ServiceStatus, bool) { + allHealthy := true healthStatus := func(healthy *bool) string { if healthy != nil && *healthy { return "healthy" } + allHealthy = false return "unhealthy" } + if healthy := deployment.Healthy; healthy == nil || !*healthy { + allHealthy = false + } var status []ServiceStatus for _, resource := range deployment.Resources.Elasticsearch { @@ -306,7 +301,7 @@ func (cp *cloudProvider) Status(options Options) ([]ServiceStatus, error) { }) } } - return status, nil + return status, allHealthy } func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { @@ -360,3 +355,10 @@ func (*cloudProvider) getServiceURL(resourcesResponse any) (string, error) { } return "", fmt.Errorf("url not found") } + +type cloudTrackWriter struct{} + +func (*cloudTrackWriter) Write(p []byte) (n int, err error) { + logger.Debug(string(p)) + return len(p), nil +} From 280b0ab504d2a1928e90530c70a92cd5c5913aeb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 17:26:21 +0100 Subject: [PATCH 22/35] Print urls only when available --- cmd/stack.go | 23 ++++------------------- internal/stack/boot.go | 26 +++++++++++++++----------- internal/stack/cloud.go | 7 +++++++ internal/stack/config.go | 10 ++++++++++ internal/stack/options.go | 1 + internal/stack/providers.go | 7 +++++++ 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/cmd/stack.go b/cmd/stack.go index 51d836baac..3c9d9750b4 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -88,20 +88,14 @@ func setupStackCommand() *cobraext.Command { return err } - // Print information before starting the stack, for cases where - // this is executed in the foreground, without daemon mode. cmd.Printf("Using profile %s.\n", profile.ProfilePath) cmd.Println(`Remember to load stack environment variables using 'eval "$(elastic-package stack shellinit)"'.`) - err = printInitConfig(cmd, profile) - if err != nil { - return err - } - err = provider.BootUp(stack.Options{ DaemonMode: daemonMode, StackVersion: stackVersion, Services: services, Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "booting up the stack failed") @@ -134,6 +128,7 @@ func setupStackCommand() *cobraext.Command { err = provider.TearDown(stack.Options{ Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "tearing down the stack failed") @@ -168,6 +163,7 @@ func setupStackCommand() *cobraext.Command { err = provider.Update(stack.Options{ StackVersion: stackVersion, Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "failed updating the stack images") @@ -263,6 +259,7 @@ func setupStackCommand() *cobraext.Command { servicesStatus, err := provider.Status(stack.Options{ Profile: profile, + Printer: cmd, }) if err != nil { return errors.Wrap(err, "failed getting stack status") @@ -319,18 +316,6 @@ func validateServicesFlag(services []string) error { return nil } -func printInitConfig(cmd *cobra.Command, profile *profile.Profile) error { - initConfig, err := stack.StackInitConfig(profile) - if err != nil { - return nil - } - cmd.Printf("Elasticsearch host: %s\n", initConfig.ElasticsearchHostPort) - cmd.Printf("Kibana host: %s\n", initConfig.KibanaHostPort) - cmd.Printf("Username: %s\n", initConfig.ElasticsearchUsername) - cmd.Printf("Password: %s\n", initConfig.ElasticsearchPassword) - return nil -} - func printStatus(cmd *cobra.Command, servicesStatus []stack.ServiceStatus) { if len(servicesStatus) == 0 { cmd.Printf(" - No service running\n") diff --git a/internal/stack/boot.go b/internal/stack/boot.go index 77d1fc39e3..a6422ca426 100644 --- a/internal/stack/boot.go +++ b/internal/stack/boot.go @@ -23,6 +23,18 @@ const DockerComposeProjectName = "elastic-package-stack" // BootUp function boots up the Elastic stack. func BootUp(options Options) error { + // Print information before starting the stack, for cases where + // this is executed in the foreground, without daemon mode. + config := Config{ + Provider: ProviderCompose, + ElasticsearchHost: "https://127.0.0.1:9200", + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, + KibanaHost: "https://127.0.0.1:5601", + CACertFile: options.Profile.Path(profileStackPath, CACertificateFile), + } + printUserConfig(options.Printer, config) + buildPackagesPath, found, err := builder.FindBuildPackagesDirectory() if err != nil { return errors.Wrap(err, "finding build packages directory failed") @@ -46,11 +58,11 @@ func BootUp(options Options) error { } } - fmt.Println("Packages from the following directories will be loaded into the package-registry:") - fmt.Println("- built-in packages (package-storage:snapshot Docker image)") + options.Printer.Println("Packages from the following directories will be loaded into the package-registry:") + options.Printer.Println("- built-in packages (package-storage:snapshot Docker image)") if found { - fmt.Printf("- %s\n", buildPackagesPath) + options.Printer.Printf("- %s\n", buildPackagesPath) } err = applyResources(options.Profile, options.StackVersion) @@ -68,14 +80,6 @@ func BootUp(options Options) error { return errors.Wrap(err, "running docker-compose failed") } - config := Config{ - Provider: ProviderCompose, - ElasticsearchHost: "https://127.0.0.1:9200", - ElasticsearchUsername: elasticsearchUsername, - ElasticsearchPassword: elasticsearchPassword, - KibanaHost: "https://127.0.0.1:5601", - CACertFile: options.Profile.Path(profileStackPath, CACertificateFile), - } err = storeConfig(options.Profile, config) if err != nil { return errors.Wrap(err, "failed to store config") diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index 6f75e19c84..f6fa3d6e42 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -79,6 +79,11 @@ func (cp *cloudProvider) BootUp(options Options) error { if err == nil { // Do nothing, deployment already exists. // TODO: Migrate configuration if changed. + config, err := loadConfig(cp.profile) + if err != nil { + return err + } + printUserConfig(options.Printer, config) return nil } else if err != nil && err != deploymentNotExistErr { return err @@ -179,6 +184,8 @@ func (cp *cloudProvider) BootUp(options Options) error { return fmt.Errorf("failed to get fleet host: %w", err) } + printUserConfig(options.Printer, config) + err = storeConfig(cp.profile, config) if err != nil { return fmt.Errorf("failed to store config: %w", err) diff --git a/internal/stack/config.go b/internal/stack/config.go index 67c4976443..ab7a1b0d0f 100644 --- a/internal/stack/config.go +++ b/internal/stack/config.go @@ -67,3 +67,13 @@ func storeConfig(profile *profile.Profile, config Config) error { return nil } + +func printUserConfig(printer Printer, config Config) { + if printer == nil { + return + } + printer.Printf("Elasticsearch host: %s\n", config.ElasticsearchHost) + printer.Printf("Kibana host: %s\n", config.KibanaHost) + printer.Printf("Username: %s\n", config.ElasticsearchUsername) + printer.Printf("Password: %s\n", config.ElasticsearchPassword) +} diff --git a/internal/stack/options.go b/internal/stack/options.go index 9fb2631675..a1ed877826 100644 --- a/internal/stack/options.go +++ b/internal/stack/options.go @@ -14,4 +14,5 @@ type Options struct { Services []string Profile *profile.Profile + Printer Printer } diff --git a/internal/stack/providers.go b/internal/stack/providers.go index 7ad12fbd47..81d4090aea 100644 --- a/internal/stack/providers.go +++ b/internal/stack/providers.go @@ -24,6 +24,13 @@ var ( } ) +// Printer is the interface that can be used to print information on operations. +type Printer interface { + Print(i ...interface{}) + Println(i ...interface{}) + Printf(format string, i ...interface{}) +} + // Provider is the implementation of a stack provider. type Provider interface { // BootUp starts a stack. From 092eeeed49e79dff372d40e13ae36609537cc66e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 18:18:43 +0100 Subject: [PATCH 23/35] Get provider from profile, not from flag, except for stack up --- cmd/stack.go | 35 +++++++++++++++++++++++------------ internal/stack/cloud.go | 5 +++-- internal/stack/config.go | 8 +++++++- internal/stack/initconfig.go | 2 +- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/stack.go b/cmd/stack.go index 3c9d9750b4..0f160748d1 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -44,7 +44,7 @@ By default the latest released version of the stack is spun up but it is possibl Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold. -To ęxpose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. +To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).` @@ -83,7 +83,7 @@ func setupStackCommand() *cobraext.Command { return err } - provider, err := getProviderFlag(cmd, profile) + provider, err := getProviderFromProfile(cmd, profile, true) if err != nil { return err } @@ -109,6 +109,7 @@ func setupStackCommand() *cobraext.Command { upCommand.Flags().StringSliceP(cobraext.StackServicesFlagName, "s", nil, fmt.Sprintf(cobraext.StackServicesFlagDescription, strings.Join(availableServicesAsList(), ","))) upCommand.Flags().StringP(cobraext.StackVersionFlagName, "", install.DefaultStackVersion, cobraext.StackVersionFlagDescription) + upCommand.Flags().String(cobraext.StackProviderFlagName, "", fmt.Sprintf(cobraext.StackProviderFlagDescription, strings.Join(stack.SupportedProviders, ", "))) downCommand := &cobra.Command{ Use: "down", @@ -121,7 +122,7 @@ func setupStackCommand() *cobraext.Command { return err } - provider, err := getProviderFlag(cmd, profile) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { return err } @@ -150,7 +151,7 @@ func setupStackCommand() *cobraext.Command { return err } - provider, err := getProviderFlag(cmd, profile) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { return err } @@ -222,7 +223,7 @@ func setupStackCommand() *cobraext.Command { return err } - provider, err := getProviderFlag(cmd, profile) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { return err } @@ -252,7 +253,7 @@ func setupStackCommand() *cobraext.Command { return err } - provider, err := getProviderFlag(cmd, profile) + provider, err := getProviderFromProfile(cmd, profile, false) if err != nil { return err } @@ -277,7 +278,6 @@ func setupStackCommand() *cobraext.Command { Long: stackLongDescription, } cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar)) - cmd.PersistentFlags().String(cobraext.StackProviderFlagName, "compose", fmt.Sprintf(cobraext.StackProviderFlagDescription, strings.Join(stack.SupportedProviders, ", "))) cmd.AddCommand( upCommand, downCommand, @@ -376,13 +376,24 @@ func getProfileFlag(cmd *cobra.Command) (*profile.Profile, error) { return p, nil } -func getProviderFlag(cmd *cobra.Command, profile *profile.Profile) (stack.Provider, error) { - providerName, err := cmd.Flags().GetString(cobraext.StackProviderFlagName) +func getProviderFromProfile(cmd *cobra.Command, profile *profile.Profile, checkFlag bool) (stack.Provider, error) { + var providerName = stack.DefaultProvider + stackConfig, err := stack.LoadConfig(profile) if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.StackProviderFlagName) + return nil, err } - if providerName == "" { - providerName = stack.DefaultProvider + if stackConfig.Provider != "" { + providerName = stackConfig.Provider + } + + if checkFlag { + providerFlag, err := cmd.Flags().GetString(cobraext.StackProviderFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.StackProviderFlagName) + } + if providerFlag != "" { + providerName = providerFlag + } } return stack.BuildProvider(providerName, profile) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index f6fa3d6e42..ea299f9cac 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -79,7 +79,7 @@ func (cp *cloudProvider) BootUp(options Options) error { if err == nil { // Do nothing, deployment already exists. // TODO: Migrate configuration if changed. - config, err := loadConfig(cp.profile) + config, err := LoadConfig(cp.profile) if err != nil { return err } @@ -137,6 +137,7 @@ func (cp *cloudProvider) BootUp(options Options) error { } var config Config + config.Provider = ProviderCloud config.Parameters = map[string]string{ "cloud_deployment_alias": res.Alias, } @@ -312,7 +313,7 @@ func (*cloudProvider) deploymentStatus(deployment *models.DeploymentGetResponse) } func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, error) { - config, err := loadConfig(cp.profile) + config, err := LoadConfig(cp.profile) if err != nil { return nil, err } diff --git a/internal/stack/config.go b/internal/stack/config.go index ab7a1b0d0f..aae6f559b0 100644 --- a/internal/stack/config.go +++ b/internal/stack/config.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "github.com/elastic/elastic-package/internal/profile" ) @@ -36,7 +37,7 @@ func defaultConfig() Config { } } -func loadConfig(profile *profile.Profile) (Config, error) { +func LoadConfig(profile *profile.Profile) (Config, error) { d, err := os.ReadFile(configPath(profile)) if errors.Is(err, os.ErrNotExist) { return defaultConfig(), nil @@ -60,6 +61,11 @@ func storeConfig(profile *profile.Profile, config Config) error { return fmt.Errorf("failed to encode stack config: %w", err) } + err = os.MkdirAll(filepath.Dir(configPath(profile)), 0755) + if err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + err = os.WriteFile(configPath(profile), d, 0644) if err != nil { return fmt.Errorf("failed to write stack config: %s", err) diff --git a/internal/stack/initconfig.go b/internal/stack/initconfig.go index 4711883b29..916db46be8 100644 --- a/internal/stack/initconfig.go +++ b/internal/stack/initconfig.go @@ -17,7 +17,7 @@ type InitConfig struct { } func StackInitConfig(profile *profile.Profile) (*InitConfig, error) { - config, err := loadConfig(profile) + config, err := LoadConfig(profile) if err != nil { return nil, err } From 210a2f10509038622dfd47ababb7b3f328a5bf5f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 2 Dec 2022 18:32:42 +0100 Subject: [PATCH 24/35] Return hard-coded shellinit default for backwards compatibility --- internal/stack/config.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/stack/config.go b/internal/stack/config.go index aae6f559b0..e6509f5755 100644 --- a/internal/stack/config.go +++ b/internal/stack/config.go @@ -31,16 +31,23 @@ func configPath(profile *profile.Profile) string { return profile.Path(profileStackPath, configFileName) } -func defaultConfig() Config { +func defaultConfig(profile *profile.Profile) Config { return Config{ Provider: DefaultProvider, + + // Hard-coded default values for backwards-compatibility. + ElasticsearchHost: "https://127.0.0.1:9200", + ElasticsearchUsername: elasticsearchUsername, + ElasticsearchPassword: elasticsearchPassword, + KibanaHost: "https://127.0.0.1:5601", + CACertFile: profile.Path(profileStackPath, CACertificateFile), } } func LoadConfig(profile *profile.Profile) (Config, error) { d, err := os.ReadFile(configPath(profile)) if errors.Is(err, os.ErrNotExist) { - return defaultConfig(), nil + return defaultConfig(profile), nil } if err != nil { return Config{}, fmt.Errorf("failed to read stack config: %w", err) From 7737fec94705d6a5e93b2b9cc0f15944adc6bde5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 5 Dec 2022 12:41:49 +0100 Subject: [PATCH 25/35] Linting --- internal/stack/cloud.go | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/internal/stack/cloud.go b/internal/stack/cloud.go index ea299f9cac..52a61b6296 100644 --- a/internal/stack/cloud.go +++ b/internal/stack/cloud.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net/http" - "net/url" "os" "github.com/elastic/cloud-sdk-go/pkg/api" @@ -27,29 +26,12 @@ import ( const ( paramCloudDeploymentID = "cloud_deployment_id" paramCloudDeploymentAlias = "cloud_deployment_alias" - - // Docs: https://www.elastic.co/guide/en/cloud/current/ec-api-deployment-crud.html - cloudAPI = "https://api.elastic-cloud.com/api/v1" ) var ( - deploymentNotExistErr = errors.New("deployment does not exist") - deploymentAlreadyExistErr = errors.New("deployment already exists") + errDeploymentNotExist = errors.New("deployment does not exist") ) -var cloudDeploymentsAPI string - -func init() { - mustJoinURL := func(base string, elem ...string) string { - joined, err := url.JoinPath(base, elem...) - if err != nil { - panic(err) - } - return joined - } - cloudDeploymentsAPI = mustJoinURL(cloudAPI, "deployments") -} - type cloudProvider struct { api *api.API profile *profile.Profile @@ -85,7 +67,7 @@ func (cp *cloudProvider) BootUp(options Options) error { } printUserConfig(options.Printer, config) return nil - } else if err != nil && err != deploymentNotExistErr { + } else if err != nil && err != errDeploymentNotExist { return err } @@ -202,7 +184,7 @@ func (cp *cloudProvider) BootUp(options Options) error { Format: "text", }) if err != nil { - return fmt.Errorf("failed to track cluster creation", err) + return fmt.Errorf("failed to track cluster creation: %w", err) } return nil @@ -319,7 +301,7 @@ func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, err } deploymentID, found := config.Parameters[paramCloudDeploymentID] if !found { - return nil, deploymentNotExistErr + return nil, errDeploymentNotExist } deployment, err := deploymentapi.Get(deploymentapi.GetParams{ API: cp.api, @@ -331,7 +313,7 @@ func (cp *cloudProvider) currentDeployment() (*models.DeploymentGetResponse, err // It seems that terminated deployments still exist, but hidden. if hidden := deployment.Metadata.Hidden; hidden != nil && *hidden { - return nil, deploymentNotExistErr + return nil, errDeploymentNotExist } return deployment, nil From 060b76fe95560f1dfca9689fa9275ea19d9b2653 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 19 Dec 2022 17:43:12 +0100 Subject: [PATCH 26/35] Fix nil pointer dereference --- internal/install/stack_version.go | 2 +- internal/stack/compose.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/install/stack_version.go b/internal/install/stack_version.go index 1f742edd3c..a1cc3c362b 100644 --- a/internal/install/stack_version.go +++ b/internal/install/stack_version.go @@ -6,5 +6,5 @@ package install const ( // DefaultStackVersion is the default version of the stack - DefaultStackVersion = "8.5.1" + DefaultStackVersion = "8.5.3" ) diff --git a/internal/stack/compose.go b/internal/stack/compose.go index e380cd492c..52b556ff7c 100644 --- a/internal/stack/compose.go +++ b/internal/stack/compose.go @@ -220,7 +220,7 @@ func newServiceStatus(description *docker.ContainerDescription) (*ServiceStatus, Status: description.State.Status, Version: getVersionFromDockerImage(description.Config.Image), } - if description.State.Status == "running" { + if description.State.Status == "running" && description.State.Health != nil { service.Status = fmt.Sprintf("%v (%v)", service.Status, description.State.Health.Status) } if description.State.Status == "exited" { From ed1e62bb887d7ce409efe4a4bf04e1634bd42c12 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 19 Dec 2022 19:34:04 +0100 Subject: [PATCH 27/35] Update go-resource --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c74d2b2661..072d2b7e7f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/elastic/cloud-sdk-go v1.11.0 github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/elastic/go-licenser v0.4.1 - github.com/elastic/go-resource v0.1.0 + github.com/elastic/go-resource v0.1.1 github.com/elastic/go-sysinfo v1.9.0 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/package-spec/v2 v2.2.0 diff --git a/go.sum b/go.sum index 1e25cac7e6..80df61fa2d 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73El github.com/elastic/go-elasticsearch/v7 v7.17.7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= -github.com/elastic/go-resource v0.1.0 h1:7is06Dsz1EdgjTzSZ1PcR8MYO7sypHTin0BZqHV4h8s= -github.com/elastic/go-resource v0.1.0/go.mod h1:7F1Wjs6eSFX0i/235yAK/x9bvPNd9/ML92AiULa4XYA= +github.com/elastic/go-resource v0.1.1 h1:vM173uhPoaZ7C64rKrYbbBn5kxOhstE1+YcGFteAKh8= +github.com/elastic/go-resource v0.1.1/go.mod h1:7F1Wjs6eSFX0i/235yAK/x9bvPNd9/ML92AiULa4XYA= github.com/elastic/go-sysinfo v1.9.0 h1:usICqY/Nw4Mpn9f4LdtpFrKxXroJDe81GaxxUlCckIo= github.com/elastic/go-sysinfo v1.9.0/go.mod h1:eBD1wEGVaRnRLGecc9iG1z8eOv5HnEdz9+nWd8UAxcE= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= From b5dbf6a01d2d65143808bd077fdaa9bacf364a80 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 20 Dec 2022 13:23:20 +0100 Subject: [PATCH 28/35] Update gobuffalo/packr --- go.mod | 2 ++ go.sum | 37 ++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index b80baba3c3..63c9d8070d 100644 --- a/go.mod +++ b/go.mod @@ -173,3 +173,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/gobuffalo/packr/v2 => github.com/gobuffalo/packr/v2 v2.3.2 diff --git a/go.sum b/go.sum index edbe4b6949..2096ba3fe8 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,7 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -79,7 +80,11 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -133,6 +138,7 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSY github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -268,13 +274,9 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= @@ -282,12 +284,10 @@ github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxs github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/packr/v2 v2.3.2/go.mod h1:93elRVdDhpUgox9GnXswWK5dzpVBQsnlQjnnncSLoiU= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -352,6 +352,7 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= @@ -375,7 +376,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= @@ -411,6 +411,7 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -441,6 +442,7 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -477,6 +479,7 @@ github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -500,10 +503,10 @@ github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -514,19 +517,23 @@ github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3 github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -550,6 +557,7 @@ github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+Kd github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -574,6 +582,7 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -602,7 +611,7 @@ go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sf go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20221205180719-3fd0dac74452 h1:JZtNuL6LPB+scU5yaQ6hqRlJFRiddZm2FwRt2AQqtHA= go.starlark.net v0.0.0-20221205180719-3fd0dac74452/go.mod h1:kIVgS18CjmEC3PqMd5kaJSGEifyV/CeB9x506ZJ1Vbk= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -680,8 +689,6 @@ golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -692,6 +699,7 @@ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -765,11 +773,10 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190603152906-08e0b306e832/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From d400e9d19246bee45284c1914fb8fb6cdbf52b8b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 21 Dec 2022 11:53:25 +0100 Subject: [PATCH 29/35] Fix custom agent deployer --- internal/configuration/locations/locations.go | 19 ++------ .../_static/docker-custom-agent-base.yml | 0 .../runners/system/servicedeployer/compose.go | 18 +++---- .../system/servicedeployer/custom_agent.go | 48 +++++++++++++++---- 4 files changed, 50 insertions(+), 35 deletions(-) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/docker-custom-agent-base.yml (100%) diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 6c912019ec..210fc60266 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -26,18 +26,15 @@ const ( fieldsCachedDir = "cache/fields" terraformDeployerYmlFile = "terraform-deployer.yml" - - dockerCustomAgentDeployerYmlFile = "docker-custom-agent-base.yml" ) var ( // elasticPackageDataHome is the name of the environment variable used to override data folder for elastic-package elasticPackageDataHome = environment.WithElasticPackagePrefix("DATA_HOME") - serviceLogsDir = filepath.Join(temporaryDir, "service_logs") - kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") - terraformDeployerDir = filepath.Join(deployerDir, "terraform") - dockerCustomAgentDeployerDir = filepath.Join(deployerDir, "docker_custom_agent") + serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") + terraformDeployerDir = filepath.Join(deployerDir, "terraform") ) // LocationManager maintains an instance of a config path location @@ -101,16 +98,6 @@ func (loc LocationManager) TerraformDeployerYml() string { return filepath.Join(loc.stackPath, terraformDeployerDir, terraformDeployerYmlFile) } -// DockerCustomAgentDeployerDir returns the DockerCustomAgent Directory -func (loc LocationManager) DockerCustomAgentDeployerDir() string { - return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir) -} - -// DockerCustomAgentDeployerYml returns the DockerCustomAgent deployer yml file -func (loc LocationManager) DockerCustomAgentDeployerYml() string { - return filepath.Join(loc.stackPath, dockerCustomAgentDeployerDir, dockerCustomAgentDeployerYmlFile) -} - // ServiceLogDir returns the log directory func (loc LocationManager) ServiceLogDir() string { return filepath.Join(loc.stackPath, serviceLogsDir) diff --git a/internal/install/_static/docker-custom-agent-base.yml b/internal/testrunner/runners/system/servicedeployer/_static/docker-custom-agent-base.yml similarity index 100% rename from internal/install/_static/docker-custom-agent-base.yml rename to internal/testrunner/runners/system/servicedeployer/_static/docker-custom-agent-base.yml diff --git a/internal/testrunner/runners/system/servicedeployer/compose.go b/internal/testrunner/runners/system/servicedeployer/compose.go index ade566b3b1..b462e39375 100644 --- a/internal/testrunner/runners/system/servicedeployer/compose.go +++ b/internal/testrunner/runners/system/servicedeployer/compose.go @@ -24,7 +24,7 @@ import ( // a Docker Compose file. type DockerComposeServiceDeployer struct { ymlPaths []string - sv ServiceVariant + variant ServiceVariant } type dockerComposeDeployedService struct { @@ -32,14 +32,14 @@ type dockerComposeDeployedService struct { ymlPaths []string project string - sv ServiceVariant + variant ServiceVariant } // NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. func NewDockerComposeServiceDeployer(ymlPaths []string, sv ServiceVariant) (*DockerComposeServiceDeployer, error) { return &DockerComposeServiceDeployer{ ymlPaths: ymlPaths, - sv: sv, + variant: sv, }, nil } @@ -49,7 +49,7 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer service := dockerComposeDeployedService{ ymlPaths: d.ymlPaths, project: "elastic-package-service", - sv: d.sv, + variant: d.variant, } outCtxt := inCtxt @@ -71,15 +71,15 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer } // Boot up service - if d.sv.active() { - logger.Infof("Using service variant: %s", d.sv.String()) + if d.variant.active() { + logger.Infof("Using service variant: %s", d.variant.String()) } serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, - d.sv.Env...), + d.variant.Env...), ExtraArgs: []string{"--build", "-d"}, } err = p.Up(opts) @@ -161,14 +161,14 @@ func (s *dockerComposeDeployedService) TearDown() error { opts := compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, - s.sv.Env...), + s.variant.Env...), } processServiceContainerLogs(p, opts, s.ctxt.Name) if err := p.Down(compose.CommandOptions{ Env: append( []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, - s.sv.Env...), + s.variant.Env...), ExtraArgs: []string{"--volumes"}, // Remove associated volumes. }); err != nil { return errors.Wrap(err, "could not shut down service using Docker Compose") diff --git a/internal/testrunner/runners/system/servicedeployer/custom_agent.go b/internal/testrunner/runners/system/servicedeployer/custom_agent.go index 206452c2e1..02165d0e00 100644 --- a/internal/testrunner/runners/system/servicedeployer/custom_agent.go +++ b/internal/testrunner/runners/system/servicedeployer/custom_agent.go @@ -8,6 +8,7 @@ import ( _ "embed" "fmt" "os" + "path/filepath" "github.com/pkg/errors" @@ -21,18 +22,25 @@ import ( "github.com/elastic/elastic-package/internal/stack" ) -const dockerCustomAgentName = "docker-custom-agent" +const ( + dockerCustomAgentName = "docker-custom-agent" + dockerCustomAgentDir = "docker_custom_agent" + dockerCustomAgentDockerfile = "docker-custom-agent-base.yml" +) + +//go:embed _static/docker-custom-agent-base.yml +var dockerCustomAgentDockerfileContent []byte // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via // a Docker Compose file. type CustomAgentDeployer struct { - cfg string + dockerComposeFile string } // NewCustomAgentDeployer returns a new instance of a deployedCustomAgent. -func NewCustomAgentDeployer(cfgPath string) (*CustomAgentDeployer, error) { +func NewCustomAgentDeployer(dockerComposeFile string) (*CustomAgentDeployer, error) { return &CustomAgentDeployer{ - cfg: cfgPath, + dockerComposeFile: dockerComposeFile, }, nil } @@ -66,15 +74,20 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err fmt.Sprintf("%s=%s", localCACertEnv, caCertPath), ) - ymlPaths, err := d.loadComposeDefinitions() + configDir, err := d.installDockerfile() if err != nil { - return nil, err + return nil, errors.Wrap(err, "could not create resources for custom agent") + } + + ymlPaths := []string{ + d.dockerComposeFile, + filepath.Join(configDir, dockerCustomAgentDockerfile), } service := dockerComposeDeployedService{ ymlPaths: ymlPaths, project: "elastic-package-service", - sv: ServiceVariant{ + variant: ServiceVariant{ Name: dockerCustomAgentName, Env: env, }, @@ -149,10 +162,25 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err return &service, nil } -func (d *CustomAgentDeployer) loadComposeDefinitions() ([]string, error) { +// installDockerfile creates the files needed to run the custom elastic agent and returns +// the directory with these files. +func (d *CustomAgentDeployer) installDockerfile() (string, error) { locationManager, err := locations.NewLocationManager() if err != nil { - return nil, errors.Wrap(err, "can't locate Docker Compose file for Custom Agent deployer") + return "", errors.Wrap(err, "failed to find the configuration directory") } - return []string{d.cfg, locationManager.DockerCustomAgentDeployerYml()}, nil + + customAgentDir := filepath.Join(locationManager.DeployerDir(), dockerCustomAgentDir) + err = os.MkdirAll(customAgentDir, 0755) + if err != nil { + return "", errors.Wrap(err, "failed to create directory for custom agent files") + } + + customAgentDockerfile := filepath.Join(customAgentDir, dockerCustomAgentDockerfile) + err = os.WriteFile(customAgentDockerfile, dockerCustomAgentDockerfileContent, 0644) + if err != nil { + return "", errors.Wrap(err, "failed to create docker compose file for custom agent") + } + + return customAgentDir, nil } From a17fcdd9619e60e598b5c99fba5c3ed255bfd344 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 1 Feb 2023 14:44:44 +0100 Subject: [PATCH 30/35] Add variant for Kibana >= 8.7.0 with Fleet experimental toggles enabled (#1128) --- .ci/Jenkinsfile | 1 + Makefile | 8 ++++++-- internal/stack/_static/kibana.yml.tmpl | 4 ++++ internal/stack/variants.go | 4 +++- internal/stack/variants_test.go | 20 ++++++++++++-------- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 278b4f8b76..193bb4198e 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -99,6 +99,7 @@ pipeline { 'stack-command-default': generateTestCommandStage(command: 'test-stack-command-default', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-oldest': generateTestCommandStage(command: 'test-stack-command-oldest', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-7x': generateTestCommandStage(command: 'test-stack-command-7x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), + 'stack-command-86': generateTestCommandStage(command: 'test-stack-command-86', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'stack-command-8x': generateTestCommandStage(command: 'test-stack-command-8x', artifacts: ['build/elastic-stack-dump/stack/*/logs/*.log', 'build/elastic-stack-dump/stack/*/logs/fleet-server-internal/*', 'build/elastic-stack-status/*/*']), 'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), 'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true), diff --git a/Makefile b/Makefile index 23ee32d965..60f1559f52 100644 --- a/Makefile +++ b/Makefile @@ -58,10 +58,14 @@ test-stack-command-oldest: ./scripts/test-stack-command.sh 7.14.2 test-stack-command-7x: - ./scripts/test-stack-command.sh 7.17.3-SNAPSHOT + ./scripts/test-stack-command.sh 7.17.8 + +# Keeping a test for 8.6 because it has an specific configuration file. +test-stack-command-86: + ./scripts/test-stack-command.sh 8.6.1 test-stack-command-8x: - ./scripts/test-stack-command.sh 8.6.0-SNAPSHOT + ./scripts/test-stack-command.sh 8.7.0-SNAPSHOT test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index e40db946c7..95def56276 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -28,6 +28,10 @@ xpack.fleet.registryUrl: "https://package-registry:8080" xpack.fleet.agents.enabled: true xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] +{{ if not (semverLessThan $version "8.7.0") }} +xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI +{{ end }} + xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" {{ if not (semverLessThan $version "8.1.0") }} diff --git a/internal/stack/variants.go b/internal/stack/variants.go index 986c875691..15bdef78bc 100644 --- a/internal/stack/variants.go +++ b/internal/stack/variants.go @@ -14,13 +14,15 @@ import ( // This map is used to deploy different versions of the Elastic stack with matching configurations. var configurationVariantMap = map[string]string{ "8.0-0 - 8.1.x-x": "80", - "^8.2-0": "8x", + "8.2-0 - 8.6.x-x": "86", + "^8.7-0": "8x", } // stackVariantAsEnv function returns a stack variant based on the given stack version. // We identified three variants: // * default, covers all of 7.x branches // * 80, covers stack versions 8.0.0 to 8.1.x +// * 86, covers stack versions 8.2.0 to 8.6.x, they don't support experimental toggles in Fleet // * 8x, supports different configuration options in Kibana, covers stack versions 8.2.0+ func stackVariantAsEnv(version string) string { return fmt.Sprintf("STACK_VERSION_VARIANT=%s", selectStackVersion(version)) diff --git a/internal/stack/variants_test.go b/internal/stack/variants_test.go index 5b5e5a6aa7..143389a573 100644 --- a/internal/stack/variants_test.go +++ b/internal/stack/variants_test.go @@ -32,14 +32,18 @@ var tests = []struct { {"8.1.58", "80"}, {"8.1.99-beta", "80"}, {"8.1.999-SNAPSHOT", "80"}, - {"8.2-0", "8x"}, - {"8.2", "8x"}, - {"8.2.0-alpha", "8x"}, - {"8.2.0", "8x"}, - {"8.2.58", "8x"}, - {"8.2.99-gamma", "8x"}, - {"8.2.777-SNAPSHOT+arm64", "8x"}, - {"8.5", "8x"}, + {"8.2-0", "86"}, + {"8.2", "86"}, + {"8.2.0-alpha", "86"}, + {"8.2.0", "86"}, + {"8.2.58", "86"}, + {"8.2.99-gamma", "86"}, + {"8.2.777-SNAPSHOT+arm64", "86"}, + {"8.5", "86"}, + {"8.6.1", "86"}, + {"8.7.0", "8x"}, + {"8.7.0-SNAPSHOT", "8x"}, + {"8.7.1-SNAPSHOT", "8x"}, {"9", "default"}, } From 12e4139985ac06550cb03a8df0bb449b10641a5f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 9 Feb 2023 17:47:07 +0100 Subject: [PATCH 31/35] Fix terraform deployer --- internal/configuration/locations/locations.go | 13 ----- .../_static/terraform_deployer.yml | 0 .../_static/terraform_deployer_run.sh | 0 .../system/servicedeployer/terraform.go | 53 ++++++++++++++----- 4 files changed, 40 insertions(+), 26 deletions(-) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/terraform_deployer.yml (100%) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/terraform_deployer_run.sh (100%) diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 210fc60266..e6252e75a8 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -24,8 +24,6 @@ const ( deployerDir = "deployer" fieldsCachedDir = "cache/fields" - - terraformDeployerYmlFile = "terraform-deployer.yml" ) var ( @@ -34,7 +32,6 @@ var ( serviceLogsDir = filepath.Join(temporaryDir, "service_logs") kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") - terraformDeployerDir = filepath.Join(deployerDir, "terraform") ) // LocationManager maintains an instance of a config path location @@ -88,16 +85,6 @@ func (loc LocationManager) KubernetesDeployerDir() string { return filepath.Join(loc.stackPath, kubernetesDeployerDir) } -// TerraformDeployerDir returns the Terraform Directory -func (loc LocationManager) TerraformDeployerDir() string { - return filepath.Join(loc.stackPath, terraformDeployerDir) -} - -// TerraformDeployerYml returns the Terraform deployer yml file -func (loc LocationManager) TerraformDeployerYml() string { - return filepath.Join(loc.stackPath, terraformDeployerDir, terraformDeployerYmlFile) -} - // ServiceLogDir returns the log directory func (loc LocationManager) ServiceLogDir() string { return filepath.Join(loc.stackPath, serviceLogsDir) diff --git a/internal/install/_static/terraform_deployer.yml b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml similarity index 100% rename from internal/install/_static/terraform_deployer.yml rename to internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer.yml diff --git a/internal/install/_static/terraform_deployer_run.sh b/internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh similarity index 100% rename from internal/install/_static/terraform_deployer_run.sh rename to internal/testrunner/runners/system/servicedeployer/_static/terraform_deployer_run.sh diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 47279581a4..2c3423b4b1 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -5,6 +5,7 @@ package servicedeployer import ( + _ "embed" "os" "path/filepath" @@ -16,6 +17,18 @@ import ( "github.com/elastic/elastic-package/internal/logger" ) +const ( + terraformDeployerDir = "terraform" + terraformDeployerYml = "terraform-deployer.yml" + terraformDeployerRun = "run.sh" +) + +//go:embed _static/terraform_deployer.yml +var terraformDeployerYmlContent []byte + +//go:embed _static/terraform_deployer_run.sh +var terraformDeployerRunContent []byte + // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { definitionsDir string @@ -23,6 +36,7 @@ type TerraformServiceDeployer struct { // NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer. func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { + logger.Debug("%+v", definitionsDir) return &TerraformServiceDeployer{ definitionsDir: definitionsDir, }, nil @@ -32,11 +46,18 @@ func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeploy func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { logger.Debug("setting up service using Terraform deployer") - ymlPaths, err := tsd.loadComposeDefinitions() + configDir, err := tsd.installDockerfile() if err != nil { return nil, errors.Wrap(err, "can't load Docker Compose definitions") } + ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)} + envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) + _, err = os.Stat(envYmlPath) + if err == nil { + ymlPaths = append(ymlPaths, envYmlPath) + } + service := dockerComposeDeployedService{ ymlPaths: ymlPaths, project: "elastic-package-service", @@ -89,25 +110,31 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return &service, nil } -func (tsd TerraformServiceDeployer) loadComposeDefinitions() ([]string, error) { +func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { locationManager, err := locations.NewLocationManager() if err != nil { - return nil, errors.Wrap(err, "can't locate Docker Compose file for Terraform deployer") + return "", errors.Wrap(err, "failed to find the configuration directory") } - envYmlPath := filepath.Join(tsd.definitionsDir, envYmlFile) - _, err = os.Stat(envYmlPath) - if errors.Is(err, os.ErrNotExist) { - return []string{ - locationManager.TerraformDeployerYml(), - }, nil + tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir) + err = os.MkdirAll(tfDir, 0755) + if err != nil { + return "", errors.Wrap(err, "failed to create directory for terraform deployer files") } + + deployerYmlFile := filepath.Join(tfDir, terraformDeployerYml) + err = os.WriteFile(deployerYmlFile, terraformDeployerYmlContent, 0644) if err != nil { - return nil, errors.Wrapf(err, "stat failed (path: %s)", envYmlPath) + return "", errors.Wrap(err, "failed to create terraform deployer yaml") } - return []string{ - locationManager.TerraformDeployerYml(), envYmlPath, - }, nil + + deployerRunFile := filepath.Join(tfDir, terraformDeployerRun) + err = os.WriteFile(deployerRunFile, terraformDeployerRunContent, 0644) + if err != nil { + return "", errors.Wrap(err, "failed to create terraform deployer run script") + } + + return tfDir, nil } var _ ServiceDeployer = new(TerraformServiceDeployer) From 1b9fb1e1369dea71182f597e1aafd14228498579 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 9 Feb 2023 18:23:01 +0100 Subject: [PATCH 32/35] Fix terraform deployer dockerfile --- .../_static/Dockerfile.terraform_deployer | 0 .../system/servicedeployer/terraform.go | 62 +++++++++++++------ 2 files changed, 43 insertions(+), 19 deletions(-) rename internal/{install => testrunner/runners/system/servicedeployer}/_static/Dockerfile.terraform_deployer (100%) diff --git a/internal/install/_static/Dockerfile.terraform_deployer b/internal/testrunner/runners/system/servicedeployer/_static/Dockerfile.terraform_deployer similarity index 100% rename from internal/install/_static/Dockerfile.terraform_deployer rename to internal/testrunner/runners/system/servicedeployer/_static/Dockerfile.terraform_deployer diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 2c3423b4b1..81b9655924 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -6,8 +6,10 @@ package servicedeployer import ( _ "embed" + "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" @@ -15,19 +17,24 @@ import ( "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/go-resource" ) const ( - terraformDeployerDir = "terraform" - terraformDeployerYml = "terraform-deployer.yml" - terraformDeployerRun = "run.sh" + terraformDeployerDir = "terraform" + terraformDeployerYml = "terraform-deployer.yml" + terraformDeployerDockerfile = "Dockerfile" + terraformDeployerRun = "run.sh" ) //go:embed _static/terraform_deployer.yml -var terraformDeployerYmlContent []byte +var terraformDeployerYmlContent string //go:embed _static/terraform_deployer_run.sh -var terraformDeployerRunContent []byte +var terraformDeployerRunContent string + +//go:embed _static/Dockerfile.terraform_deployer +var terraformDeployerDockerfileContent string // TerraformServiceDeployer is responsible for deploying infrastructure described with Terraform definitions. type TerraformServiceDeployer struct { @@ -36,7 +43,6 @@ type TerraformServiceDeployer struct { // NewTerraformServiceDeployer creates an instance of TerraformServiceDeployer. func NewTerraformServiceDeployer(definitionsDir string) (*TerraformServiceDeployer, error) { - logger.Debug("%+v", definitionsDir) return &TerraformServiceDeployer{ definitionsDir: definitionsDir, }, nil @@ -75,8 +81,12 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic return nil, errors.Wrap(err, "removing service logs failed") } + tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) + // Set custom aliases, which may be used in agent policies. - serviceComposeConfig, err := p.Config(compose.CommandOptions{}) + serviceComposeConfig, err := p.Config(compose.CommandOptions{ + Env: tfEnvironment, + }) if err != nil { return nil, errors.Wrap(err, "could not get Docker Compose configuration for service") } @@ -86,7 +96,6 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic } // Boot up service - tfEnvironment := tsd.buildTerraformExecutorEnvironment(inCtxt) opts := compose.CommandOptions{ Env: tfEnvironment, ExtraArgs: []string{"--build", "-d"}, @@ -117,21 +126,36 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { } tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir) - err = os.MkdirAll(tfDir, 0755) - if err != nil { - return "", errors.Wrap(err, "failed to create directory for terraform deployer files") - } - deployerYmlFile := filepath.Join(tfDir, terraformDeployerYml) - err = os.WriteFile(deployerYmlFile, terraformDeployerYmlContent, 0644) - if err != nil { - return "", errors.Wrap(err, "failed to create terraform deployer yaml") + resources := []resource.Resource{ + &resource.File{ + Path: terraformDeployerYml, + Content: resource.FileContentLiteral(terraformDeployerYmlContent), + }, + &resource.File{ + Path: terraformDeployerRun, + Content: resource.FileContentLiteral(terraformDeployerRunContent), + }, + &resource.File{ + Path: terraformDeployerDockerfile, + Content: resource.FileContentLiteral(terraformDeployerDockerfileContent), + }, } - deployerRunFile := filepath.Join(tfDir, terraformDeployerRun) - err = os.WriteFile(deployerRunFile, terraformDeployerRunContent, 0644) + resourceManager := resource.NewManager() + resourceManager.RegisterProvider("file", &resource.FileProvider{ + Prefix: tfDir, + }) + + results, err := resourceManager.Apply(resources) if err != nil { - return "", errors.Wrap(err, "failed to create terraform deployer run script") + var errors []string + for _, result := range results { + if err := result.Err(); err != nil { + errors = append(errors, err.Error()) + } + } + return "", fmt.Errorf("%w: %s", err, strings.Join(errors, ", ")) } return tfDir, nil From 77bc1d59d467bcd21b06a964cddde0be7ee3fb17 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 9 Feb 2023 18:33:16 +0100 Subject: [PATCH 33/35] Remove unused kibana config --- internal/profile/_static/kibana_config_80.yml | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 internal/profile/_static/kibana_config_80.yml diff --git a/internal/profile/_static/kibana_config_80.yml b/internal/profile/_static/kibana_config_80.yml deleted file mode 100644 index 4e3c72ec05..0000000000 --- a/internal/profile/_static/kibana_config_80.yml +++ /dev/null @@ -1,59 +0,0 @@ -server.name: kibana -server.host: "0.0.0.0" -server.ssl.enabled: true -server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" -server.ssl.key: "/usr/share/kibana/config/certs/key.pem" -server.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca-cert.pem"] - -elasticsearch.hosts: [ "https://elasticsearch:9200" ] -elasticsearch.ssl.certificateAuthorities: "/usr/share/kibana/config/certs/ca-cert.pem" -elasticsearch.serviceAccountToken: "AAEAAWVsYXN0aWMva2liYW5hL2VsYXN0aWMtcGFja2FnZS1raWJhbmEtdG9rZW46b2x4b051SWNRa0tYMHdXazdLWmFBdw" - -monitoring.ui.container.elasticsearch.enabled: true - -xpack.fleet.registryUrl: "https://package-registry:8080" -xpack.fleet.agents.enabled: true -xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch:9200"] -xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server:8220"] - -xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" - -xpack.fleet.packages: - - name: system - version: latest - - name: elastic_agent - version: latest - - name: fleet_server - version: latest -xpack.fleet.agentPolicies: - - name: Elastic-Agent (elastic-package) - id: elastic-agent-managed-ep - is_default: true - is_managed: false - namespace: default - monitoring_enabled: - - logs - - metrics - package_policies: - - name: system-1 - id: default-system - package: - name: system - - name: Fleet Server (elastic-package) - id: fleet-server-policy - is_default_fleet_server: true - is_managed: false - namespace: default - package_policies: - - name: fleet_server-1 - id: default-fleet-server - package: - name: fleet_server -xpack.fleet.outputs: - - id: fleet-default-output - name: default - type: elasticsearch - hosts: [ https://elasticsearch:9200 ] - ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" - is_default: true - is_default_monitoring: true From 065571394602ce3a254f8ca1f0e333e61cd1a82a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 9 Feb 2023 18:36:10 +0100 Subject: [PATCH 34/35] Fix linting --- .../testrunner/runners/system/servicedeployer/terraform.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index 81b9655924..bd01432a1e 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -13,11 +13,12 @@ import ( "github.com/pkg/errors" + "github.com/elastic/go-resource" + "github.com/elastic/elastic-package/internal/compose" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/files" "github.com/elastic/elastic-package/internal/logger" - "github.com/elastic/go-resource" ) const ( From b5b5b26b4794fcaadcca13a13a1accc2fb88cf87 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 9 Feb 2023 19:46:09 +0100 Subject: [PATCH 35/35] Fix creation of parent directories --- .../runners/system/servicedeployer/terraform.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/testrunner/runners/system/servicedeployer/terraform.go b/internal/testrunner/runners/system/servicedeployer/terraform.go index bd01432a1e..329a18b4e9 100644 --- a/internal/testrunner/runners/system/servicedeployer/terraform.go +++ b/internal/testrunner/runners/system/servicedeployer/terraform.go @@ -55,7 +55,7 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic configDir, err := tsd.installDockerfile() if err != nil { - return nil, errors.Wrap(err, "can't load Docker Compose definitions") + return nil, errors.Wrap(err, "can't install Docker Compose definitions") } ymlPaths := []string{filepath.Join(configDir, terraformDeployerYml)} @@ -130,16 +130,19 @@ func (tsd TerraformServiceDeployer) installDockerfile() (string, error) { resources := []resource.Resource{ &resource.File{ - Path: terraformDeployerYml, - Content: resource.FileContentLiteral(terraformDeployerYmlContent), + Path: terraformDeployerYml, + Content: resource.FileContentLiteral(terraformDeployerYmlContent), + CreateParent: true, }, &resource.File{ - Path: terraformDeployerRun, - Content: resource.FileContentLiteral(terraformDeployerRunContent), + Path: terraformDeployerRun, + Content: resource.FileContentLiteral(terraformDeployerRunContent), + CreateParent: true, }, &resource.File{ - Path: terraformDeployerDockerfile, - Content: resource.FileContentLiteral(terraformDeployerDockerfileContent), + Path: terraformDeployerDockerfile, + Content: resource.FileContentLiteral(terraformDeployerDockerfileContent), + CreateParent: true, }, }