diff --git a/CHANGELOG.md b/CHANGELOG.md index 52776f1..53138a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.8.0 (2024-11-09) + +- Added option for command specific authentication + ## 1.7.7 (2024-10-09) - Supported version pinning for providers(aws, gcp, azure and etc) in `manifest` file diff --git a/LICENSE b/LICENSE index 81436d1..f0e0b3c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 StackQL Studios +Copyright (c) 2022-2025 StackQL Studios Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/examples/confluent/cmd-specific-auth/README.md b/examples/confluent/cmd-specific-auth/README.md new file mode 100644 index 0000000..e56f49d --- /dev/null +++ b/examples/confluent/cmd-specific-auth/README.md @@ -0,0 +1,63 @@ +# `stackql-deploy` starter project for `aws` + +> for starter projects using other providers, try `stackql-deploy cmd-specific-auth --provider=azure` or `stackql-deploy cmd-specific-auth --provider=google` + +see the following links for more information on `stackql`, `stackql-deploy` and the `aws` provider: + +- [`aws` provider docs](https://stackql.io/registry/aws) +- [`stackql`](https://github.com/stackql/stackql) +- [`stackql-deploy` PyPI home page](https://pypi.org/project/stackql-deploy/) +- [`stackql-deploy` GitHub repo](https://github.com/stackql/stackql-deploy) + +## Overview + +__`stackql-deploy`__ is a stateless, declarative, SQL driven Infrastructure-as-Code (IaC) framework. There is no state file required as the current state is assessed for each resource at runtime. __`stackql-deploy`__ is capable of provisioning, deprovisioning and testing a stack which can include resources across different providers, like a stack spanning `aws` and `azure` for example. + +## Prerequisites + +This example requires `stackql-deploy` to be installed using __`pip install stackql-deploy`__. The host used to run `stackql-deploy` needs the necessary environment variables set to authenticate to your specific provider, in the case of the `aws` provider, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optionally `AWS_SESSION_TOKEN` must be set, for more information on authentication to `aws` see the [`aws` provider documentation](https://aws.stackql.io/providers/aws). + +## Usage + +Adjust the values in the [__`stackql_manifest.yml`__](stackql_manifest.yml) file if desired. The [__`stackql_manifest.yml`__](stackql_manifest.yml) file contains resource configuration variables to support multiple deployment environments, these will be used for `stackql` queries in the `resources` folder. + +The syntax for the `stackql-deploy` command is as follows: + +```bash +stackql-deploy { build | test | teardown } { stack-directory } { deployment environment} [ optional flags ] +``` + +### Deploying a stack + +For example, to deploy the stack named cmd-specific-auth to an environment labeled `sit`, run the following: + +```bash +stackql-deploy build cmd-specific-auth sit \ +-e AWS_REGION=ap-southeast-2 +``` + +Use the `--dry-run` flag to view the queries to be run without actually running them, for example: + +```bash +stackql-deploy build cmd-specific-auth sit \ +-e AWS_REGION=ap-southeast-2 \ +--dry-run +``` + +### Testing a stack + +To test a stack to ensure that all resources are present and in the desired state, run the following (in our `sit` deployment example): + +```bash +stackql-deploy test cmd-specific-auth sit \ +-e AWS_REGION=ap-southeast-2 +``` + +### Tearing down a stack + +To destroy or deprovision all resources in a stack for our `sit` deployment example, run the following: + +```bash +stackql-deploy teardown cmd-specific-auth sit \ +-e AWS_REGION=ap-southeast-2 +``` \ No newline at end of file diff --git a/examples/confluent/cmd-specific-auth/resources/example_vpc.iql b/examples/confluent/cmd-specific-auth/resources/example_vpc.iql new file mode 100644 index 0000000..463dbc1 --- /dev/null +++ b/examples/confluent/cmd-specific-auth/resources/example_vpc.iql @@ -0,0 +1,67 @@ +/* defines the provisioning and deprovisioning commands +used to create, update or delete the resource +replace queries with your queries */ + +/*+ exists */ +SELECT COUNT(*) as count FROM +( +SELECT vpc_id, +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t; + +/*+ create */ +INSERT INTO aws.ec2.vpcs ( + CidrBlock, + Tags, + EnableDnsSupport, + EnableDnsHostnames, + region +) +SELECT + '{{ vpc_cidr_block }}', + '{{ vpc_tags }}', + true, + true, + '{{ region }}'; + +/*+ statecheck, retries=5, retry_delay=5 */ +SELECT COUNT(*) as count FROM +( +SELECT vpc_id, +cidr_block, +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t +WHERE cidr_block = '{{ vpc_cidr_block }}'; + +/*+ exports, retries=5, retry_delay=5 */ +SELECT vpc_id, vpc_cidr_block FROM +( +SELECT vpc_id, cidr_block as "vpc_cidr_block", +json_group_object(tag_key, tag_value) as tags +FROM aws.ec2.vpc_tags +WHERE region = '{{ region }}' +AND cidr_block = '{{ vpc_cidr_block }}' +GROUP BY vpc_id +HAVING json_extract(tags, '$.Provisioner') = 'stackql' +AND json_extract(tags, '$.StackName') = '{{ stack_name }}' +AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' +) t; + +/*+ delete */ +DELETE FROM aws.ec2.vpcs +WHERE data__Identifier = '{{ vpc_id }}' +AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/confluent/cmd-specific-auth/stackql_manifest.yml b/examples/confluent/cmd-specific-auth/stackql_manifest.yml new file mode 100644 index 0000000..7450964 --- /dev/null +++ b/examples/confluent/cmd-specific-auth/stackql_manifest.yml @@ -0,0 +1,40 @@ +# +# aws starter project manifest file, add and update values as needed +# +version: 1 +name: "cmd-specific-auth" +description: description for "cmd-specific-auth" +providers: + - aws +globals: + - name: region + description: aws region + value: "{{ AWS_REGION }}" + - name: global_tags + value: + - Key: Provisioner + Value: stackql + - Key: StackName + Value: "{{ stack_name }}" + - Key: StackEnv + Value: "{{ stack_env }}" +resources: + - name: example_vpc + description: example vpc resource + props: + - name: vpc_cidr_block + values: + prd: + value: "10.0.0.0/16" + sit: + value: "10.1.0.0/16" + dev: + value: "10.2.0.0/16" + - name: vpc_tags + value: + - Key: Name + Value: "{{ stack_name }}-{{ stack_env }}-vpc" + merge: ['global_tags'] + exports: + - vpc_id + - vpc_cidr_block \ No newline at end of file diff --git a/setup.py b/setup.py index f5b040b..6b086b8 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='stackql-deploy', - version='1.7.7', + version='1.8.0', description='Model driven resource provisioning and deployment framework using StackQL.', long_description=readme, long_description_content_type='text/x-rst', diff --git a/stackql_deploy/__init__.py b/stackql_deploy/__init__.py index ed2901d..ce77fa0 100644 --- a/stackql_deploy/__init__.py +++ b/stackql_deploy/__init__.py @@ -1 +1 @@ -__version__ = '1.7.7' +__version__ = '1.8.0' diff --git a/stackql_deploy/cli.py b/stackql_deploy/cli.py index 372b2f3..b3b450a 100644 --- a/stackql_deploy/cli.py +++ b/stackql_deploy/cli.py @@ -80,7 +80,14 @@ def parse_env_var(ctx, param, value): return env_vars def setup_logger(command, args_dict): - log_level = args_dict.get('log_level', 'INFO') + log_level = args_dict.get('log_level', 'INFO').upper() # Normalize to uppercase + valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'} + + if log_level not in valid_levels: + raise click.ClickException( + f"Invalid log level: {log_level}. Valid levels are: {', '.join(valid_levels)}" + ) + logger.setLevel(log_level) logger.debug(f"'{command}' command called with args: {str(args_dict)}") diff --git a/stackql_deploy/cmd/base.py b/stackql_deploy/cmd/base.py index 73b38bc..ece348f 100644 --- a/stackql_deploy/cmd/base.py +++ b/stackql_deploy/cmd/base.py @@ -29,9 +29,40 @@ def __init__(self, stackql, vars, logger, stack_dir, stack_env): self.logger ) + def process_custom_auth( + self, + resource, + full_context + ): + custom_auth = resource.get('auth', {}) + env_vars = {} + + if custom_auth: + self.logger.info(f"🔑 custom auth is configured for [{resource['name']}]") + + # Function to recursively search for keys of interest and populate env_vars + def extract_env_vars(auth_config): + for key, value in auth_config.items(): + if key in {"username_var", "password_var", "credentialsenvvar", "keyIDenvvar"}: + # Retrieve the variable's value from full_context + env_var_name = value + env_var_value = full_context.get(env_var_name) + if env_var_value: + env_vars[env_var_name] = env_var_value + elif isinstance(value, dict): + # Recursively check nested dictionaries + extract_env_vars(value) + + # Start extracting env vars from custom_auth + extract_env_vars(custom_auth) + + # If no custom auth, return None for both custom_auth and env_vars + return (custom_auth if custom_auth else None, env_vars if env_vars else None) + def process_exports( self, resource, + full_context, exports_query, exports_retries, exports_retry_delay, @@ -66,13 +97,16 @@ def process_exports( else: self.logger.info(f"📦 exporting variables for [{resource['name']}]...") show_query(show_queries, exports_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) exports = run_stackql_query( exports_query, self.stackql, True, self.logger, - exports_retries, - exports_retry_delay + custom_auth=custom_auth, + env_vars=env_vars, + retries=exports_retries, + delay=exports_retry_delay ) self.logger.debug(f"exports: {exports}") @@ -123,6 +157,7 @@ def check_if_resource_exists( self, resource_exists, resource, + full_context, exists_query, exists_retries, exists_retry_delay, @@ -141,6 +176,7 @@ def check_if_resource_exists( else: self.logger.info(f"🔎 running {check_type} check for [{resource['name']}]...") show_query(show_queries, exists_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) resource_exists = perform_retries( resource, exists_query, @@ -148,7 +184,9 @@ def check_if_resource_exists( exists_retry_delay, self.stackql, self.logger, - delete_test + delete_test, + custom_auth=custom_auth, + env_vars=env_vars ) else: self.logger.info(f"{check_type} check not configured for [{resource['name']}]") @@ -160,6 +198,7 @@ def check_if_resource_is_correct_state( self, is_correct_state, resource, + full_context, statecheck_query, statecheck_retries, statecheck_retry_delay, @@ -174,13 +213,17 @@ def check_if_resource_is_correct_state( else: self.logger.info(f"🔎 running state check for [{resource['name']}]...") show_query(show_queries, statecheck_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) is_correct_state = perform_retries( resource, statecheck_query, statecheck_retries, statecheck_retry_delay, self.stackql, - self.logger + self.logger, + False, + custom_auth=custom_auth, + env_vars=env_vars ) if is_correct_state: self.logger.info(f"👍 [{resource['name']}] is in the desired state") @@ -195,6 +238,7 @@ def create_resource( self, is_created_or_updated, resource, + full_context, create_query, create_retries, create_retry_delay, @@ -209,10 +253,13 @@ def create_resource( else: self.logger.info(f"[{resource['name']}] does not exist, creating 🚧...") show_query(show_queries, create_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) msg = run_stackql_command( create_query, self.stackql, self.logger, + custom_auth=custom_auth, + env_vars=env_vars, ignore_errors=ignore_errors, retries=create_retries, retry_delay=create_retry_delay @@ -225,6 +272,7 @@ def update_resource( self, is_created_or_updated, resource, + full_context, update_query, update_retries, update_retry_delay, @@ -238,10 +286,13 @@ def update_resource( else: self.logger.info(f"🔧 updating [{resource['name']}]...") show_query(show_queries, update_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) msg = run_stackql_command( update_query, self.stackql, self.logger, + custom_auth=custom_auth, + env_vars=env_vars, ignore_errors=ignore_errors, retries=update_retries, retry_delay=update_retry_delay @@ -255,6 +306,7 @@ def update_resource( def delete_resource( self, resource, + full_context, delete_query, delete_retries, delete_retry_delay, @@ -268,10 +320,13 @@ def delete_resource( else: self.logger.info(f"🚧 deleting [{resource['name']}]...") show_query(show_queries, delete_query, self.logger) + custom_auth, env_vars = self.process_custom_auth(resource, full_context) msg = run_stackql_command( delete_query, self.stackql, self.logger, + custom_auth=custom_auth, + env_vars=env_vars, ignore_errors=ignore_errors, retries=delete_retries, retry_delay=delete_retry_delay diff --git a/stackql_deploy/cmd/build.py b/stackql_deploy/cmd/build.py index c4d2a13..d5fe343 100644 --- a/stackql_deploy/cmd/build.py +++ b/stackql_deploy/cmd/build.py @@ -119,6 +119,7 @@ def run(self, dry_run, show_queries, on_failure): resource_exists = self.check_if_resource_exists( resource_exists, resource, + full_context, exists_query, exists_retries, exists_retry_delay, @@ -133,6 +134,7 @@ def run(self, dry_run, show_queries, on_failure): is_correct_state = self.check_if_resource_is_correct_state( is_correct_state, resource, + full_context, statecheck_query, statecheck_retries, statecheck_retry_delay, @@ -152,6 +154,7 @@ def run(self, dry_run, show_queries, on_failure): is_created_or_updated = self.create_resource( is_created_or_updated, resource, + full_context, create_query, create_retries, create_retry_delay, @@ -167,6 +170,7 @@ def run(self, dry_run, show_queries, on_failure): is_created_or_updated = self.update_resource( is_created_or_updated, resource, + full_context, update_query, update_retries, update_retry_delay, @@ -182,6 +186,7 @@ def run(self, dry_run, show_queries, on_failure): is_correct_state = self.check_if_resource_is_correct_state( is_correct_state, resource, + full_context, statecheck_query, statecheck_retries, statecheck_retry_delay, @@ -205,6 +210,7 @@ def run(self, dry_run, show_queries, on_failure): if exports_query: self.process_exports( resource, + full_context, exports_query, exports_retries, exports_retry_delay, diff --git a/stackql_deploy/cmd/teardown.py b/stackql_deploy/cmd/teardown.py index cc4f113..44bf769 100644 --- a/stackql_deploy/cmd/teardown.py +++ b/stackql_deploy/cmd/teardown.py @@ -28,6 +28,7 @@ def collect_exports(self, show_queries, dry_run): if exports_query: self.process_exports( resource, + full_context, exports_query, exports_retries, exports_retry_delay, @@ -107,7 +108,14 @@ def run(self, dry_run, show_queries, on_failure): ignore_errors = True # multi resources ignore errors on create or update elif type == 'resource': resource_exists = self.check_if_resource_exists( - resource_exists, resource, exists_query, exists_retries, exists_retry_delay, dry_run, show_queries + resource_exists, + resource, + full_context, + exists_query, + exists_retries, + exists_retry_delay, + dry_run, + show_queries ) # @@ -115,7 +123,14 @@ def run(self, dry_run, show_queries, on_failure): # if resource_exists: self.delete_resource( - resource, delete_query, delete_retries, delete_retry_delay, dry_run, show_queries, ignore_errors + resource, + full_context, + delete_query, + delete_retries, + delete_retry_delay, + dry_run, + show_queries, + ignore_errors ) else: self.logger.info(f"resource [{resource['name']}] does not exist, skipping delete") @@ -127,6 +142,7 @@ def run(self, dry_run, show_queries, on_failure): resource_deleted = self.check_if_resource_exists( False, resource, + full_context, exists_query, postdelete_exists_retries, postdelete_exists_retry_delay, diff --git a/stackql_deploy/cmd/test.py b/stackql_deploy/cmd/test.py index 8757bcf..b146e75 100644 --- a/stackql_deploy/cmd/test.py +++ b/stackql_deploy/cmd/test.py @@ -48,7 +48,14 @@ def run(self, dry_run, show_queries, on_failure): if type in ('resource', 'multi'): is_correct_state = self.check_if_resource_is_correct_state( - False, resource, statecheck_query, statecheck_retries, statecheck_retry_delay, dry_run, show_queries + False, + resource, + full_context, + statecheck_query, + statecheck_retries, + statecheck_retry_delay, + dry_run, + show_queries ) if not is_correct_state and not dry_run: @@ -59,7 +66,7 @@ def run(self, dry_run, show_queries, on_failure): # if exports_query: self.process_exports( - resource, exports_query, exports_retries, exports_retry_delay, dry_run, show_queries + resource, full_context, exports_query, exports_retries, exports_retry_delay, dry_run, show_queries ) if type == 'resource' and not dry_run: diff --git a/stackql_deploy/lib/utils.py b/stackql_deploy/lib/utils.py index 787cc53..805fe17 100644 --- a/stackql_deploy/lib/utils.py +++ b/stackql_deploy/lib/utils.py @@ -15,12 +15,12 @@ def get_type(resource, logger): else: return type -def run_stackql_query(query, stackql, suppress_errors, logger, retries=0, delay=5): +def run_stackql_query(query, stackql, suppress_errors, logger, custom_auth=None, env_vars=None, retries=0, delay=5): attempt = 0 while attempt <= retries: try: logger.debug(f"(utils.run_stackql_query) executing stackql query on attempt {attempt + 1}:\n\n{query}\n") - result = stackql.execute(query, suppress_errors) + result = stackql.execute(query, suppress_errors=suppress_errors, custom_auth=custom_auth, env_vars=env_vars) logger.debug(f"(utils.run_stackql_query) stackql query result (type:{type(result)}): {result}") # Check if result is a list (expected outcome) @@ -102,7 +102,15 @@ def error_detected(result): return True return False -def run_stackql_command(command, stackql, logger, ignore_errors=False, retries=0, retry_delay=5): +def run_stackql_command(command, + stackql, + logger, + custom_auth=None, + env_vars=None, + ignore_errors=False, + retries=0, + retry_delay=5 + ): attempt = 0 while attempt <= retries: try: @@ -125,7 +133,7 @@ def run_stackql_command(command, stackql, logger, ignore_errors=False, retries=0 ) ) - result = stackql.executeStmt(command) + result = stackql.executeStmt(command, custom_auth, env_vars) logger.debug(f"(utils.run_stackql_command) stackql command result:\n\n{result}, type: {type(result)}\n") if isinstance(result, dict): @@ -291,9 +299,15 @@ def is_installed_version_higher(installed_version, requested_version, logger): logger ) -def run_test(resource, rendered_test_iql, stackql, logger, delete_test=False): +def run_test(resource, rendered_test_iql, stackql, logger, delete_test=False, custom_auth=None, env_vars=None): try: - test_result = run_stackql_query(rendered_test_iql, stackql, True, logger) + test_result = run_stackql_query( + rendered_test_iql, + stackql, + True, + logger, + custom_auth=custom_auth, + env_vars=env_vars) logger.debug(f"(utils.run_test) test query result for [{resource['name']}]:\n\n{test_result}\n") if test_result == []: @@ -338,11 +352,20 @@ def show_query(show_queries, query, logger): if show_queries: logger.info(f"🔍 query:\n\n{query}\n") -def perform_retries(resource, query, retries, delay, stackql, logger, delete_test=False): +def perform_retries(resource, + query, + retries, + delay, + stackql, + logger, + delete_test=False, + custom_auth=None, + env_vars=None + ): attempt = 0 start_time = time.time() # Capture the start time of the operation while attempt < retries: - result = run_test(resource, query, stackql, logger, delete_test) + result = run_test(resource, query, stackql, logger, delete_test, custom_auth=custom_auth, env_vars=env_vars) if result: return True elapsed = time.time() - start_time # Calculate elapsed time diff --git a/website/docs/manifest-file.md b/website/docs/manifest-file.md index 23687dd..28fc31a 100644 --- a/website/docs/manifest-file.md +++ b/website/docs/manifest-file.md @@ -99,6 +99,12 @@ the fields within the __`stackql_manifest.yml`__ file are described in further d *** +### `resource.auth` + + + +*** + ### `resource.exports` diff --git a/website/docs/manifest_fields/index.js b/website/docs/manifest_fields/index.js index 62b4cdc..43f5937 100644 --- a/website/docs/manifest_fields/index.js +++ b/website/docs/manifest_fields/index.js @@ -13,6 +13,7 @@ export { default as ResourceDescription } from './resources/description.mdx'; export { default as ResourceExports } from './resources/exports.mdx'; export { default as ResourceProps } from './resources/props.mdx'; export { default as ResourceProtected } from './resources/protected.mdx'; +export { default as ResourceAuth } from './resources/auth.mdx'; export { default as ResourcePropName } from './resources/props/name.mdx'; export { default as ResourcePropDescription } from './resources/props/description.mdx'; export { default as ResourcePropValue } from './resources/props/value.mdx'; diff --git a/website/docs/manifest_fields/resources/auth.mdx b/website/docs/manifest_fields/resources/auth.mdx new file mode 100644 index 0000000..82ea7e3 --- /dev/null +++ b/website/docs/manifest_fields/resources/auth.mdx @@ -0,0 +1,65 @@ +import File from '@site/src/components/File'; +import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; + +## Custom Authentication at Resource Level + +This feature allows for custom authentication settings to be specified at the resource level within the `stackql_manifest.yml` file. This enables context-specific authentication configurations, such as control plane or data plane context switching within the same stack. Authentication parameters can be overridden by setting specific variable references in the `auth` section. + +:::note + +This feature requires version 1.8.0 of `stackql-deploy` and version 3.7.0 of `pystackql`, use the following to upgrade components: + +```bash +stackql-deploy upgrade +``` + +::: + + + +The `auth` object will depend upon the provider the resource belongs to, consult the provider documentation in the [StackQL Provider Registry Docs](https://stackql.io/registry). + +### Example Usage + + + +```yaml {4,12-18} +resources: + - name: app_manager_api_key + props: + - name: display_name + value: "{{ stack_name }}-{{ stack_env }}-app-manager-api-key" + - name: description + value: "Kafka API Key owned by 'app-manager' service account" + - name: owner + value: + id: app_manager_id + api_version: app_manager_api_version + kind: app_manager_kind + exports: + - app_manager_api_key_id + - app_manager_api_key_secret + + - name: users_topic + auth: + confluent: + type: basic + username_var: app_manager_api_key_id + password_var: app_manager_api_key_secret + props: + - name: topic_name + value: "users" + - name: kafka_cluster + value: {{ cluster_id }} + - name: rest_endpoint + value: {{ cluster_rest_endpoint }} +``` + + + +This configuration sets up a custom `basic` authentication for the `users_topic` resource, where: + +- `username_var` is set to `app_manager_api_key_id` +- `password_var` is set to `app_manager_api_key_secret` + +These variables are defined in the exported section of the `app_manager_api_key` resource and dynamically referenced within the authentication configuration.