diff --git a/tofu/kubernetes/BACKEND.md b/tofu/kubernetes/BACKEND.md new file mode 100644 index 000000000..11ca5cb18 --- /dev/null +++ b/tofu/kubernetes/BACKEND.md @@ -0,0 +1,54 @@ +## Setup Environment + +### Using infisical + +1. Setup [gcloud cli](/DEVCONTAINER.md). +1. Setup TF_VAR_proxmox_api_token, TF_VAR_tofu_encryption_passphrase and save them to infisical. +1. Setup .env file in root folder and commit it to git. +1. Follow devcontainers docs [here](/DEVCONTAINER.md). If done properly, all secrets from infisical will be available in the container environment. + +Note: By default all secrets in /tofu folder will be populated. /tofu_rw is the folder where secrets are written. Also checkout default branch to env mapping. + +## Setup Backend + +### Local + +```shell +# Local Backend +cp samples/backend_local.tofu.sample ./backend.tofu +``` + +```shell +# Local Backend +tofu init +``` + +**Note: If your are using local backend with dev devcontainers and git repo, your state file will be deleted when the container is removed. So be very careful.** + +### R2 + +1. Follow instructions in [Cloudflare R2](../remote_state/cf/README.md) to setup R2 bucket for remote state. + +```shell +# R2 Backend +cp samples/backend_r2.tofu.sample ./backend.tofu +``` + +```shell +# Initialize tofu +tofu init +``` + +### GCS + +1. Follow instructions in [Google Cloud](../state/gcs/README.md) to setup GCS bucket for remote state. + +```shell +# GCS Backend +cp samples/backend_gcs.tofu.sample ./backend.tofu +``` + +```shell +# Initialize tofu +tofu init +``` diff --git a/tofu/kubernetes/README.md b/tofu/kubernetes/README.md index 49fe61296..e461f891a 100644 --- a/tofu/kubernetes/README.md +++ b/tofu/kubernetes/README.md @@ -3,7 +3,7 @@ Read [Talos Kubernetes on Proxmox using OpenTofu](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/) for a more thorough explanation of how everything works. -## Install pre-requisites +## Install pre-requisites - Pre-installed in devContainer 1. [tofu](https://opentofu.org/docs/intro/install/) 2. [talosctl](https://www.talos.dev/v1.9/talos-guides/install/talosctl/) @@ -11,27 +11,10 @@ a more thorough explanation of how everything works. ## Initialize tofu -```shell -tofu init -``` - -## Proxmox - -### Environment variable - -```shell -export TF_VAR_proxmox_api_token="" -``` - -### Optional External Secrets Manager / Other methods +One cluster/state per branch. -**Bitwarden Secrets Manager** - Name your secret TF_VAR_proxmox_api_token in bws. - -```shell -bws run -- tofu ... -``` - -Note: By default, the shell is sh. Change with --shell if required. +1. Setup and initialize [remote backend](BACKEND.md). +1. Keep the environment populated with [required secrets](BACKEND.md) when running `tofu plan/apply`. ## Sealed-secrets @@ -59,27 +42,7 @@ tofu output -raw talos_config ## Upgrading Talos and Kubernetes -[Upgrade](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/#upgrading-the-cluster) talos nodes one by -one. - -1. Set talos_image.auto.tfvars -> image -> update_version to the required update version. -2. Set talos_cluster.auto.tfvars -> talos_cluster_config -> kubernetes_version to the required kubernetes version. -3. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_1 -> update = true and run tofu apply. -4. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_2 -> update = true, leave the previous nodes update = true and - run tofu apply. -5. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_3 -> update = true, leave the previous nodes update = true and - run tofu apply. -6. ... -7. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_n -> update = true, leave the previous nodes update = true and - run tofu apply. -8. After upgrading all nodes, Set talos_image.auto.tfvars -> image -> version to match the update version and set - update = false for all nodes. - -## Upgrading Talos Schematic - -1. Create a new schematic file. -2. Same process as above instead of `image.version` and `image.update_version`, change `image.schematic` and - `image.update_schematic`, in `talos_image.auto.tfvars`. +Follow these [instructions](UPGRADE.md). ## Reuse machine secrets @@ -87,4 +50,4 @@ one. tofu state rm module.talos.talos_machine_secrets.this tofu import module.talos.talos_machine_secrets.this output/talos-machine-secrets.yaml tofu apply --refresh=false -``` \ No newline at end of file +``` diff --git a/tofu/kubernetes/REMOTE_BACKEND.md b/tofu/kubernetes/REMOTE_BACKEND.md deleted file mode 100644 index 9dd569750..000000000 --- a/tofu/kubernetes/REMOTE_BACKEND.md +++ /dev/null @@ -1,42 +0,0 @@ -## GCS Remote - -1. Create a [Service Account](https://cloud.google.com/iam/docs/service-accounts-create) named tofu (after enabling the - IAM API if needed). Leave the permissions blank. -2. Create and download the [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating). -3. Create a GCS bucket for tofu state with public access prevention and versioning as necessary. -4. In the permissions tab of the bucket, give **Storage Object Admin** access to the service account. -5. Copy backend.tf.sample to backend.tf and make necessary changes. - -```shell -cp remote_backend.tf.sample remote_backend.tf -``` - -### Encryption key - -Generate the encryption key - -```shell -python3 -c 'import os;import base64;print(base64.b64encode(os.urandom(32)).decode("utf-8"))' -``` - -`Without the encryption key, your state would not be recoverable. Store in a password manager, if not using any kms like bws.` - -### Environment variables - -```shell -export GOOGLE_APPLICATION_CREDENTIALS="" -export GOOGLE_ENCRYPTION_KEY="" -``` - -Run tofu init / plan / apply as usual. - -### Bitwarden Secrets Manager - -Store the downloaded key contents and generated encryption key into GOOGLE_CREDENTIALS and GOOGLE_ENCRYPTION_KEY -respectively in bws. - -Run bws run -- tofu init / plan / apply as usual. - -### Beta Notice - -`Please treat this as beta and only use for air-gapped installations as of now. Will remove the beta tag after testing it in due course.` diff --git a/tofu/kubernetes/UPGRADE.md b/tofu/kubernetes/UPGRADE.md new file mode 100644 index 000000000..0da516c52 --- /dev/null +++ b/tofu/kubernetes/UPGRADE.md @@ -0,0 +1,22 @@ +## Upgrading Talos +[Upgrade](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/#upgrading-the-cluster) talos nodes one by +one. + +1. Set talos_image.auto.tfvars -> image -> update_version to the required update version. +2. Set talos_cluster.auto.tfvars -> talos_cluster_config -> kubernetes_version to the required kubernetes version. +3. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_1 -> update = true and run tofu apply. +4. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_2 -> update = true, leave the previous nodes update = true and + run tofu apply. +5. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_3 -> update = true, leave the previous nodes update = true and + run tofu apply. +6. ... +7. Set talos_nodes.auto.tfvars -> talos_nodes -> $node_n -> update = true, leave the previous nodes update = true and + run tofu apply. +8. After upgrading all nodes, Set talos_image.auto.tfvars -> image -> version to match the update version and set + update = false for all nodes. + +## Upgrading Talos Schematic + +1. Create a new schematic file. +2. Same process as above instead of `image.version` and `image.update_version`, change `image.schematic` and + `image.update_schematic`, in `talos_image.auto.tfvars`. diff --git a/tofu/kubernetes/backend.tofu b/tofu/kubernetes/backend.tofu new file mode 100644 index 000000000..3af0be55a --- /dev/null +++ b/tofu/kubernetes/backend.tofu @@ -0,0 +1,5 @@ +terraform { + backend "local" { + path = "${var.gcs_prefix_env}.tfstate" + } +} diff --git a/tofu/kubernetes/infisical/git_branch.tofu b/tofu/kubernetes/infisical/git_branch.tofu new file mode 100644 index 000000000..040660087 --- /dev/null +++ b/tofu/kubernetes/infisical/git_branch.tofu @@ -0,0 +1,3 @@ +data "external" "git-branch" { + program = ["/bin/bash", "-c", "jq -n --arg branch `git rev-parse --abbrev-ref HEAD` '{\"branch\":$branch}'"] +} diff --git a/tofu/kubernetes/infisical/infisical.tofu b/tofu/kubernetes/infisical/infisical.tofu new file mode 100644 index 000000000..84b7448fc --- /dev/null +++ b/tofu/kubernetes/infisical/infisical.tofu @@ -0,0 +1,25 @@ +locals { + infisical_environment = lookup(var.infisical.branch_env_mapping, data.external.git-branch.result.branch, "dev") +} + +provider "infisical" { + host = var.infisical_domain + auth = { + universal = { + client_id = var.infisical_client_id + client_secret = var.infisical_client_secret + } + } +} + +resource "infisical_secret" "created_secrets" { + # Create one resource per entry in the input map + for_each = var.secrets_to_create + + name = each.key + value = each.value + + env_slug = local.infisical_environment + folder_path = var.infisical_rw_secrets_path + workspace_id = var.infisical_project_id +} diff --git a/tofu/kubernetes/infisical/variables.tofu b/tofu/kubernetes/infisical/variables.tofu new file mode 100644 index 000000000..3da7b59cb --- /dev/null +++ b/tofu/kubernetes/infisical/variables.tofu @@ -0,0 +1,48 @@ +variable "infisical_domain" { + description = "Infisical Domain" + type = string + default = "https://app.infisical.com" +} + +variable "infisical_client_id" { + description = "Infisical Client ID" + type = string + default = null +} + +variable "infisical_project_id" { + description = "Infisical Project ID" + type = string + default = null +} + +variable "infisical_rw_secrets_path" { + description = "Infisical Client Secret" + type = string + default = "/tofu_rw" +} + +variable "infisical_branch_env_mapping" { + description = "Infisical Branch Environment Mapping" + type = map(string) + default = { + "main" = "prod" + "prod" = "prod" + "staging" = "staging" + "dev" = "dev" + } +} + +variable "infisical_client_secret" { + description = "Infisical Client Secret" + type = string + sensitive = true + default = null +} + +variable "secrets_to_create" { + description = "A map of secrets to create in Infisical. Key: secret name, Value: secret content." + type = map(string) + default = {} + sensitive = true +} diff --git a/tofu/kubernetes/infisical_variables.tofu b/tofu/kubernetes/infisical_variables.tofu new file mode 100644 index 000000000..7876b45f1 --- /dev/null +++ b/tofu/kubernetes/infisical_variables.tofu @@ -0,0 +1,41 @@ +variable "infisical_domain" { + description = "Infisical Domain" + type = string + default = "https://app.infisical.com" +} + +variable "infisical_client_id" { + description = "Infisical Client ID" + type = string + default = null +} + +variable "infisical_project_id" { + description = "Infisical Project ID" + type = string + default = null +} + +variable "infisical_rw_secrets_path" { + description = "Infisical Client Secret" + type = string + default = "/tofu_rw" +} + +variable "infisical_branch_env_mapping" { + description = "Infisical Branch Environment Mapping" + type = map(string) + default = { + "main" = "prod" + "prod" = "prod" + "staging" = "staging" + "dev" = "dev" + } +} + +variable "infisical_client_secret" { + description = "Infisical Client Secret" + type = string + sensitive = true + default = null +} diff --git a/tofu/kubernetes/main.tofu b/tofu/kubernetes/main.tofu index fb7510aaa..e16f53ddf 100644 --- a/tofu/kubernetes/main.tofu +++ b/tofu/kubernetes/main.tofu @@ -45,3 +45,27 @@ module "volumes" { proxmox_api = var.proxmox volumes = var.kubernetes_volumes } + +module "infisical_secrets" { + # Conditionally create infisical module based on where var.infisical is set in infisical.auto.tfvars + count = (var.infisical_project_id != null && var.infisical_client_id != null) ? 1 : 0 + + source = "./infisical" + + infisical_domain = var.infisical_domain + infisical_client_id = var.infisical_client_id + infisical_project_id = var.infisical_project_id + infisical_rw_secrets_path = var.infisical_rw_secrets_path + infisical_branch_env_mapping = var.infisical_branch_env_mapping + infisical_client_secret = var.infisical_client_secret + + secrets_to_create = { + # Create map entries only for non-null values + for k, v in { + "kubeconfig" = module.talos.kube_config.kubeconfig_raw + "talos_config" = module.talos.client_configuration.talos_config + "kube_certificate" = file("${path.root}/${var.sealed_secrets_config.certificate_path}") + "kube_certificate_key" = file("${path.root}/${var.sealed_secrets_config.certificate_key_path}") + } : k => v if v != null + } +} diff --git a/tofu/kubernetes/providers.tofu b/tofu/kubernetes/providers.tofu index cfba06135..c6e9f67fe 100644 --- a/tofu/kubernetes/providers.tofu +++ b/tofu/kubernetes/providers.tofu @@ -16,14 +16,43 @@ terraform { source = "Mastercard/restapi" version = "2.0.1" } + infisical = { + source = "infisical/infisical" + version = ">= 0.15.7" + } + external = { + source = "hashicorp/external" + version = ">= 2.3.4" + } } + encryption { + key_provider "pbkdf2" "my_passphrase" { + passphrase = local.tofu_encryption_passphrase + } + method "aes_gcm" "my_method" { + keys = key_provider.pbkdf2.my_passphrase + } + state { + method = method.aes_gcm.my_method + enforced = true + } + plan { + method = method.aes_gcm.my_method + enforced = true + } + } +} + +locals { + proxmox_api_token = var.proxmox_api_token + tofu_encryption_passphrase = var.tofu_encryption_passphrase } provider "proxmox" { endpoint = var.proxmox.endpoint insecure = var.proxmox.insecure - api_token = var.proxmox_api_token + api_token = local.proxmox_api_token ssh { agent = true username = var.proxmox.username @@ -37,7 +66,7 @@ provider "restapi" { headers = { "Content-Type" = "application/json" - "Authorization" = "PVEAPIToken=${var.proxmox_api_token}" + "Authorization" = "PVEAPIToken=${local.proxmox_api_token}" } } diff --git a/tofu/kubernetes/remote_backend.tf.sample b/tofu/kubernetes/remote_backend.tf.sample deleted file mode 100644 index 6404b0bcb..000000000 --- a/tofu/kubernetes/remote_backend.tf.sample +++ /dev/null @@ -1,6 +0,0 @@ -terraform { - backend "gcs" { - bucket = "" - prefix = "prod/kubernetes" - } -} diff --git a/tofu/kubernetes/samples/backend_gcs.tofu.sample b/tofu/kubernetes/samples/backend_gcs.tofu.sample new file mode 100644 index 000000000..b138b87c8 --- /dev/null +++ b/tofu/kubernetes/samples/backend_gcs.tofu.sample @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = var.bucket_name + prefix = "kubernetes/${var.branch_env}" + } +} diff --git a/tofu/kubernetes/samples/backend_local.tofu b/tofu/kubernetes/samples/backend_local.tofu new file mode 100644 index 000000000..3af0be55a --- /dev/null +++ b/tofu/kubernetes/samples/backend_local.tofu @@ -0,0 +1,5 @@ +terraform { + backend "local" { + path = "${var.gcs_prefix_env}.tfstate" + } +} diff --git a/tofu/kubernetes/samples/backend_local.tofu.sample b/tofu/kubernetes/samples/backend_local.tofu.sample new file mode 100644 index 000000000..8a1272ae6 --- /dev/null +++ b/tofu/kubernetes/samples/backend_local.tofu.sample @@ -0,0 +1,5 @@ +terraform { + backend "local" { + path = "${var.branch_env}.tfstate" + } +} diff --git a/tofu/kubernetes/samples/backend_r2.tofu.sample b/tofu/kubernetes/samples/backend_r2.tofu.sample new file mode 100644 index 000000000..3a62c3190 --- /dev/null +++ b/tofu/kubernetes/samples/backend_r2.tofu.sample @@ -0,0 +1,18 @@ +terraform { + backend "s3" { + bucket = var.bucket_name + key = "kubernetes/${var.branch_env}/terraform.tfstate" + region = "auto" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + skip_s3_checksum = true + use_path_style = true + endpoints = { + s3 = "https://${var.cloudflare_account_id}.r2.cloudflarestorage.com" + } + access_key = var.cloudflare_r2_tofu_access_key + secret_key = var.cloudflare_r2_tofu_access_secret + } +} diff --git a/tofu/kubernetes/variables.tofu b/tofu/kubernetes/variables.tofu index b2035b092..bae2fd96c 100644 --- a/tofu/kubernetes/variables.tofu +++ b/tofu/kubernetes/variables.tofu @@ -10,9 +10,17 @@ variable "proxmox" { } variable "proxmox_api_token" { - description = "API token for Proxmox" + description = "Proxmox API Token" type = string sensitive = true + default = null +} + +variable "tofu_encryption_passphrase" { + description = "State encryption passphrase" + type = string + sensitive = true + default = null } variable "talos_image" { @@ -35,6 +43,7 @@ variable "talos_cluster_config" { name = string vip = optional(string) gateway = string + subnet_mask = optional(string, "24") talos_machine_config_version = optional(string) proxmox_cluster = string kubernetes_version = string @@ -91,3 +100,27 @@ variable "kubernetes_volumes" { }) ) } + +# Define a variable for the bucket name. +# Set this in .env file in root, which should automatically set it in devcontainer env. +variable "bucket_name" { + description = "The globally unique name for the bucket." + type = string + + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.bucket_name)) + error_message = "Bucket name must be lowercase, alphanumeric, and hyphens only." + } + + validation { + condition = length(var.bucket_name) >= 3 && length(var.bucket_name) <= 63 + error_message = "Bucket name must be between 3 and 63 characters." + } +} + +# Define a variable for part of bucket prefix. +# This should be set automatically based on branch logic in devcontainer. +variable "branch_env" { + description = "Part of bucket prefix." + type = string +}