Add tool for partial generation of AWS Cloud spec document

modules/ivs_aws_instance/backup.tf

@@ -1,6 +1,7 @@
 resource "aws_iam_role" "backup_iam_role" {
   count              = var.backup_service_enable ? 1 : 0
   name               = "${local.instance_identifier}-backup-role"
+  description        = "Role for enabling backup job execution in AWS"
   assume_role_policy = <<POLICY
   {
     "Version": "2012-10-17",
@@ -15,6 +16,7 @@ resource "aws_iam_role" "backup_iam_role" {
     ]
   }
   POLICY
+  tags               = var.tags
 }
 
 resource "aws_iam_role_policy_attachment" "service_backup" {
@@ -44,6 +46,7 @@ resource "aws_iam_role_policy_attachment" "restore_s3" {
 resource "aws_backup_vault" "backup_vault" {
   count = var.backup_service_enable ? 1 : 0
   name  = "${local.instance_identifier}-backup-vault"
+  tags  = var.tags
 }
 
 resource "aws_backup_plan" "backup_plan" {

modules/k8s_eks_addons/ingress-nginx.tf

@@ -1,3 +1,6 @@
+locals {
+  string_tags = join(",", [for key, value in var.tags : "${key}=${value}"])
+}
 resource "kubernetes_namespace_v1" "ingress_nginx" {
   count = var.ingress_nginx_config.enable ? 1 : 0
 
@@ -18,6 +21,7 @@ resource "helm_release" "ingress_nginx" {
   dependency_update = true
   values = [
     templatefile("${path.module}/templates/nginx_values.yaml", {
+      tags                   = local.string_tags
       public_subnets         = join(", ", var.ingress_nginx_config.subnets_ids)
       protocol               = var.aws_load_balancer_controller_config.enable ? "ssl" : "tcp"
       aws_load_balancer_type = var.aws_load_balancer_controller_config.enable ? "external" : "nlb"

modules/k8s_eks_addons/templates/nginx_values.yaml

@@ -1,6 +1,7 @@
 controller:
   service:
     annotations:
+      service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "${tags}"
       service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "${protocol}"
       service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '60'
       service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true'

modules/simphera_aws_instance/backup.tf

@@ -1,6 +1,7 @@
 resource "aws_backup_vault" "backup-vault" {
   count = var.enable_backup_service ? 1 : 0
   name  = local.backup_vault_name
+  tags  = var.tags
 }
 
 resource "aws_backup_plan" "backup-plan" {
@@ -31,6 +32,7 @@ resource "aws_backup_selection" "backup-selection-rds-s3" {
 resource "aws_iam_role" "backup_iam_role" {
   count              = var.enable_backup_service ? 1 : 0
   name               = "${var.name}-backup-role"
+  description        = "Role for enabling backup job execution in AWS"
   assume_role_policy = <<POLICY
 {
   "Version": "2012-10-17",
@@ -45,6 +47,7 @@ resource "aws_iam_role" "backup_iam_role" {
   ]
 }
 POLICY
+  tags               = var.tags
 }
 
 resource "aws_iam_role_policy_attachment" "backup_rds_policy" {

scripts/aws_cloud_spec_gen/README.md

@@ -0,0 +1,47 @@
+# AWS Cloud spec generation
+
+This script is used for generating `AWSCloudSpec.md` of the current state of infrastructure.
+Using AWS SDK cli, script is filtering AWS resources by the tag `Cluster` and generate `AWSCloudSpec.md`.
+
+## Prerequisit
+- reference architecture is deployed with everything enabled
+    - `var.tags` must contain key `Cluster`, with the name of EKS cluster being deployed as a value
+- since script is replacing custom names with generic strings your `*.tfvars` must not have same string set for certain resources, and neither of them cannot be substring of the other, e.g.:
+    ```terraform
+    infrastructurename = "CLUSTER_NAME"
+    simpheraInstances = {
+    "SIMPHERA_STAGE_NAME" : {
+        "name" : "SIMPHERA_INSTANCE_NAME"
+    }
+    }
+    ivsInstances = {
+    "IVS_STAGE_NAME" : {}
+    }
+    ```
+- use only one GPU driver version in tfvars
+    ```terraform
+    gpu_operator_config = {
+        driver_versions = ["DRIVER_VERSION"]
+    }
+    ivsGpuDriverVersion = "DRIVER_VERSION"
+    ```
+
+## How to run a script
+- move to directory `.\scripts\aws_cloud_spec_gen`
+- install python requirements
+    - pip install -r requirements.txt
+- set environment variables for AWS credentials, `AWS_PROFILE` and `AWS_REGION`
+- run script:
+    ```powershell
+    python main.py --cluster_id CLUSTER_NAME `
+        --simphera_stage SIMPHERA_STAGE_NAME `
+        --simphera_instance SIMPHERA_INSTANCE_NAME `
+        --ivs_stage IVS_STAGE_NAME `
+        --gpu_driver DRIVER_VERSION
+    ```
+- output is found at location `.\scripts\aws_cloud_spec_gen\AWSCloudSpec.md`
+
+## Manual adaptation
+- For each resource that has `Mandatory` column, specify is it mandatory or not
+- Any missing resource description should be added
+- In `Policies` section, for column `Policy name`, add missing links to the policy definition, either online or relative local link

scripts/aws_cloud_spec_gen/helpers.py

@@ -0,0 +1,231 @@
+import json
+import subprocess
+import yaml
+
+from models.iam import Role, Policy
+from models.security_group import SecurityGroup
+from models.structure import Category, Service, ResourceType, Instance
+
+
+def execute(cmd: str) -> dict:
+    """! Runs given string command in underlying shell and returns it's output.
+    @param cmd (string) shell command to run
+
+    @return dictionary
+
+    @exception Exception Output of subrocess.stderr
+    """
+    process = subprocess.run(cmd, capture_output=True, encoding="utf-8")
+    if process.stderr:
+        raise Exception(process.stderr)
+    output = process.stdout
+    return json.loads(output)
+
+
+def index_value(arn: str):
+    """! Formats given string (ARN) in short format, e.g. "resource_type:subtype", unless resource is "s3".
+    @param arn (string) arn string
+
+    @return string or literal "s3"
+    """
+    splits = arn.split(":")
+    subtype = splits[5]
+    try:
+        subtype = subtype[0 : subtype.index("/")]
+    except:
+        pass
+    if splits[2] == "s3":
+        return "s3"
+    return splits[2] + ":" + subtype
+
+
+def get_security_groups(vpc_id: str) -> list[SecurityGroup]:
+    """! Queries all security groups in given VPC, and returns their object representation.
+    @param vpc_id (string) AWS VPC id
+
+    @return list[models.security_group.SecurityGroup]
+    """
+    security_groups = list()
+    security_groups_query = execute(f"aws ec2 describe-security-groups --filters Name=vpc-id,Values={vpc_id}")
+    for sg in security_groups_query["SecurityGroups"]:
+        security_groups.append(SecurityGroup.from_data(sg))
+
+    return security_groups
+
+
+def get_roles_and_policies(cluster_id: str) -> tuple[list[Role], list[Policy]]:
+    """! Queries all AWS roles and policies and returns their object representation.
+    @param cluster_id (string) AWS VPC id
+
+    @return tuple[list[models.iam.Role], list[models.iam.Policy]]
+    """
+    roles = execute("aws iam list-roles")
+    role_objects: list[Role] = list()
+    policies: list[Policy] = list()
+    for role in roles["Roles"]:
+        role_name = role["RoleName"]
+        if (
+            not role_name.startswith("AmazonSSM")
+            and not role_name.startswith("AWSServiceRole")
+            and not role_name.startswith("AWSReservedSSO")
+        ):
+            role_query = execute(f"aws iam get-role --role-name {role_name}")
+            role_content: dict = role_query["Role"]
+            if {"Key": "Cluster", "Value": cluster_id} in role_content.get("Tags", []):
+                role_object = Role(role_name, role_content.get("Description", "").replace("\n", ""))
+                attached_policies = execute(f"aws iam list-attached-role-policies --role-name {role_name}")
+                role_policies = execute(f"aws iam list-role-policies --role-name {role_name}")
+
+                for attached_policy in attached_policies["AttachedPolicies"]:
+                    policy_query = execute(f"aws iam get-policy --policy-arn {attached_policy['PolicyArn']}")
+                    policy = Policy(
+                        attached_policy["PolicyName"],
+                        policy_query["Policy"].get("Description", "").replace("\n", ""),
+                        "AWS" if attached_policy["PolicyArn"].startswith("arn:aws:iam::aws:policy/") else "Customer",
+                    )
+                    policies.append(policy)
+                    role_object.related_policies.append(policy)
+                for policy_name in role_policies["PolicyNames"]:
+                    policy = Policy(policy_name, None, None)
+                    role_object.related_policies.append(policy)
+                role_objects.append(role_object)
+
+    return role_objects, policies
+
+
+def get_roles_buffer(roles: list[Role]) -> str:
+    """! Creates Markdown string from given roles.
+    @param roles (list[models.iam.Role]) List of Role objects.
+
+    @return string
+    """
+    buffer_roles = list()
+    buffer_roles.append("| Role name | Description | Policies  |")
+    buffer_roles.append("| --------- | ----------- | --------- |")
+    for role in roles:
+        buffer_roles.append(role.markdown())
+    return "\n".join(buffer_roles)
+
+
+def get_policies_buffer(policies: list[Policy]) -> str:
+    """! Creates Markdown string from given policies.
+    @param policies (list[models.iam.Policy]) List of Role objects.
+
+    @return string
+    """
+    buffer_policies = list()
+    buffer_policies.append("| Policy name | Description | Managed By |")
+    buffer_policies.append("| ----------- | ----------- | ---------- |")
+
+    for policy in policies:
+        buffer_policies.append(policy.markdown())
+    return "\n".join(buffer_policies)
+
+
+def get_security_group_buffer(security_groups: list[SecurityGroup]) -> str:
+    """! Creates HTML table from given SecurityGroups.
+    @param security_groups (list[models.security_group.SecurityGroup]) List of SecurityGroup objects.
+
+    @return string
+    """
+    header = "<tr><th>Group name</th><th>Group description</th><th>Direction</th><th>Protocol</th><th>Port range</th><th>Rule description</th></tr>"
+    buffer = list()
+    for group in security_groups:
+        buffer.append(group.markdown_table())
+    return f"<table>{header}\n\n{'\n'.join(buffer)}\n\n</table>"
+
+
+def get_vpc_id(cluster_name: str) -> str:
+    """! Queries AWS vpcs and returns one tagged with "Name" of value "cluster_name".
+    @param cluster_name (string) EKS cluster name.
+
+    @return string
+    """
+    vpc_id = execute(
+        f'aws ec2 describe-vpcs --filters "Name=tag:Name,Values={cluster_name}-vpc" --query "Vpcs[].VpcId"'
+    )[0]
+    return vpc_id
+
+
+def get_categories(structure_file_path: str, cluster_name: str) -> list[Category]:
+    """! Queries AWS resources tagged with tag "Cluster" of value "cluster_name", parses "structure.yaml" file and creates list of models.structure.Category objects.
+    @param structure_file_path (string) Path to "structure.yaml" file.
+    @param cluster_name (string) EKS cluster name.
+
+    @return list[models.structure.Category]
+    """
+    with open(structure_file_path, "r") as structure_file:
+        structure = yaml.safe_load(structure_file)
+    arn_index = {}
+    categories: list[Category] = list()
+
+    for el in structure:
+        category = Category.from_data(el)
+        for service_el in el["services"]:
+            service = Service(service_el["name"], category.path + "/" + service_el["icon"])
+            for resource_el in service_el["resources"]:
+                resource_ = ResourceType(
+                    resource_el["name"],
+                    category.path + "/" + resource_el.get("icon") if resource_el.get("icon", False) else None,
+                    resource_el.get("arn", None),
+                    resource_el.get("type", "normal"),
+                    resource_el.get("source", None),
+                    resource_el.get("arn_regex_name", None),
+                    resource_el.get("arn_regex_id", None),
+                )
+                service.resources.append(resource_)
+                arn_index.update({resource_.arn: resource_})
+            category.services.append(service)
+        categories.append(category)
+    resources = execute(f"aws resourcegroupstaggingapi get-resources --tag-filters Key=Cluster,Values={cluster_name}")
+
+    for resource in resources["ResourceTagMappingList"]:
+        original_arn = resource["ResourceARN"]
+        indexed_arn = index_value(original_arn)
+        resource_ = arn_index.get(indexed_arn)
+        if resource_:
+            name = None
+            for el in resource["Tags"]:
+                key = el["Key"]
+                value = el["Value"]
+                if key == "Name":
+                    name = value
+            resource_.instances.append(Instance(name, original_arn))
+    return categories
+
+
+def populate_categories(
+    categories: list[Category], security_groups: list[SecurityGroup], roles: list[Role], policies: list[Policy]
+) -> list[str]:
+    """! Creates list of Markdown strings from given data.
+    @param categories (list[models.structure.Category]) List of Category objects.
+    @param securty_groups (list[models.security_group.SecurityGroup]) List of SecurityGroup objects.
+    @param roles (list[models.iam.Role]) List of Role objects.
+    @param policies (string) List of Policy objects.
+
+    @return list[str]
+    """
+    categories_buffer = list()
+    for category in categories:
+        categories_buffer.append(f"{category.get_title()}\n\n")
+        for service in category.services:
+            categories_buffer.append(f"{service.get_title()}\n\n")
+            for resource in service.resources:
+                if resource.type == "normal":
+                    if resource.name == "Security group":
+                        categories_buffer.append(
+                            f"{resource.get_title()}\n\n{get_security_group_buffer(security_groups)}\n\n"
+                        )
+                    elif resource.name == "Roles":
+                        categories_buffer.append(f"{resource.get_title()}\n\n{get_roles_buffer(roles)}\n\n")
+                    elif resource.name == "Policies":
+                        categories_buffer.append(f"{resource.get_title()}\n\n{get_policies_buffer(policies)}\n\n")
+                    else:
+                        resource_markdown = resource.markdown()
+                        categories_buffer.append(resource_markdown)
+                else:
+                    with open(f"static/{resource.source}", "r", encoding="utf-8") as file:
+                        requirement = file.read()
+                    categories_buffer.append(f"{resource.get_title()}\n\n{requirement}\n")
+
+    return categories_buffer