Fix account map creation (#1218)

Author: iakov-gan

cfn-templates/cid-cfn.yml

@@ -1,6 +1,6 @@
 # https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/cfn-templates/cid-cfn.yml
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v4.2.2 - AWS Solution SO9011
+Description: Deployment of Cloud Intelligence Dashboards v4.2.3 - AWS Solution SO9011
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -2070,7 +2070,7 @@ Resources:
       SourceBucket: !Ref ReferenceAssetsBucket
       DestinationBucket: !Ref LocalAssetsBucket
       Keys:
-        - 'cid-resource-lambda-layer/cid-4.2.2.zip' #replace version here if needed
+        - 'cid-resource-lambda-layer/cid-4.2.3.zip' #replace version here if needed
 
   CidResourceLambdaLayer:
     Type: AWS::Lambda::LayerVersion
@@ -2085,7 +2085,7 @@ Resources:
             - LambdaLayerBucketPrefixIsManaged
             - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName]
             - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility
-        S3Key: 'cid-resource-lambda-layer/cid-4.2.2.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-4.2.3.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11

cid/_version.py

@@ -1,2 +1,2 @@
-__version__ = '4.2.2'
+__version__ = '4.2.3'
 

cid/common.py

@@ -131,10 +131,10 @@ def parameters_controller(self):
         """ get/create parameters_controller """
         return ParametersController(self.athena)
 
-    def get_cur(self, target_cur_version):
+    def get_cur(self, target_cur_version=None):
         """ get a cur """
         cur_version = self.cur.version
-        if cur_version != target_cur_version or get_parameters().get('use-cur-proxy'):
+        if (target_cur_version and cur_version != target_cur_version) or get_parameters().get('use-cur-proxy'):
             return ProxyCUR(self.cur, target_cur_version=target_cur_version)
         return self.cur
 
@@ -170,7 +170,7 @@ def create_or_update_account_map(self, name):
         account_map = AccountMap(
             self.base.session,
             self.athena,
-            self.cur, # can be any CUR. But it is only needed for trends and dummy
+            self.get_cur, # can be any CUR. But it is only needed for trends and dummy
         )
         return account_map.create_or_update(name)
 
@@ -1605,9 +1605,15 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non
         # patch dataset for tags
         cur_tags_json_required = False
         for dep_view_name in dataset_definition.get('dependsOn', {}).get('views', []):
-            cur_tags_json_required = 'tags_json' in str()
-            if self.resources['views'].get(dep_view_name, {}).get('dependsOn',{}).get('tags') == 'json' \
-                or self.resources['views'].get(dep_view_name, {}).get('parameters',{}).get('resource-tags'):
+            try:
+                tags_type = self.resources['views'][dep_view_name]['dependsOn']['tags']
+            except (KeyError, TypeError, AttributeError):
+                tags_type = None
+            try:
+                param_res_tag =  self.resources['views'][dep_view_name]['parameters']['resource-tags']
+            except (KeyError, TypeError, AttributeError):
+                param_res_tag = None
+            if tags_type == 'json' or param_res_tag:
                 cur_tags_json_required = True
                 break
         custom_fields = {}

cid/helpers/account_map.py

@@ -65,8 +65,8 @@ class AccountMap(CidBase):
         }
     }
 
-    def __init__(self, session: Session, athena: Athena, cur=None) -> None:
-        self.cur = cur # Only for trends and dummy
+    def __init__(self, session: Session, athena: Athena, get_cur=None) -> None:
+        self.get_cur = get_cur # get_cur callback Only for trends and dummy
         self.athena = athena
         super().__init__(session)
 
@@ -121,7 +121,7 @@ def create_sql_from_metadata(self, name, metadata_table):
             template = Template(
                 resource_string(
                     view_definition.get('providedBy'),
-                    f'data/queries/{view_file}'
+                    f'data/{view_file}'
                 ).decode('utf-8')
             )
         elif view_definition.get('data'):
@@ -133,8 +133,8 @@ def create_sql_from_metadata(self, name, metadata_table):
         vars = {
             'metadata_table_name': metadata_table['Name'],
             'metadata_database_name': metadata_table['Database'],
-            'cur_database': self.cur.database, # only for trends in metadata
-            'cur_table_name': self.cur.table_name, # only for trends in metadata
+            'cur_database': self.get_cur().database if name == 'aws_accounts' and self.get_cur else None, # only for trends in metadata
+            'cur_table_name': self.get_cur().table_name if name == 'aws_accounts' and self.get_cur else None, # only for trends in metadata
         }
         for key, val in self.mappings.get(name).get(metadata_table['Name']).items():
             logger.debug(f'Mapping field {key} to {val}')
@@ -151,7 +151,8 @@ def create_or_update(self, name) -> bool:
     def get_dummy_account_mapping_sql(self, name) -> list:
         """Create dummy account mapping"""
         logger.info(f'Creating dummy account mapping for {name}')
-        if self.cur.version.startswith('2'):
+        cur = self.get_cur()
+        if cur.version.startswith('2'):
             sql_file = 'data/queries/shared/account_map_cur2.sql'
         else:
             sql_file = 'data/queries/shared/account_map_dummy.sql'
@@ -161,8 +162,8 @@ def get_dummy_account_mapping_sql(self, name) -> list:
         ).decode('utf-8'))
         columns_tpl = {
             'athena_view_name': name,
-            'cur_table_name': self.cur.table_name,
-            'cur_database': self.cur.database,
+            'cur_table_name': cur.table_name,
+            'cur_database': cur.database,
         }
         compiled_query = template.safe_substitute(columns_tpl)
         return compiled_query
@@ -278,20 +279,23 @@ def create_account_mapping_sql(self, name) -> str:
             except CidError as exc:
                 logger.debug(exc)
 
-        if self.cur.version.startswith('2'):
+        if self.get_cur and self.get_cur().version.startswith('2'):
             cid_print('Looks like CUR2 is used. Will use it for account map.')
             set_parameters({'account-map-source': 'dummy'})
 
         for attempt in range(3):
             try:
+                choices = {
+                    'AWS Organizations (one time account listing)': 'organization',
+                    'CSV file (relative path required)': 'csv',
+                }
+                if self.get_cur:
+                    choices['Dummy (CUR account data, no names for CUR1)'] = 'dummy'
+
                 metadata_source = get_parameter(
                     param_name='account-map-source',
                     message="Please select account metadata collection method",
-                    choices={
-                        'Dummy (CUR account data, no names for CUR1)': 'dummy',
-                        'AWS Organizations (one time account listing)': 'organization',
-                        'CSV file (relative path required)': 'csv',
-                    },
+                    choices=choices,
                 )
                 logger.info(f'Attempt {attempt + 2}' )
                 # Collect account list from different sources of user choice