diff --git a/clone_dataset/clone_dataset_tool.py b/clone_dataset/clone_dataset_tool.py index 7ae50dd..c7f959e 100644 --- a/clone_dataset/clone_dataset_tool.py +++ b/clone_dataset/clone_dataset_tool.py @@ -36,9 +36,17 @@ # do you want to clone only the first layer or all of the layers? clone_first_layer_only = False +# Application for which you want to clone from and to. The source application +# does not apply to datasets (it is cloned regardless) but only to layers, +# widgets, metadata and vocabularies +source_application = 'rw' +cloned_application = 'rw' + # do you want to clone only the default editable widget or all the widgets? -clone_default_widget_only = True +clone_default_widget_only = False +# Do you want to do a dry run and just print info on what will be done? +dry_run = False API_TOKEN = os.getenv('RW_API_KEY') @@ -68,7 +76,7 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase clone_dataset_attr[k] = dataset_params.get(k, '') payload = { 'dataset': { - 'application': ['rw'], + 'application': [cloned_application], 'connectorType': clone_dataset_attr['connectorType'], 'connectorUrl': clone_dataset_attr['connectorUrl'], 'tableName': clone_dataset_attr['tableName'], @@ -95,35 +103,42 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase print(f'Creating clone dataset') url = f'{clone_server}/dataset' headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json', 'Cache-Control': 'no-cache'} - r = requests.post(url, data=json.dumps(payload), headers=headers) - if r.status_code == 200: - clone_dataset_id = r.json()['data']['id'] - clone_dataset = LMIPy.Dataset(id_hash=clone_dataset_id, server=clone_server) - else: - print(r.status_code) - print(f'{clone_server}/v1/dataset/{clone_dataset_id}') + if not dry_run: + r = requests.post(url, data=json.dumps(payload), headers=headers) + if r.status_code == 200: + clone_dataset_id = r.json()['data']['id'] + clone_dataset = LMIPy.Dataset(id_hash=clone_dataset_id, server=clone_server) + else: + print(r.status_code) + print(f'{clone_server}/v1/dataset/{clone_dataset_id}') if clone_children: layers = self.layers if len(layers) > 0: if clone_first_layer_only: l = layers[0] - if l.attributes['application'] == ['rw']: + if source_application in l.attributes['application']: try: layer_name = l.attributes['name'] print('Cloning layer: {}'.format(layer_name)) - l.clone(token=token, env='production', layer_params={'name': layer_name}, - target_dataset_id=clone_dataset_id) - time.sleep(2) + if not dry_run: + l.clone(token=token, env='production', + layer_params={ + 'name': layer_name, + 'application': [cloned_application] + }, + target_dataset_id=clone_dataset_id) + time.sleep(2) except: raise ValueError(f'Layer cloning failed for {l.id}') else: for l in layers: - if l.attributes['application']==['rw']: + if source_application in l.attributes['application']: try: layer_name = l.attributes['name'] print('Cloning layer: {}'.format(layer_name)) - l.clone(token=token, env='production', layer_params={'name': layer_name}, target_dataset_id=clone_dataset_id) - time.sleep(2) + if not dry_run: + l.clone(token=token, env='production', layer_params={'name': layer_name}, target_dataset_id=clone_dataset_id) + time.sleep(2) except: raise ValueError(f'Layer cloning failed for {l.id}') else: @@ -145,36 +160,35 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase widget_config = widget['widgetConfig'], app = widget['application'] ds_id = clone_dataset_id - if app == ['rw']: + if source_application in app : if name and widget_config and app: widget_payload = { "name": widget['name'], "description": widget.get('description', None), "env": widget['env'], "widgetConfig": widget['widgetConfig'], - "application": ['rw'] + "application": [cloned_application] } - try: - url = f'{self.server}/v1/dataset/{ds_id}/widget' - print(url) - headers = {'Authorization': 'Bearer ' + token, - 'Content-Type': 'application/json'} - r = requests.post(url, data=json.dumps(widget_payload), headers=headers) - print(r.json()) - except: - raise ValueError(f'Widget creation failed.') - if r.status_code == 200: - print(f'Widget created.') - # self.attributes = self.get_dataset() - else: - print(f'Failed with error code {r.status_code}') + print(f"Cloning widget {widget['name']}") + if not dry_run: + try: + url = f'{self.server}/v1/dataset/{ds_id}/widget' + headers = {'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/json'} + r = requests.post(url, data=json.dumps(widget_payload), headers=headers) + print(r.json()) + except: + raise ValueError(f'Widget creation failed.') + if r.status_code == 200: + print(f'Widget created.') + # self.attributes = self.get_dataset() + else: + print(f'Failed with error code {r.status_code}') else: raise ValueError( f'Widget creation requires name string, application list and a widgetConfig object.') # clone_dataset.add_widget(token=token, widget_params=widget_payload) - else: - print("Non-rw app. Not cloning.") except: raise ValueError(f'Widget cloning failed for {widget.id}') else: @@ -184,37 +198,38 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase name = widget['name'], widget_config = widget['widgetConfig'], app = widget['application'] - ds_id = clone_dataset_id - if app==['rw']: + print('Widget application - ', widget['application']) + if source_application in app: if name and widget_config and app: + print(f"Cloning widget {widget['name']}") widget_payload = { "name": widget['name'], "description": widget.get('description', None), "env": widget['env'], "widgetConfig": widget['widgetConfig'], - "application": ['rw'] + "application": [cloned_application] } - try: - url = f'{self.server}/v1/dataset/{ds_id}/widget' - print(url) - headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'} - r = requests.post(url, data=json.dumps(widget_payload), headers=headers) - print(r.json()) - except: - raise ValueError(f'Widget creation failed.') - if r.status_code == 200: - print(f'Widget created.') - #self.attributes = self.get_dataset() - else: - print(f'Failed with error code {r.status_code}') + if not dry_run: + ds_id = clone_dataset_id + try: + url = f'{self.server}/v1/dataset/{ds_id}/widget' + headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'} + r = requests.post(url, data=json.dumps(widget_payload), headers=headers) + print(r.json()) + except: + raise ValueError(f'Widget creation failed.') + if r.status_code == 200: + print(f'Widget created.') + #self.attributes = self.get_dataset() + else: + print(f'Failed with error code {r.status_code}') else: raise ValueError( f'Widget creation requires name string, application list and a widgetConfig object.') #clone_dataset.add_widget(token=token, widget_params=widget_payload) - else: - print("Non-rw app. Not cloning.") - except: + except Exception as e: + print(str(e)) raise ValueError(f'Widget cloning failed for {widget.id}') else: print("No child widgets to clone!") @@ -222,41 +237,51 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase if len(vocabs) > 0: for v in vocabs: vocab = v.attributes - if vocab['application']=='rw': + print('Vocab application - ', vocab['application']) + if vocab['application']==source_application: + print(f"Cloning vocab {vocab['name']}") vocab_payload = { - 'application': vocab['application'], + 'application': cloned_application, 'name': vocab['name'], 'tags': vocab['tags'] } - try: - clone_dataset.add_vocabulary(vocab_params=vocab_payload, token=token) - except: - raise ValueError('Failed to clone Vocabulary.') + if not dry_run: + try: + clone_dataset.add_vocabulary(vocab_params=vocab_payload, token=token) + except: + raise ValueError('Failed to clone Vocabulary.') metas = self.metadata if len(metas) > 0: for m in metas: meta = m.attributes - if meta['application']=='rw': - meta_payload = { - "dataset": meta['dataset'], - 'application': meta['application'], - 'language': meta['language'], - "name": meta['name'], - 'description': meta['description'], - "source": meta['source'], - 'info': meta['info'], - } - if 'columns' in meta: - meta_payload.update({'columns': meta['columns']}) - try: - rw_api_url = 'https://api.resourcewatch.org/v1/dataset/{}/metadata'.format(clone_dataset.id) - res = requests.request("POST", rw_api_url, data=json.dumps(meta_payload), - headers=create_headers()) - print('Metadata created.') - except: - raise ValueError('Failed to clone Metadata.') + print('Metadata application - ', meta['application']) + if meta['application']==source_application: + print(f'Cloning metadata {meta["name"]}') + if not dry_run: + meta_payload = { + "dataset": clone_dataset.id, + 'application': cloned_application, + 'language': meta['language'], + "name": meta['name'], + 'description': meta['description'], + "source": meta['source'], + 'info': meta['info'], + } + if 'columns' in meta: + meta_payload.update({'columns': meta['columns']}) + try: + rw_api_url = 'https://api.resourcewatch.org/v1/dataset/{}/metadata'.format(clone_dataset.id) + headers = {'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'} + res = requests.request("POST", rw_api_url, data=json.dumps(meta_payload), + headers=headers) + print('Metadata created.') + except: + raise ValueError('Failed to clone Metadata.') # self.attributes = Dataset(clone_dataset_id, server=clone_server).attributes - return clone_dataset_id + if not dry_run: + return clone_dataset_id + else: + return "Dry run so no cloned dataset ID" # Make a copy dataset_to_copy = LMIPy.Dataset(dataset_id) @@ -265,4 +290,4 @@ def clone_ds(self, token=None, enviro='preproduction', clone_server=None, datase } # Clone dataset new_dataset_id = clone_ds(dataset_to_copy, token=API_TOKEN, enviro='production', dataset_params=clone_attributes, clone_children=True, clone_first_layer_only=clone_first_layer_only, clone_default_widget_only=clone_default_widget_only) -print('new dataset API ID:' + new_dataset_id) \ No newline at end of file +print('new dataset API ID:' + new_dataset_id)