Mapping/oemofb3

oedatamodel_api/mapping_custom.py

@@ -238,6 +238,8 @@ def apply_custom_mapping(raw_json: dict, mapping: str):
     dict
         Resulting json/dict after applying all custom/default mappings.
     """
+    print(mapping)
+    # print(raw_json)
     if mapping in (m for m in OedataMapping):
         return map_data(raw_json, mapping)
     try:
@@ -247,9 +249,12 @@ def apply_custom_mapping(raw_json: dict, mapping: str):
     # Recursively apply base mappings if one exists:
     if mapping_json['base_mapping'] == "":
         pre_json = raw_json
+        print(pre_json)
     else:
-        pre_json = apply_custom_mapping(raw_json, mapping_json['base_mapping'])
+        pre_json = apply_custom_mapping(raw_json, mapping_json['base_mapping'])   
+        print(pre_json)
     # Recursively apply custom mapping on pre json:
+    print(iterate_mapping(pre_json, mapping_json['mapping']))
     return iterate_mapping(pre_json, mapping_json['mapping'])
 
 
@@ -271,6 +276,8 @@ def iterate_mapping(raw_json, value):
         Either dict containing next iteration or result from jmespath function
         applied to given json.
     """
+    print(raw_json)
+    print(value)
     if isinstance(value, dict):
         return {key: iterate_mapping(raw_json, mapping) for key, mapping in value.items()}
     return jmespath.search(value, raw_json, options=jmespath_options)

oedatamodel_api/mappings/oemofb3_minimize.json

@@ -0,0 +1,9 @@
+{
+    "base_mapping": "regions",
+    "mapping": {
+      "scenario": "scenario",
+      "scalars": "map(&exclude(@, ['output_energy_vector', 'year', 'tags', 'method']), scalars)",
+      "timeseries": "map(&exclude(@, ['input_energy_vector', 'output_energy_vector', 'technology', 'tags', 'method']), timeseries)"
+    }
+  }
+

oedatamodel_api/mappings/oemofb3_names.json

@@ -0,0 +1,9 @@
+{
+    "base_mapping": "oemofb3_minimize",
+    "mapping": {
+      "scenario": "scenario",
+      "scalars": "@.scalars[*].{id_scal:id, scenario:scenario_id, name: join('-', [@.region, @.input_energy_vector, @.technology_type]), var_name:parameter_name, carrier:input_energy_vector, region:region, tech:technology_type, type:technology, var_value:value, var_unit:unit, source:source, comment:comment}",
+      "timeseries": "@.timeseries[*].{id_ts:id, scenario:scenario_id, region:region, timeindex_start:timeindex_start, timeindex_stop:timeindex_stop, timeindex_resolution:timeindex_resolution, series:series, var_unit:unit, source:source, comment:comment}"
+    }
+  }
+

oedatamodel_api/mappings/oemofb3_to_oed_concrete.json

@@ -0,0 +1,9 @@
+{
+  "base_mapping": "",
+  "mapping": {
+    "oed_scenario": "oemofb3_oed_scenario",
+    "oed_data": "[map(&exclude(@, ['var_value']) | set(@, 'type', 'scalar'), oemofb3_oed_scalar), map(&exclude(@, ['timeindex_start', 'timeindex_stop', 'timeindex_resolution', 'series']) | set(@, 'type', 'timeseries'), oemofb3_oed_timeseries)][] | map(&set(@[1], 'id', @[0]), zip(range(`1`, sum([length(@), `1`])), @))",
+    "oed_scalar": "@.oemofb3_oed_scalar[*].{id:id, scenario_id:scenario, parameter_name:var_name, input_energy_vector:carrier, region:region, technology_type:tech, technology:type, value:var_value, unit:var_unit, source:source, comment:comment}",
+    "oed_timeseries": "@.oemofb3_oed_timeseries[*].{id:id, scenario_id:scenario, region:region, timeindex_start:timeindex_start, timeindex_stop:timeindex_stop, timeindex_resolution:timeindex_resolution, series:to_array(series), unit:var_unit, source:source, comment:comment}"
+  }
+}

oedatamodel_api/mappings/oemofb3_to_oed_normalize.json

@@ -0,0 +1,9 @@
+{
+    "base_mapping": "oemofb3_to_oed_concrete",
+    "mapping": {
+      "oemofb3_oed_scenario": "oed_scenario",
+      "oemofb3_oed_data": "@.oed_data[*].{id:id, scenario_id:scenario, region:region, input_energy_vector:carrier, output_energy_vector:'', parameter_name:var_name, technology:type, technology_type:tech, type:type, unit:var_unit, source:source, comment:comment}",
+      "oemofb3_oed_scalar": "map(&filter(@, ['value']), oed_scalar) | map(&set(@[1], 'id', @[0]), zip(range(`1`, sum([length(@), `1`])), @))",
+      "oemofb3_oed_timeseries": "map(&set(filter(@[1], ['timeindex_start', 'timeindex_stop', 'timeindex_resolution', 'series']), 'id', @[0]), zip(range(sum([length(oed_scalar), `1`]), sum([length(oed_scalar), length(oed_timeseries), `1`])), oed_timeseries))"
+    }
+}

oedatamodel_api/sources/oemofb3.json

@@ -0,0 +1,73 @@
+{
+    "from": {
+        "type": "join",
+        "left": {
+            "type": "table",
+            "table": "oemofb3_oed_scenario",
+            "schema": "model_draft",
+            "alias": "s"
+        },
+        "right": {
+            "type": "join",
+            "is_full": true,
+            "left": {
+                "type": "join",
+                "is_full": true,
+                "left": {
+                    "type": "table",
+                    "table": "oemofb3_oed_data",
+                    "schema": "model_draft",
+                    "alias": "d"
+                },
+                "right": {
+                    "type": "table",
+                    "table": "oemofb3_oed_timeseries",
+                    "schema": "model_draft",
+                    "alias": "ts"
+                },
+                "on": {
+                    "operands": [
+                        {"type": "column", "column": "id", "table": "d"},
+                        {"type": "column", "column": "id", "table": "ts"}
+                    ],
+                    "operator": "=",
+                    "type": "operator"
+                }
+            },
+            "right": {
+                "type": "table",
+                "table": "oemofb3_oed_scalar",
+                "schema": "model_draft",
+                "alias": "sc"
+            },
+            "on": {
+                "operands": [
+                    {"type": "column", "column": "id", "table": "d"},
+                    {"type": "column", "column": "id", "table": "sc"}
+                ],
+                "operator": "=",
+                "type": "operator"
+            }
+        },
+        "on": {
+            "operands": [
+                {"type": "column", "column": "id", "table": "s"},
+                {"type": "column", "column": "scenario_id", "table": "d"}
+            ],
+            "operator": "=",
+            "type": "operator"
+        }
+    },
+    "where": {
+        "operands": [
+            {
+                "type": "column",
+                "table": "s",
+                "column": "id"
+            },
+            "<scenario_id>"
+        ],
+        "operator": "=",
+        "type": "operator"
+    }
+}

oedatamodel_api/upload.py

@@ -131,6 +131,7 @@ def validate_upload_data(data, schema):
     for table, data_dict in data.items():
         # Get datapackage format for each table in data
         meta_url = f"{OEP_URL}/api/v0/schema/{schema}/tables/{table}/meta/"
+        print(meta_url)
         metadata = json.loads(requests.get(meta_url).content)
         try:
             oep_schema = metadata["resources"][0]["schema"]