correct mapping code from csv to json for land use

notebooks/ghg-emissions/cdiac/mapping/cidac_fossil_fuel_nation_mapping_diemb.ipynb

@@ -0,0 +1 @@
+{"cells":[{"metadata":{"trusted":true},"id":"boolean-investing","cell_type":"code","source":"!pip install pandas\n!pip install matplotlib\n!pip install seaborn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"id":"traditional-session","cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport json\nimport math","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"id":"enabling-retirement","cell_type":"code","source":"filepath = \"https://raw.githubusercontent.com/OpenGeoScales/ogs-data-exploration/cdiac/data/ghg-emissions/cdiac/raw/Fossil-Fuel_CO2_Emissions_by_Nation.csv\"\ndata = pd.read_csv(filepath)\ndata.head()","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"                                              Nation    Year  \\\n0              (Note: missing values denoted by \".\")     NaN   \n1  Source: Tom Boden and Bob Andres (Oak Ridge Na...     NaN   \n2                     DOI: 10.3334/CDIAC/00001_V2017     NaN   \n3                                        AFGHANISTAN  1949.0   \n4                                        AFGHANISTAN  1950.0   \n\n   Total CO2 emissions from fossil-fuels and cement production (thousand metric tons of C)  \\\n0                                                NaN                                         \n1                                                NaN                                         \n2                                                NaN                                         \n3                                                4.0                                         \n4                                               23.0                                         \n\n  Emissions from solid fuel consumption  \\\n0                                   NaN   \n1                                   NaN   \n2                                   NaN   \n3                                     4   \n4                                     6   \n\n  Emissions from liquid fuel consumption Emissions from gas fuel consumption  \\\n0                                    NaN                                 NaN   \n1                                    NaN                                 NaN   \n2                                    NaN                                 NaN   \n3                                      0                                   0   \n4                                     18                                   0   \n\n   Emissions from cement production Emissions from gas flaring  \\\n0                               NaN                        NaN   \n1                               NaN                        NaN   \n2                               NaN                        NaN   \n3                               0.0                          .   \n4                               0.0                          0   \n\n  Per capita CO2 emissions (metric tons of carbon)  \\\n0                                              NaN   \n1                                              NaN   \n2                                              NaN   \n3                                                .   \n4                                                0   \n\n   Emissions from bunker fuels (not included in the totals)  \n0                                                NaN         \n1                                                NaN         \n2                                                NaN         \n3                                                0.0         \n4                                                0.0         ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Nation</th>\n      <th>Year</th>\n      <th>Total CO2 emissions from fossil-fuels and cement production (thousand metric tons of C)</th>\n      <th>Emissions from solid fuel consumption</th>\n      <th>Emissions from liquid fuel consumption</th>\n      <th>Emissions from gas fuel consumption</th>\n      <th>Emissions from cement production</th>\n      <th>Emissions from gas flaring</th>\n      <th>Per capita CO2 emissions (metric tons of carbon)</th>\n      <th>Emissions from bunker fuels (not included in the totals)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>(Note: missing values denoted by \".\")</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Source: Tom Boden and Bob Andres (Oak Ridge Na...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>DOI: 10.3334/CDIAC/00001_V2017</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AFGHANISTAN</td>\n      <td>1949.0</td>\n      <td>4.0</td>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>.</td>\n      <td>.</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>AFGHANISTAN</td>\n      <td>1950.0</td>\n      <td>23.0</td>\n      <td>6</td>\n      <td>18</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true},"id":"beginning-butterfly","cell_type":"code","source":"columns = [\"country\", \"year\", \"total_gas\", \"solid_fuel\", \"liquid_fuel\"\n           , \"gas_fuel\", \"cement\", \"gas_flaring\", \"per_capita_co2\", \"bunker_fuels\"]\n# rename the columns\ndata.columns = columns\n# drop first 3 lines & reset index\ndata = data.iloc[3:, :].reset_index(drop=True)\ndata = data.dropna()\n# convert year into datetime and drop out the time part, keep the date part only\ndata.year = data.year.astype('int64')\n#data = data.set_index(['year','country'])\n#data = data.applymap(np.integer)\ndata.head()","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"       country  year  total_gas solid_fuel liquid_fuel gas_fuel  cement  \\\n0  AFGHANISTAN  1949        4.0          4           0        0     0.0   \n1  AFGHANISTAN  1950       23.0          6          18        0     0.0   \n2  AFGHANISTAN  1951       25.0          7          18        0     0.0   \n3  AFGHANISTAN  1952       25.0          9          17        0     0.0   \n4  AFGHANISTAN  1953       29.0         10          18        0     0.0   \n\n  gas_flaring per_capita_co2  bunker_fuels  \n0           .              .           0.0  \n1           0              0           0.0  \n2           0              0           0.0  \n3           0              0           0.0  \n4           0              0           0.0  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>country</th>\n      <th>year</th>\n      <th>total_gas</th>\n      <th>solid_fuel</th>\n      <th>liquid_fuel</th>\n      <th>gas_fuel</th>\n      <th>cement</th>\n      <th>gas_flaring</th>\n      <th>per_capita_co2</th>\n      <th>bunker_fuels</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>AFGHANISTAN</td>\n      <td>1949</td>\n      <td>4.0</td>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>.</td>\n      <td>.</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>AFGHANISTAN</td>\n      <td>1950</td>\n      <td>23.0</td>\n      <td>6</td>\n      <td>18</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>AFGHANISTAN</td>\n      <td>1951</td>\n      <td>25.0</td>\n      <td>7</td>\n      <td>18</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AFGHANISTAN</td>\n      <td>1952</td>\n      <td>25.0</td>\n      <td>9</td>\n      <td>17</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>AFGHANISTAN</td>\n      <td>1953</td>\n      <td>29.0</td>\n      <td>10</td>\n      <td>18</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true},"id":"inner-length","cell_type":"code","source":"source_name = \"cdiac\"\nsource_url = \"https://cdiac.ess-dive.lbl.gov/trends/emis/tre_regional.html\"\ngeo_scale = \"country\"\ngeo_id_type = \"name\"\nemission_unit = \"Mtc\"\ngas_name = \"CO2\"\n# Create the output file, appending to the json exisiting file\npath = \"../../data/json/mapped_data_\" + source_name+\".json\"\nfile = open(path, \"a\")\n# Copy from GCP's Mapping_gcp.ipynb notebook\nmapped_sectors={}\nmapped_sectors[\"total_gas\"]=\"fossil_emissions_total\"\nmapped_sectors[\"gas_fuel\"]=\"fossil_emissions_gas\"\nmapped_sectors[\"cement\"]=\"fossil_emissions_cement\"\nmapped_sectors[\"gas_flaring\"]=\"fossil_emissions_flaring\"\nmapped_sectors[\"bunker_fuels\"]=\"bunker_fuels\"\nmapped_sectors[\"solid_fuel\"]=\"fossil_emissions_other\"\nmapped_sectors[\"liquid_fuel\"]=\"fossil_emissions_other\"\nmapped_sectors[\"per_capita_co2\"]=\"fossil_emissions_other\"\n\nlist_sector = mapped_sectors.keys()\nfor sect_origin in list_sector:\n    subdata = data[['country', 'year', sect_origin]]\n    for i in range(len(subdata)):\n        country = subdata.iloc[i].country\n        year = subdata.iloc[i].year\n        emission_value = subdata.iloc[i][sect_origin]\n        if (emission_value == '.'):\n            continue\n        dict_mapped_entry={}  \n        ############## Entries associated with geo_component\n        dict_geo_component={}\n        dict_geo_component[\"scale\"] = geo_scale\n        dict_geo_component[\"name\"] = country      \n        dict_geo_component[\"identifier\"] = {\"id\": country\n                                            , \"type\": geo_id_type }\n        ############# Entries associated with emission\n        dict_emission={}\n        dict_emission[\"gas\"] = gas_name\n        dict_emission[\"value\"] = str(emission_value)\n        dict_emission[\"unit\"] = {\"unit_used\" : emission_unit} \n        dict_emission[\"sector\"] = {\"sector_origin_name\": sect_origin\n                                    , \"sector_mapped_name\": mapped_sectors.get(sect_origin)} \n        ############# Combine everything together\n        ############## Entries associated with data_source\n        dict_mapped_entry[\"data_source\"] = {\"name\": source_name\n                                            , \"link\":source_url}\n        dict_mapped_entry[\"geo_component\"] = dict_geo_component \n        dict_mapped_entry[\"date\"] = str(year)+\"-01-01\"\n        dict_mapped_entry[\"emission\"] = dict_emission\n        # Write the json object to a file and add a line break (every line is a json object)\n        json.dump(dict_mapped_entry, file)\n        file.write(\"\\n\")\n        \n\nfile.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":5}