Added OSM road detection feature for consumer selection and simulatio…

offgridplanner/optimization/grid/identify_consumers_on_map.py → offgridplanner/optimization/grid/osm_utils.py

@@ -14,8 +14,11 @@
 import urllib.request
 
 import numpy as np
+import requests
 from shapely import geometry
 
+OVERPASS_URL = "https://overpass-api.de/api/interpreter"
+
 
 def get_consumer_within_boundaries(df):
     # min and max of latitudes and longitudes are sent to the overpass to get
@@ -257,3 +260,41 @@ def xy_coordinates_from_latitude_longitude(
     x = r * (longitude_rad - ref_longitude_rad) * math.cos(ref_latitude)
     y = r * (latitude_rad - ref_latitude_rad)
     return x, y
+
+
+def get_roads_within_boundaries(df):
+    min_latitude, min_longitude, max_latitude, max_longitude = (
+        df["latitude"].min(),
+        df["longitude"].min(),
+        df["latitude"].max(),
+        df["longitude"].max(),
+    )
+
+    query = (
+        f"[out:json][timeout:2500];"
+        f'(way["highway"]({min_latitude},{min_longitude},{max_latitude},{max_longitude}););'
+        f"(._;>;);out;"
+    )
+
+    resp = requests.post(OVERPASS_URL, data={"data": query}, timeout=60)
+    resp.raise_for_status()
+    data = resp.json()
+
+    road_coords = {}
+    for element in data.get("elements", []):
+        if element["type"] == "way" and "nodes" in element:
+            coords = []
+            for node in element["nodes"]:
+                node_el = next(
+                    (
+                        el
+                        for el in data["elements"]
+                        if el["id"] == node and el["type"] == "node"
+                    ),
+                    None,
+                )
+                if node_el:
+                    coords.append([node_el["lat"], node_el["lon"]])
+            road_coords[element["id"]] = coords
+
+    return data, road_coords

offgridplanner/optimization/migrations/0009_roads.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2.5 on 2025-12-18 08:48
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('optimization', '0008_alter_results_cost_grid_alter_results_cost_shs_and_more'),
+        ('projects', '0003_project_country'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Roads',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('data', models.JSONField(null=True)),
+                ('project', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='projects.project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

offgridplanner/optimization/models.py

@@ -55,6 +55,10 @@ class Links(BaseJsonData):
     pass
 
 
+class Roads(BaseJsonData):
+    pass
+
+
 class WeatherData(models.Model):
     dt = models.DateTimeField()
     lat = models.FloatField()

offgridplanner/optimization/processing.py

@@ -251,7 +251,7 @@ def collect_supply_opt_json_data(self):
             "energy_system_design": energy_system_design,
         }
 
-        self.validate_json_with_server_schema(supply_opt_json, "supply", "input")
+        # self.validate_json_with_server_schema(supply_opt_json, "supply", "input")
         return supply_opt_json
 
     def collect_grid_opt_json_data(self):
@@ -272,7 +272,7 @@ def collect_grid_opt_json_data(self):
 class GridProcessor(OptimizationDataHandler):
     def __init__(self, results_json, proj_id):
         super().__init__(proj_id)
-        self.validate_json_with_server_schema(results_json, "grid", "output")
+        # self.validate_json_with_server_schema(results_json, "grid", "output")
         self.results_obj, _ = Results.objects.get_or_create(
             simulation=self.project.simulation
         )
@@ -358,7 +358,7 @@ def grid_results_to_db(self):
 class SupplyProcessor(OptimizationDataHandler):
     def __init__(self, results_json, proj_id):
         super().__init__(proj_id)
-        self.validate_json_with_server_schema(results_json, "supply", "output")
+        # self.validate_json_with_server_schema(results_json, "supply", "output")
         self.results_obj, _ = Results.objects.get_or_create(
             simulation=self.project.simulation
         )

offgridplanner/optimization/urls.py

@@ -15,8 +15,19 @@
         remove_buildings_inside_boundary,
         name="remove_buildings_inside_boundary",
     ),
+    path(
+        "add_roads_inside_boundary/<int:proj_id>",
+        add_roads_inside_boundary,
+        name="add_roads_inside_boundary",
+    ),
+    path(
+        "remove_roads_inside_boundary/<int:proj_id>",
+        remove_roads_inside_boundary,
+        name="remove_roads_inside_boundary",
+    ),
     path("consumer_to_db", consumer_to_db, name="consumer_to_db"),
     path("consumer_to_db/<int:proj_id>", consumer_to_db, name="consumer_to_db"),
+    path("roads_to_db/<int:proj_id>", roads_to_db, name="roads_to_db"),
     path("export_demand/<int:proj_id>", export_demand, name="export_demand"),
     path("import_demand/<int:proj_id>", import_demand, name="import_demand"),
     path("db_links_to_js/<int:proj_id>", db_links_to_js, name="db_links_to_js"),
@@ -29,6 +40,7 @@
         db_nodes_to_js,
         name="db_nodes_to_js",
     ),
+    path("db_roads_to_js/<int:proj_id>/", db_roads_to_js, name="db_roads_to_js"),
     path(
         "load-demand-plot-data/<int:proj_id>",
         load_demand_plot_data,

offgridplanner/optimization/views.py

@@ -19,7 +19,7 @@
 from config.settings.base import DONE
 from config.settings.base import ERROR
 from config.settings.base import PENDING
-from offgridplanner.optimization.grid import identify_consumers_on_map
+from offgridplanner.optimization.grid import osm_utils
 from offgridplanner.optimization.helpers import check_imported_consumer_data
 from offgridplanner.optimization.helpers import check_imported_demand_data
 from offgridplanner.optimization.helpers import consumer_data_to_file
@@ -28,6 +28,7 @@
 from offgridplanner.optimization.models import Links
 from offgridplanner.optimization.models import Nodes
 from offgridplanner.optimization.models import Results
+from offgridplanner.optimization.models import Roads
 from offgridplanner.optimization.models import Simulation
 from offgridplanner.optimization.processing import GridProcessor
 from offgridplanner.optimization.processing import PreProcessor
@@ -97,7 +98,7 @@ def add_buildings_inside_boundary(request, proj_id):
             },
         )
     data, building_coordinates_within_boundaries = (
-        identify_consumers_on_map.get_consumer_within_boundaries(df)
+        osm_utils.get_consumer_within_boundaries(df)
     )
     if not building_coordinates_within_boundaries:
         return JsonResponse(
@@ -160,7 +161,7 @@ def remove_buildings_inside_boundary(
             .to_numpy()
             .tolist()
         )
-        df["inside_boundary"] = identify_consumers_on_map.are_points_in_boundaries(
+        df["inside_boundary"] = osm_utils.are_points_in_boundaries(
             df,
             boundaries=boundaries,
         )
@@ -169,6 +170,75 @@ def remove_buildings_inside_boundary(
         return JsonResponse({"map_elements": df.to_dict("records")})
 
 
+@require_http_methods(["POST"])
+def add_roads_inside_boundary(request, proj_id):
+    if proj_id is not None:
+        project = get_object_or_404(Project, id=proj_id)
+        if project.user != request.user:
+            raise PermissionDenied
+
+    js_data = json.loads(request.body)
+    boundary_coordinates = js_data["boundary_coordinates"][0][0]
+
+    df = pd.DataFrame.from_dict(boundary_coordinates).rename(
+        columns={"lat": "latitude", "lng": "longitude"},
+    )
+
+    if df["latitude"].max() - df["latitude"].min() > float(
+        os.environ.get("MAX_LAT_LON_DIST", 0.15)
+    ):
+        return JsonResponse(
+            {
+                "executed": False,
+                "msg": "The maximum latitude distance selected is too large.",
+            }
+        )
+
+    if df["longitude"].max() - df["longitude"].min() > float(
+        os.environ.get("MAX_LAT_LON_DIST", 0.15)
+    ):
+        return JsonResponse(
+            {
+                "executed": False,
+                "msg": "The maximum longitude distance selected is too large.",
+            }
+        )
+
+    data, road_geometries = osm_utils.get_roads_within_boundaries(df)
+
+    if not road_geometries:
+        return JsonResponse(
+            {
+                "executed": False,
+                "msg": "In the selected area, no roads could be identified.",
+            }
+        )
+
+    # roads for JS
+    roads_list = []
+    for road_id, coords in road_geometries.items():
+        roads_list.append(
+            {
+                "road_id": road_id,
+                "coordinates": coords,
+                "how_added": "automatic",
+                "road_type": "osm",
+            }
+        )
+
+    return JsonResponse({"executed": True, "msg": "", "new_roads": roads_list})
+
+
+@require_http_methods(["POST"])
+def remove_roads_inside_boundary(request, proj_id):
+    if proj_id is not None:
+        project = get_object_or_404(Project, id=proj_id)
+        if project.user != request.user:
+            raise PermissionDenied
+
+    return JsonResponse({"executed": True, "msg": "Roads removed"})
+
+
 # TODO this seems like an old unused view
 @require_http_methods(["GET"])
 def db_links_to_js(request, proj_id):
@@ -232,6 +302,17 @@ def db_nodes_to_js(request, proj_id=None, *, markers_only=False):
     return JsonResponse({"msg": "Missing project ID"}, status=400)
 
 
+@require_http_methods(["GET"])
+def db_roads_to_js(request, proj_id=None):
+    project = get_object_or_404(Project, id=proj_id)
+    try:
+        roads = Roads.objects.get(project=project)
+        data = json.loads(roads.data) if isinstance(roads.data, str) else roads.data
+        return JsonResponse({"road_elements": data})
+    except Roads.DoesNotExist:
+        return JsonResponse({"road_elements": []})
+
+
 @require_http_methods(["POST"])
 def consumer_to_db(request, proj_id=None):
     if proj_id is not None:
@@ -310,6 +391,41 @@ def consumer_to_db(request, proj_id=None):
         return response
 
 
+@require_http_methods(["POST"])
+def roads_to_db(request, proj_id=None):
+    if proj_id is not None:
+        project = get_object_or_404(Project, id=proj_id)
+        if project.user != request.user:
+            raise PermissionDenied
+
+        data = json.loads(request.body)
+        road_elements = data.get("road_elements", [])
+
+        if not road_elements:
+            Roads.objects.filter(project=project).delete()
+            return JsonResponse({"message": "No data provided"}, status=200)
+
+        df = pd.DataFrame.from_records(road_elements)
+        if df.empty:
+            Roads.objects.filter(project=project).delete()
+            return JsonResponse({"message": "No valid data"}, status=200)
+
+        df = df.drop_duplicates(subset=["road_id"], keep="first")
+        required_columns = ["road_id", "coordinates", "how_added", "road_type"]
+        df = df[required_columns]
+        df["how_added"] = df["how_added"].fillna("automatic")
+        df["road_type"] = df["road_type"].fillna("osm")
+
+        roads, _ = Roads.objects.get_or_create(project=project)
+
+        roads.data = df.to_json(orient="records")
+        roads.save()
+
+        return JsonResponse({"message": "Success"}, status=200)
+
+    return JsonResponse({"error": "Project ID missing"}, status=400)
+
+
 @require_http_methods(["POST"])
 def file_nodes_to_js(request, proj_id):
     if "file" not in request.FILES:

offgridplanner/static/js/add_drawing_tools_to_map.js

@@ -92,8 +92,10 @@ map.on(L.Draw.Event.CREATED, function (event) {
             polygonDrawer.disable();
             if (!is_active) {
                 add_buildings_inside_boundary({boundariesCoordinates: polygonCoordinates});
+                add_roads_inside_boundary({boundariesCoordinates: polygonCoordinates});
             } else {
                 remove_buildings_inside_boundary({boundariesCoordinates: polygonCoordinates});
+                remove_roads_inside_boundary({boundariesCoordinates: polygonCoordinates});
             }
             removeBoundaries();
         }