diff --git a/INSTALL.md b/INSTALL.md index 5b9645c0..e19e0ea2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -6,13 +6,13 @@ This guide is compatible with Ubuntu 18.04 and Ubuntu 20.04 ### Dependencies ``` -apt install python3 python3-dev virtualenv gcc pkg-config libpng-dev libjpeg-dev libfreetype6-dev postgresql-server-dev-all libgeos-dev g++ python3-shapely nodejs npm +apt install python3 python3-dev virtualenv gcc pkg-config libpng-dev libjpeg-dev libfreetype6-dev postgresql-server-dev-all libgeos-dev g++ nodejs npm ``` #### Database ``` -apt install postgresql postgresql-contrib +apt install postgresql postgresql-contrib postgresql-postgis ``` #### Web server diff --git a/api/issues_tiles.py b/api/issues_tiles.py index 588d2dc3..d57254d7 100644 --- a/api/issues_tiles.py +++ b/api/issues_tiles.py @@ -1,10 +1,7 @@ -import math from typing import Any, Dict, List, Optional -import mapbox_vector_tile # type: ignore from asyncpg import Connection from fastapi import APIRouter, Depends, HTTPException, Request, Response -from shapely.geometry import Point, Polygon # type: ignore from modules import query, tiles from modules.dependencies import commons_params, database @@ -17,68 +14,6 @@ class MVTResponse(Response): media_type = "application/vnd.mapbox-vector-tile" - def render(self, content: Any) -> bytes: - return mapbox_vector_tile.encode( - content["content"], - extents=content.get("extents", 2048), - quantize_bounds=content.get("quantize_bounds"), - ) - - -def mvtResponse(content) -> Response: - if not content or not content["content"]: - return Response(status_code=204) - else: - return MVTResponse(content, media_type="application/vnd.mapbox-vector-tile") - - -def _errors_mvt( - results: List[Dict[str, Any]], - z: int, - min_lon: float, - min_lat: float, - max_lon: float, - max_lat: float, - limit: int, -) -> Optional[Dict[str, Any]]: - if not results or len(results) == 0: - return None - else: - limit_feature = [] - if len(results) == limit and z < 18: - limit_feature = [ - { - "name": "limit", - "features": [ - { - "geometry": Point( - (min_lon + max_lon) / 2, (min_lat + max_lat) / 2 - ) - } - ], - } - ] - - issues_features = [] - for res in sorted(results, key=lambda res: -res["lat"]): - issues_features.append( - { - "id": res["id"], - "geometry": Point(res["lon"], res["lat"]), - "properties": { - "uuid": str(res["uuid"]), - "item": res["item"] or 0, - "class": res["class"] or 0, - }, - } - ) - - return { - "content": [{"name": "issues", "features": issues_features}] - + limit_feature, - "quantize_bounds": (min_lon, min_lat, max_lon, max_lat), - } - def _errors_geojson( results: List[Dict[str, Any]], @@ -179,15 +114,15 @@ async def heat( sql = ( f""" SELECT - COUNT(*), + COUNT(*) AS count, ( (lon-${len(sql_params)-4}) * ${len(sql_params)} / - (${len(sql_params)-2}-${len(sql_params)-4}) + 0.5 - )::int AS latn, + (${len(sql_params)-2}-${len(sql_params)-4}) - 0.5 + )::int AS x, ( (lat-${len(sql_params)-3}) * ${len(sql_params)} / (${len(sql_params)-1}-${len(sql_params)-3}) + 0.5 - )::int AS lonn, + )::int AS y, mode() WITHIN GROUP (ORDER BY items.marker_color) AS color FROM """ @@ -198,54 +133,72 @@ async def heat( + where + """ GROUP BY - latn, - lonn + x, y """ ) - features = [] - for row in await db.fetch(sql, *sql_params): - count, x, y, color = row - count = max( - int( - math.log(count) - / math.log(limit / ((z - 4 + 1 + math.sqrt(COUNT)) ** 2)) - * 255 - ), - 1 if count > 0 else 0, - ) - if count > 0: - count = 255 if count > 255 else count - features.append( - { - "geometry": Polygon( - [(x, y), (x - 1, y), (x - 1, y - 1), (x, y - 1)] - ), - "properties": {"color": int(color[1:], 16), "count": count}, - } - ) - - return mvtResponse( - { - "content": [{"name": "issues", "features": features}], - "extents": COUNT, - } + sql_params += [params.limit, params.zoom] + sql = f""" + WITH + grid AS ({sql}), + grid_count AS ( + SELECT + greatest( + ( + log(count) + / log(${len(sql_params)-1} / ((${len(sql_params)} - 4 + 1 + sqrt(${len(sql_params)-2})) ^ 2)) + * 255 + )::int, + CASE WHEN count > 0 THEN 1 ELSE 0 END + ) AS count, + x AS x, ${len(sql_params)-2} - y AS y, color + FROM + grid + ), + a AS ( + SELECT + least(count, 255) AS count, + ('0x' || substring(color, 2))::int AS color, + ST_MakeEnvelope(x, y, x+1, y+1) AS geom + FROM + grid_count + WHERE + count > 0 ) + SELECT ST_AsMVT(a, 'issues', ${len(sql_params)-2}::int, 'geom') FROM a + """ + results = await db.fetchval(sql, *sql_params) + if results is None or len(results) == 0: + return Response(status_code=204) + else: + return MVTResponse(results) -async def _issues( +def _issues_params( z: int, x: int, y: int, db: Connection, params: commons_params.Params, -) -> List[Dict[str, Any]]: +) -> commons_params.Params: params.limit = min(params.limit, 50 if z > 18 else 10000) params.tilex = x params.tiley = y params.zoom = z params.full = False + return params + + +async def _issues( + z: int, + x: int, + y: int, + db: Connection, + params: commons_params.Params, +) -> List[Dict[str, Any]]: + params = _issues_params(z, x, y, db, params) + if params.zoom > 18 or params.zoom < 7: return [] @@ -260,11 +213,16 @@ async def issues_mvt( db: Connection = Depends(database.db), params: commons_params.Params = Depends(commons_params.params), ) -> Response: - lon1, lat2 = tiles.tile2lonlat(x, y, z) - lon2, lat1 = tiles.tile2lonlat(x + 1, y + 1, z) + params = _issues_params(z, x, y, db, params) + + if params.zoom > 18 or params.zoom < 7: + return Response(status_code=204) - results = await _issues(z, x, y, db, params) - return mvtResponse(_errors_mvt(results, z, lon1, lat1, lon2, lat2, params.limit)) + results = await query._gets(db, params, mvt=True) + if results is None or len(results) == 0: + return Response(status_code=204) + else: + return MVTResponse(results) @router.get( @@ -277,5 +235,10 @@ async def issues_geojson( db: Connection = Depends(database.db), params: commons_params.Params = Depends(commons_params.params), ) -> GeoJSONFeatureCollection: - results = await _issues(z, x, y, db, params) + params = _issues_params(z, x, y, db, params) + + if params.zoom > 18 or params.zoom < 7: + return Response(status_code=204) + + results = await query._gets(db, params) return _errors_geojson(results, z, params.limit) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f6363af6..d1592e06 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:17-alpine + image: postgis/postgis:17-3.5-alpine volumes: - type: bind source: ../tools/database/schema.sql diff --git a/docker/postgres-init.sh b/docker/postgres-init.sh index 5fe277ae..f8162eb4 100755 --- a/docker/postgres-init.sh +++ b/docker/postgres-init.sh @@ -10,6 +10,7 @@ EOSQL psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "osmose_frontend" <<-EOSQL CREATE EXTENSION IF NOT EXISTS pgcrypto; + CREATE EXTENSION IF NOT EXISTS postgis; EOSQL psql -v ON_ERROR_STOP=1 --username "osmose" --dbname "osmose_frontend" < /schema.sql diff --git a/modules/query.py b/modules/query.py index 3a19de10..2646d269 100644 --- a/modules/query.py +++ b/modules/query.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from asyncpg import Connection @@ -260,7 +260,9 @@ def fixes_default(fixes: List[List[Dict[str, Any]]]) -> List[List[Dict[str, Any] ) -async def _gets(db: Connection, params: Params) -> List[Dict[str, Any]]: +async def _gets( + db: Connection, params: Params, mvt: bool = False +) -> Union[List[Dict[str, Any]], bytes]: sqlbase = """ SELECT uuid_to_bigint(uuid) as id, @@ -335,6 +337,40 @@ async def _gets(db: Connection, params: Params) -> List[Dict[str, Any]]: ${len(sql_params)}""" sql = sqlbase % (join, where) + + if mvt: + sql_params.extend([params.limit, params.zoom, params.tilex, params.tiley]) + sql = f""" + WITH + query AS ({sql}), + issues AS ( + SELECT + (id >> 32)::integer AS id, uuid, coalesce(item, 0) AS item, coalesce(class, 0) AS class, + ST_AsMVTGeom( + ST_Transform(ST_SetSRID(ST_MakePoint(lon, lat), 4326), 3857), + ST_TileEnvelope(${len(sql_params)-2}, ${len(sql_params)-1}, ${len(sql_params)}), + 4096, 0, false + ) AS geom + FROM query + ), + limit_ AS ( + SELECT + ST_AsMVTGeom( + ST_Centroid(ST_TileEnvelope(${len(sql_params)-2}, ${len(sql_params)-1}, ${len(sql_params)})), + ST_TileEnvelope(${len(sql_params)-2}, ${len(sql_params)-1}, ${len(sql_params)}), + 4096, 0, false + ) AS geom + WHERE (SELECT COUNT(*) FROM query) >= ${len(sql_params)-3} + ), + layers AS ( + SELECT ST_AsMVT(issues, 'issues', 4096, 'geom', 'id') AS layer FROM issues + UNION ALL + SELECT ST_AsMVT(limit_, 'limit', 4096, 'geom') AS layer FROM limit_ + ) + SELECT string_agg(layer, ''::bytea) FROM layers + """ + return await db.fetchval(sql, *sql_params) + results = list(await db.fetch(sql, *sql_params)) return list( map( diff --git a/requirements.txt b/requirements.txt index 94b99817..e5c6c34d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,6 @@ asyncpg matplotlib >= 1.1 requests >= 2.0 polib -protobuf < 4 # 4.x binary not yet compatible with system package, deps of mapbox-vector-tile -mapbox-vector-tile pyclipper fastapi fastapi-sessions