Merge pull request #41 from openstreetmap-polska/dev

Release to main

.github/workflows/deploy.yaml

@@ -33,6 +33,10 @@ jobs:
         run: |
           nix-shell --run "true"
 
+      - name: Compile cython
+        run: |
+          nix-shell --run "pipenv run make"
+
       - name: Build
         run: |
           echo "IMAGE_PATH=$(nix-build --no-out-link)" >> $GITHUB_ENV

.gitignore

@@ -199,3 +199,6 @@ pyrightconfig.json
 result
 data/*
 cert/*
+
+cython_lib/*.c
+cython_lib/*.html

Makefile

@@ -1,6 +1,9 @@
-.PHONY: build version dev-start dev-stop dev-logs
+.PHONY: setup docker version dev-start dev-stop dev-logs
 
-build:
+setup:
+	python setup.py build_ext --build-lib cython_lib
+
+docker:
 	docker load < $$(nix-build --no-out-link)
 
 version:

Pipfile

@@ -9,6 +9,7 @@ asyncache = "*"
 authlib = "*"
 brotlipy = "*"
 cachetools = "*"
+cython = "*"
 dacite = "*"
 fastapi = "*"
 feedgen = "*"
@@ -18,11 +19,11 @@ jinja2 = "*"
 mapbox-vector-tile = "*"
 motor = "*"
 networkx = "*"
-numba = "*"
 numpy = "*"
 orjson = "*"
 pillow = "*"
 psutil = "*"
+pyinstrument = "*"
 pyproj = "*"
 python-dateutil = "*"
 python-magic = "*"

api/v1/countries.py

@@ -4,7 +4,7 @@
 import anyio
 from anyio.streams.memory import MemoryObjectSendStream
 from fastapi import APIRouter, Path, Request, Response
-from shapely.geometry import Point, mapping
+from shapely.geometry import mapping
 
 from middlewares.cache_middleware import configure_cache
 from models.country import Country
@@ -21,7 +21,12 @@ async def _count_aed_in_country(country: Country, aed_state: AEDState, send_stre
 
 @router.get('/names')
 @configure_cache(timedelta(hours=1), stale=timedelta(days=7))
-async def get_names(request: Request, country_state: CountryStateDep, aed_state: AEDStateDep):
+async def get_names(
+    request: Request,
+    country_state: CountryStateDep,
+    aed_state: AEDStateDep,
+    language: str | None = None,
+):
     countries = await country_state.get_all_countries()
 
     send_stream, receive_stream = anyio.create_memory_object_stream()
@@ -34,24 +39,39 @@ async def get_names(request: Request, country_state: CountryStateDep, aed_state:
         for _ in range(len(countries)):
             country, count = await receive_stream.receive()
             country_count_map[country.name] = count
+
+    def limit_country_names(names: dict[str, str]):
+        if language and (name := names.get(language)):
+            return {language: name}
+        return names
+
     return [
         {
             'country_code': country.code,
-            'country_names': country.names,
+            'country_names': limit_country_names(country.names),
             'feature_count': country_count_map[country.name],
             'data_path': f'/api/v1/countries/{country.code}.geojson',
-        } for country in countries
-    ] + [{
-        'country_code': 'WORLD',
-        'country_names': {'default': 'World'},
-        'feature_count': sum(country_count_map.values()),
-        'data_path': '/api/v1/countries/WORLD.geojson',
-    }]
+        }
+        for country in countries
+    ] + [
+        {
+            'country_code': 'WORLD',
+            'country_names': {'default': 'World'},
+            'feature_count': sum(country_count_map.values()),
+            'data_path': '/api/v1/countries/WORLD.geojson',
+        }
+    ]
 
 
 @router.get('/{country_code}.geojson')
 @configure_cache(timedelta(hours=1), stale=timedelta(seconds=0))
-async def get_geojson(request: Request, response: Response, country_code: Annotated[str, Path(min_length=2, max_length=5)], country_state: CountryStateDep, aed_state: AEDStateDep):
+async def get_geojson(
+    request: Request,
+    response: Response,
+    country_code: Annotated[str, Path(min_length=2, max_length=5)],
+    country_state: CountryStateDep,
+    aed_state: AEDStateDep,
+):
     if country_code == 'WORLD':
         aeds = await aed_state.get_all_aeds()
     else:
@@ -64,12 +84,9 @@ async def get_geojson(request: Request, response: Response, country_code: Annota
         'features': [
             {
                 'type': 'Feature',
-                'geometry': mapping(Point(*aed.position)),
-                'properties': {
-                    '@osm_type': 'node',
-                    '@osm_id': int(aed.id),
-                    **aed.tags
-                }
-            } for aed in aeds
-        ]
+                'geometry': mapping(aed.position.shapely),
+                'properties': {'@osm_type': 'node', '@osm_id': int(aed.id), **aed.tags},
+            }
+            for aed in aeds
+        ],
     }

api/v1/node.py

@@ -47,10 +47,10 @@ async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoState
 
     # TODO: support other image sources
     if (
-        (image_url := aed.tags.get('image', '')) and
-        (photo_id_match := photo_id_re.search(image_url)) and
-        (photo_id := photo_id_match.group('id')) and
-        (photo_info := await photo_state.get_photo_by_id(photo_id))
+        (image_url := aed.tags.get('image', ''))
+        and (photo_id_match := photo_id_re.search(image_url))
+        and (photo_id := photo_id_match.group('id'))
+        and (photo_info := await photo_state.get_photo_by_id(photo_id))
     ):
         photo_dict = {
             '@photo_id': photo_info.id,
@@ -67,14 +67,16 @@ async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoState
         'copyright': 'OpenStreetMap and contributors',
         'attribution': 'https://www.openstreetmap.org/copyright',
         'license': 'https://opendatacommons.org/licenses/odbl/1-0/',
-        'elements': [{
-            **photo_dict,
-            **timezone_dict,
-            'type': 'node',
-            'id': int(aed.id),
-            'lat': aed.position.lat,
-            'lon': aed.position.lon,
-            'tags': aed.tags,
-            'version': 0,
-        }]
+        'elements': [
+            {
+                **photo_dict,
+                **timezone_dict,
+                'type': 'node',
+                'id': int(aed.id),
+                'lat': aed.position.lat,
+                'lon': aed.position.lon,
+                'tags': aed.tags,
+                'version': 0,
+            }
+        ],
     }

api/v1/photos.py

@@ -1,11 +1,9 @@
-import traceback
 from datetime import UTC, datetime, timedelta
 from typing import Annotated
 
 import magic
 import orjson
-from fastapi import (APIRouter, File, Form, HTTPException, Request, Response,
-                     UploadFile)
+from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile
 from fastapi.responses import FileResponse
 from feedgen.feed import FeedGenerator
 
@@ -32,7 +30,15 @@ async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileRes
 
 
 @router.post('/upload')
-async def upload(request: Request, node_id: Annotated[str, Form()], file_license: Annotated[str, Form()], file: Annotated[UploadFile, File()], oauth2_credentials: Annotated[str, Form()], aed_state: AEDStateDep, photo_state: PhotoStateDep) -> bool:
+async def upload(
+    request: Request,
+    node_id: Annotated[str, Form()],
+    file_license: Annotated[str, Form()],
+    file: Annotated[UploadFile, File()],
+    oauth2_credentials: Annotated[str, Form()],
+    aed_state: AEDStateDep,
+    photo_state: PhotoStateDep,
+) -> bool:
     file_license = file_license.upper()
     accept_licenses = ('CC0',)
 
@@ -50,8 +56,8 @@ async def upload(request: Request, node_id: Annotated[str, Form()], file_license
 
     try:
         oauth2_credentials_ = orjson.loads(oauth2_credentials)
-    except Exception:
-        raise HTTPException(400, 'OAuth2 credentials must be a JSON object')
+    except Exception as e:
+        raise HTTPException(400, 'OAuth2 credentials must be a JSON object') from e
 
     if 'access_token' not in oauth2_credentials_:
         raise HTTPException(400, 'OAuth2 credentials must contain an access_token field')
@@ -75,22 +81,32 @@ async def upload(request: Request, node_id: Annotated[str, Form()], file_license
 
     node_xml = await osm.get_node_xml(node_id)
 
-    osm_change = update_node_tags_osm_change(node_xml, {
-        'image': photo_url,
-        'image:license': file_license,
-    })
+    osm_change = update_node_tags_osm_change(
+        node_xml,
+        {
+            'image': photo_url,
+            'image:license': file_license,
+        },
+    )
 
     await osm.upload_osm_change(osm_change)
     return True
 
 
 @router.post('/report')
-async def report(id: Annotated[str, Form()], photo_report_state: PhotoReportStateDep) -> bool:
+async def report(
+    id: Annotated[str, Form()],
+    photo_report_state: PhotoReportStateDep,
+) -> bool:
     return await photo_report_state.report_by_photo_id(id)
 
 
 @router.get('/report/rss.xml')
-async def report_rss(request: Request, photo_state: PhotoStateDep, photo_report_state: PhotoReportStateDep) -> Response:
+async def report_rss(
+    request: Request,
+    photo_state: PhotoStateDep,
+    photo_report_state: PhotoReportStateDep,
+) -> Response:
     fg = FeedGenerator()
     fg.title('AED Photo Reports')
     fg.description('This feed contains a list of recent AED photo reports')
@@ -105,10 +121,15 @@ async def report_rss(request: Request, photo_state: PhotoStateDep, photo_report_
         fe = fg.add_entry(order='append')
         fe.id(report.id)
         fe.title('🚨 Received photo report')
-        fe.content('<br>'.join((
-            f'File name: {info.path.name}',
-            f'Node: https://osm.org/node/{info.node_id}',
-        )), type='CDATA')
+        fe.content(
+            '<br>'.join(
+                (
+                    f'File name: {info.path.name}',
+                    f'Node: https://osm.org/node/{info.node_id}',
+                )
+            ),
+            type='CDATA',
+        )
         fe.link(href=upgrade_https(f'{request.base_url}api/v1/photos/view/{report.photo_id}.webp'))
         fe.published(datetime.utcfromtimestamp(report.timestamp).astimezone(tz=UTC))