Merge pull request #45 from openstreetmap-polska/dev

Release to main

.github/workflows/deploy.yaml

@@ -33,11 +33,11 @@ jobs:
         run: |
           nix-shell --pure --run true
 
-      - name: Build Cython
+      - name: Build Cython modules
         run: |
           nix-shell --pure --run cython-build
 
-      - name: Build Docker image
+      - name: Build container image
         run: |
           echo "IMAGE_PATH=$(nix-build --no-out-link)" >> $GITHUB_ENV
 
@@ -53,11 +53,11 @@ jobs:
             IdentityFile ~/.ssh/id_rsa
           " > ~/.ssh/config
 
-      - name: Upload Docker image
+      - name: Upload container image
         run: |
           scp "${{ env.IMAGE_PATH }}" remote:~
 
-      - name: Deploy
+      - name: Deploy on remote
         run: |
           ssh remote <<\EOF
             set -e

.vscode/launch.json

@@ -12,7 +12,7 @@
             "args": [
                 "main:app",
                 "--workers",
-                "8",
+                "1",
             ],
             "jinja": true,
             "justMyCode": true,

api/v1/node.py

@@ -1,5 +1,6 @@
 import re
 from datetime import datetime
+from urllib.parse import quote_plus
 
 from fastapi import APIRouter, HTTPException
 from pytz import timezone
@@ -42,9 +43,10 @@ async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoState
         '@timezone_offset': timezone_offset,
     }
 
-    # TODO: support other image sources
+    image_url = aed.tags.get('image', '')
+
     if (
-        (image_url := aed.tags.get('image', ''))
+        image_url
         and (photo_id_match := photo_id_re.search(image_url))
         and (photo_id := photo_id_match.group('id'))
         and (photo_info := await photo_state.get_photo_by_id(photo_id))
@@ -53,6 +55,11 @@ async def get_node(node_id: str, aed_state: AEDStateDep, photo_state: PhotoState
             '@photo_id': photo_info.id,
             '@photo_url': f'/api/v1/photos/view/{photo_info.id}.webp',
         }
+    elif image_url:
+        photo_dict = {
+            '@photo_id': None,
+            '@photo_url': f'/api/v1/photos/proxy/{quote_plus(image_url)}',
+        }
     else:
         photo_dict = {
             '@photo_id': None,

api/v1/photos.py

@@ -1,18 +1,22 @@
 from datetime import UTC, datetime, timedelta
+from io import BytesIO
 from typing import Annotated
+from urllib.parse import unquote_plus
 
 import magic
 import orjson
 from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile
 from fastapi.responses import FileResponse
 from feedgen.feed import FeedGenerator
 
+from config import IMAGE_CONTENT_TYPES, REMOTE_IMAGE_MAX_FILE_SIZE
 from middlewares.cache_middleware import configure_cache
 from openstreetmap import OpenStreetMap, osm_user_has_active_block
 from osm_change import update_node_tags_osm_change
 from states.aed_state import AEDStateDep
 from states.photo_report_state import PhotoReportStateDep
 from states.photo_state import PhotoStateDep
+from utils import get_http_client
 
 router = APIRouter(prefix='/photos')
 
@@ -28,6 +32,35 @@ async def view(request: Request, id: str, photo_state: PhotoStateDep) -> FileRes
     return FileResponse(info.path)
 
 
+@router.get('/proxy/{url_encoded:path}')
+@configure_cache(timedelta(days=7), stale=timedelta(days=7))
+async def proxy(request: Request, url_encoded: str) -> FileResponse:
+    # NOTE: ideally we would verify whether url is not a private resource
+    async with get_http_client() as http:
+        r = await http.get(unquote_plus(url_encoded))
+        r.raise_for_status()
+
+    # Early detection of unsupported types
+    content_type = r.headers.get('Content-Type')
+    if content_type and content_type not in IMAGE_CONTENT_TYPES:
+        raise HTTPException(500, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}')
+
+    with BytesIO() as buffer:
+        async for chunk in r.aiter_bytes(chunk_size=1024 * 1024):
+            buffer.write(chunk)
+            if buffer.tell() > REMOTE_IMAGE_MAX_FILE_SIZE:
+                raise HTTPException(500, f'File is too large, max allowed size is {REMOTE_IMAGE_MAX_FILE_SIZE} bytes')
+
+        file = buffer.getvalue()
+
+    # Check if file type is supported
+    content_type = magic.from_buffer(file[:2048], mime=True)
+    if content_type not in IMAGE_CONTENT_TYPES:
+        raise HTTPException(500, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}')
+
+    return Response(content=file, media_type=content_type)
+
+
 @router.post('/upload')
 async def upload(
     request: Request,
@@ -48,10 +81,9 @@ async def upload(
         raise HTTPException(400, 'File must not be empty')
 
     content_type = magic.from_buffer(file.file.read(2048), mime=True)
-    accept_content_types = ('image/jpeg', 'image/png', 'image/webp')
 
-    if content_type not in accept_content_types:
-        raise HTTPException(400, f'Unsupported file type {content_type!r}, must be one of {accept_content_types}')
+    if content_type not in IMAGE_CONTENT_TYPES:
+        raise HTTPException(400, f'Unsupported file type {content_type!r}, must be one of {IMAGE_CONTENT_TYPES}')
 
     try:
         oauth2_credentials_ = orjson.loads(oauth2_credentials)

config.py

@@ -9,7 +9,7 @@
 from pyproj import Transformer
 
 NAME = 'openaedmap-backend'
-VERSION = '2.4'
+VERSION = '2.5'
 VERSION_TIMESTAMP = 0
 CREATED_BY = f'{NAME} {VERSION}'
 WEBSITE = 'https://openaedmap.org'
@@ -50,9 +50,12 @@
 MVT_EXTENT = 4096
 MVT_TRANSFORMER = Transformer.from_crs(OSM_PROJ, MVT_PROJ, always_xy=True)
 
+IMAGE_CONTENT_TYPES = {'image/jpeg', 'image/png', 'image/webp'}
 IMAGE_LIMIT_PIXELS = 6 * 1000 * 1000  # 6 MP (e.g., 3000x2000)
 IMAGE_MAX_FILE_SIZE = 2 * 1024 * 1024  # 2 MB
 
+REMOTE_IMAGE_MAX_FILE_SIZE = 10 * 1024 * 1024  # 10 MB
+
 DATA_DIR = Path('data')
 PHOTOS_DIR = DATA_DIR / 'photos'
 

docker-compose.dev.yml

@@ -2,16 +2,8 @@ version: "3"
 
 services:
   db:
-    image: mongo
-    command:
-      [
-        "mongod",
-        "--bind_ip_all",
-        "--setParameter",
-        "transactionLifetimeLimitSeconds=90",
-        "--replSet",
-        "rs0",
-      ]
+    image: docker.io/library/mongo
+    command: ["mongod", "--bind_ip_all", "--setParameter", "transactionLifetimeLimitSeconds=90", "--replSet", "rs0"]
 
     ports:
       - 127.0.0.1:27017:27017
@@ -20,7 +12,7 @@ services:
       - ./data/db:/data/db
 
   db-setup:
-    image: mongo
+    image: docker.io/library/mongo
     entrypoint: ["/bin/sh", "/mongo-init-replica.sh"]
     command: ["127.0.0.1:27017"]
 

docker-compose.yml

@@ -2,23 +2,15 @@ version: "3"
 
 services:
   db:
-    image: mongo
+    image: docker.io/library/mongo
     restart: unless-stopped
-    command:
-      [
-        "mongod",
-        "--bind_ip_all",
-        "--setParameter",
-        "transactionLifetimeLimitSeconds=90",
-        "--replSet",
-        "rs0",
-      ]
+    command: ["mongod", "--bind_ip_all", "--setParameter", "transactionLifetimeLimitSeconds=90", "--replSet", "rs0"]
 
     volumes:
       - ./data/db:/data/db
 
   db-setup:
-    image: mongo
+    image: docker.io/library/mongo
     entrypoint: ["/bin/sh", "/mongo-init-replica.sh"]
     command: ["db:27017"]
 
@@ -59,18 +51,10 @@ services:
       - /mnt/data/${TAG:-dev}/photos:/app/data/photos
 
   cache:
-    image: varnish:alpine
+    image: docker.io/library/varnish:alpine
     restart: unless-stopped
     user: root
-    command:
-      [
-        "varnishd",
-        "-F",
-        "-f",
-        "/etc/varnish/default.vcl",
-        "-s",
-        "file,/var/lib/varnish/varnish_storage.bin,2G",
-      ]
+    command: ["varnishd", "-F", "-f", "/etc/varnish/default.vcl", "-s", "file,/var/lib/varnish/varnish_storage.bin,2G"]
 
     depends_on:
       - app