From 9a52447230bcc0f0611bf9a0b43491b31f9d94c0 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Mon, 25 Dec 2023 20:26:14 +0100 Subject: [PATCH] Setup cython instead of numba --- .github/workflows/deploy.yaml | 4 +- .gitignore | 3 + Makefile | 7 +- Pipfile | 2 +- Pipfile.lock | 116 +++++++++++++++++----------------- api/v1/tile.py | 21 +----- cython_lib/geo_utils.py | 33 ++++++++++ default.nix | 10 +-- setup.py | 31 +++++++++ utils.py | 46 -------------- 10 files changed, 140 insertions(+), 133 deletions(-) create mode 100644 cython_lib/geo_utils.py create mode 100644 setup.py diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 6ba8a7d..0292e1d 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -29,9 +29,9 @@ jobs: trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= substituters = https://cache.nixos.org/ - - name: Install dependencies + - name: Install dependencies and compile cython run: | - nix-shell --run "true" + nix-shell --run "make" - name: Build run: | diff --git a/.gitignore b/.gitignore index 9a32144..58089a3 100644 --- a/.gitignore +++ b/.gitignore @@ -199,3 +199,6 @@ pyrightconfig.json result data/* cert/* + +cython_lib/*.c +cython_lib/*.html diff --git a/Makefile b/Makefile index bf39122..42f1c2d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ -.PHONY: build version dev-start dev-stop dev-logs +.PHONY: setup docker version dev-start dev-stop dev-logs -build: +setup: + python setup.py build_ext --build-lib cython_lib + +docker: docker load < $$(nix-build --no-out-link) version: diff --git a/Pipfile b/Pipfile index f5a36fb..1573735 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ asyncache = "*" authlib = "*" brotlipy = "*" cachetools = "*" +cython = "*" dacite = "*" fastapi = "*" feedgen = "*" @@ -18,7 +19,6 @@ jinja2 = "*" mapbox-vector-tile = "*" motor = "*" networkx = "*" -numba = "*" numpy = "*" orjson = "*" pillow = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 37e791b..a76572c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8dadabee8153ba256254efae2a78e3440eb8bf10d10faefbca4f2b9a47f837fb" + "sha256": "783a4557aeaaf8e961b5aeed7b9e530c6705e76e400283fa760c19242fb37078" }, "pipfile-spec": 6, "requires": { @@ -211,6 +211,63 @@ "markers": "python_version >= '3.7'", "version": "==41.0.7" }, + "cython": { + "hashes": [ + "sha256:01b94304aab87496e81d1f546e71abf57b430b39be4269df1cd7da9928d70b5b", + "sha256:0c636c9ab92c7838231a1ba769e519d953af8294612f3f772a54d3a5250ff23f", + "sha256:0d8a98c7d86ac4d05b251c39faf49423780381aab55fbf2e147f6e006a34a58a", + "sha256:13211b67b29f6ed8e87c137496c73d93aff0330d97940b4fbed72eae37a4a2a0", + "sha256:133057ac45b6fa7fe5d7baada9d3545d09339432f75c0545f556e8c6fecc2932", + "sha256:167b3f3894dcc697cefefac1d198304fae8eb4d5860a7b8bc2459d572e838470", + "sha256:225f8bba6428b8d711ca2d6c738d2e3a4667f6a2ae40f8a7a5256f69f6a3600e", + "sha256:22d2a684122dfb531853d57c8c85c1d5d44be709e12466dca99fa6aee7d8054f", + "sha256:23ceac5315fe899c229e874328742154e331fa41337bb03f6f5264636c351c9e", + "sha256:2c67105f2c6ccf5b3adbcfaecf3c5c9fa8940f9f97955c9ad7d2542151d97d93", + "sha256:30eb2d2938b9195e2c82951713429aff3ad1be9f104437d1536a04eb0cb3dc0e", + "sha256:34d51709e10ad6213b4bf094af7be7ff82bab43216b3c92a07d05b451deeca79", + "sha256:3a83e04fde663b84905f3a20213a4333d13a07b79434300704b70dc552761f8b", + "sha256:3f02c7240abab48d59f0d5fef7064f18f01a2a204616165fa6367a8abf5a8832", + "sha256:45319d2471f4dbf19893ca53785a421107266e18b8cccd2054fce1e3f72a85f1", + "sha256:4e8bf00ec1dd1d92e9ae74d2e6891f087a939e1dfb40c9c7fa5d8d6a26c94f5a", + "sha256:51e8164b1270625ff101e95c3c1c234421520c07a0a3a20ded9e9431d98afce7", + "sha256:539ad5a21141e6420035cf616bcba48d999bf878839e52692f97fc7e2f16265c", + "sha256:55f93d3822bc196b37a8bdfa4ec6a35232a399e97f2baa714bd5ed8ea9b0ce68", + "sha256:5e3a038332973b12e72236e8884dc99601a840334c2c46cfbbb5851cb94166eb", + "sha256:612d83fd1eb5aaa5401a755c1f1aafacd9dab404cd350b90d5f404c98b33e4b3", + "sha256:6a1859af761977530df2cd5c36e31d54e8d6708ad2c4656e7125c482364dc216", + "sha256:79868ec74e4907a8a6e63effe13547c6157f196a162920b1de066da5849ffb8e", + "sha256:79f20c61114c7948cf1214585066406cef4b54a9b935160980e0b6e70ada3a69", + "sha256:7c8d579d13cb81abe704c8b0908d122b81d6e2623265a19c4a6a7377f440debb", + "sha256:812b193c26553f1f375d4f1c50f805c227b24ed2d595bc9cdaf78c992ecc64a4", + "sha256:816f5285d596062c7ef22790de7d75354b58d4417a9fc64cba914aeeb900db0b", + "sha256:82f27a0134fc6bb46032ca5f728d8af984f3be94a3cb01cb70ff1224e551b9cf", + "sha256:848a28ea49166454c3bff927e5a47629eecf1aa755d6fb3290569cba0fc93766", + "sha256:861cf254bf5836d47c2aee86aa75dd93d3de00ccd1b077c3c7a2bb22cba358e7", + "sha256:8ad7c2303a338b2c0b6c6c68f101a6768725934538756096cf3388a5c07a7525", + "sha256:8ea936cf5931297ba07bce121388c4c6266c1b63a9f4d648ae16c92ff090204b", + "sha256:931aade65f77cf59f2a702ac1f549a4836ce221107c740502cbad18d6d8e9511", + "sha256:936ec37b261b226d7404eff23a9aad284098338150d42a53d6a9af12b18d3892", + "sha256:9fcd9a18ee3ac7f460e0841954feb495102ffbdbec0e6c78562f3495cda000dd", + "sha256:b1853bc34ced5ff6473e881fcf6de29da83262552c8f268a0df53b49c2b89e2c", + "sha256:b227643d8a40b68554dc7d37fcd03fc97b4fb0bd2614aeb5f2e07ab244642d36", + "sha256:b65abca78aa5ebc8675c8480b9a53006f6efea9910ad099cf32c9fb5617ef251", + "sha256:b9d0dae6dccd349b8ccf197c10ef2d05c711ca36a649c7eddbab1de2c90b63a1", + "sha256:cd6ae43ef2e596c9a88dbf2a8895be2e32cc2f5bc3c8ba2e7753b69068fc0b2d", + "sha256:e13abb14843397b76d0472c7d33cd260d5f262ab05cc27ed423317e645e29643", + "sha256:e1bdf8a107fdf9e174991aa87a0be7504f60de1ec6bfb1ccfb30e33acac818a0", + "sha256:e34b4b08d795ccca920fa26b099558f4f1e4e3f794e4ba8d3433c5bc2454d50a", + "sha256:e3c0e19bb41de6be9d8afc85795159ca16296be81a586cd9588be0400d44a855", + "sha256:ef5bb0268bfe5992da3ef9292463a5a895ed8700b134ed2c00008d5471b3ba6e", + "sha256:f2602a5c97a3d618b3b847514204ef3349fb414c59e1126c0c2c708d2c5680f8", + "sha256:f3845c4506e0d207c5e268fb02813928f3a1e135de954a379f165ef0d581da47", + "sha256:f674be92673e87dd8ee7cfe553d5960ec4effc5ab15063b9a5e265a51585a31a", + "sha256:f6d8ff62ad55dc0393686438eac4b457a916e4d1118a0b550746bb52b4c756cc", + "sha256:fb299acf3a578573c190c858d49e0cf9d75f4bc49c3f24c5a63804997ef09213", + "sha256:fed25959e4025870fdde5f895fcb126196d22affd4f4fad85a2823e0dddc85b0" + ], + "index": "pypi", + "version": "==3.0.7" + }, "dacite": { "hashes": [ "sha256:cc31ad6fdea1f49962ea42db9421772afe01ac5442380d9a99fcf3d188c61afe" @@ -356,36 +413,6 @@ "markers": "python_version >= '3.7'", "version": "==1.3.2" }, - "llvmlite": { - "hashes": [ - "sha256:04725975e5b2af416d685ea0769f4ecc33f97be541e301054c9f741003085802", - "sha256:0dd0338da625346538f1173a17cabf21d1e315cf387ca21b294ff209d176e244", - "sha256:150d0bc275a8ac664a705135e639178883293cf08c1a38de3bbaa2f693a0a867", - "sha256:1eee5cf17ec2b4198b509272cf300ee6577229d237c98cc6e63861b08463ddc6", - "sha256:210e458723436b2469d61b54b453474e09e12a94453c97ea3fbb0742ba5a83d8", - "sha256:2181bb63ef3c607e6403813421b46982c3ac6bfc1f11fa16a13eaafb46f578e6", - "sha256:24091a6b31242bcdd56ae2dbea40007f462260bc9bdf947953acc39dffd54f8f", - "sha256:2b76acee82ea0e9304be6be9d4b3840208d050ea0dcad75b1635fa06e949a0ae", - "sha256:2d92c51e6e9394d503033ffe3292f5bef1566ab73029ec853861f60ad5c925d0", - "sha256:5940bc901fb0325970415dbede82c0b7f3e35c2d5fd1d5e0047134c2c46b3281", - "sha256:8454c1133ef701e8c050a59edd85d238ee18bb9a0eb95faf2fca8b909ee3c89a", - "sha256:855f280e781d49e0640aef4c4af586831ade8f1a6c4df483fb901cbe1a48d127", - "sha256:880cb57ca49e862e1cd077104375b9d1dfdc0622596dfa22105f470d7bacb309", - "sha256:8b0a9a47c28f67a269bb62f6256e63cef28d3c5f13cbae4fab587c3ad506778b", - "sha256:92c32356f669e036eb01016e883b22add883c60739bc1ebee3a1cc0249a50828", - "sha256:92f093986ab92e71c9ffe334c002f96defc7986efda18397d0f08534f3ebdc4d", - "sha256:9564c19b31a0434f01d2025b06b44c7ed422f51e719ab5d24ff03b7560066c9a", - "sha256:b67340c62c93a11fae482910dc29163a50dff3dfa88bc874872d28ee604a83be", - "sha256:bf14aa0eb22b58c231243dccf7e7f42f7beec48970f2549b3a6acc737d1a4ba4", - "sha256:c1e1029d47ee66d3a0c4d6088641882f75b93db82bd0e6178f7bd744ebce42b9", - "sha256:df75594e5a4702b032684d5481db3af990b69c249ccb1d32687b8501f0689432", - "sha256:f19f767a018e6ec89608e1f6b13348fa2fcde657151137cb64e56d48598a92db", - "sha256:f8afdfa6da33f0b4226af8e64cfc2b28986e005528fbf944d0a24a72acfc9432", - "sha256:fa1469901a2e100c17eb8fe2678e34bd4255a3576d1a543421356e9c14d6e2ae" - ], - "markers": "python_version >= '3.8'", - "version": "==0.41.1" - }, "lxml": { "hashes": [ "sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91", @@ -575,33 +602,6 @@ "index": "pypi", "version": "==3.2.1" }, - "numba": { - "hashes": [ - "sha256:07f2fa7e7144aa6f275f27260e73ce0d808d3c62b30cff8906ad1dec12d87bbe", - "sha256:240e7a1ae80eb6b14061dc91263b99dc8d6af9ea45d310751b780888097c1aaa", - "sha256:45698b995914003f890ad839cfc909eeb9c74921849c712a05405d1a79c50f68", - "sha256:487ded0633efccd9ca3a46364b40006dbdaca0f95e99b8b83e778d1195ebcbaa", - "sha256:4e79b6cc0d2bf064a955934a2e02bf676bc7995ab2db929dbbc62e4c16551be6", - "sha256:55a01e1881120e86d54efdff1be08381886fe9f04fc3006af309c602a72bc44d", - "sha256:5c765aef472a9406a97ea9782116335ad4f9ef5c9f93fc05fd44aab0db486954", - "sha256:6fe7a9d8e3bd996fbe5eac0683227ccef26cba98dae6e5cee2c1894d4b9f16c1", - "sha256:7bf1ddd4f7b9c2306de0384bf3854cac3edd7b4d8dffae2ec1b925e4c436233f", - "sha256:811305d5dc40ae43c3ace5b192c670c358a89a4d2ae4f86d1665003798ea7a1a", - "sha256:81fe5b51532478149b5081311b0fd4206959174e660c372b94ed5364cfb37c82", - "sha256:898af055b03f09d33a587e9425500e5be84fc90cd2f80b3fb71c6a4a17a7e354", - "sha256:9e9356e943617f5e35a74bf56ff6e7cc83e6b1865d5e13cee535d79bf2cae954", - "sha256:a1eaa744f518bbd60e1f7ccddfb8002b3d06bd865b94a5d7eac25028efe0e0ff", - "sha256:bc2d904d0319d7a5857bd65062340bed627f5bfe9ae4a495aef342f072880d50", - "sha256:bcecd3fb9df36554b342140a4d77d938a549be635d64caf8bd9ef6c47a47f8aa", - "sha256:bd3dda77955be03ff366eebbfdb39919ce7c2620d86c906203bed92124989032", - "sha256:bf68df9c307fb0aa81cacd33faccd6e419496fdc621e83f1efce35cdc5e79cac", - "sha256:d3e2fe81fe9a59fcd99cc572002101119059d64d31eb6324995ee8b0f144a306", - "sha256:e63d6aacaae1ba4ef3695f1c2122b30fa3d8ba039c8f517784668075856d79e2", - "sha256:ea5bfcf7d641d351c6a80e8e1826eb4a145d619870016eeaf20bbd71ef5caa22" - ], - "index": "pypi", - "version": "==0.58.1" - }, "numpy": { "hashes": [ "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a", diff --git a/api/v1/tile.py b/api/v1/tile.py index bad307a..4887bb5 100644 --- a/api/v1/tile.py +++ b/api/v1/tile.py @@ -1,4 +1,3 @@ -import math from collections.abc import Sequence from itertools import chain from typing import Annotated @@ -8,7 +7,6 @@ import numpy as np from anyio.streams.memory import MemoryObjectSendStream from fastapi import APIRouter, Path, Response -from numba import njit from shapely.ops import transform from config import ( @@ -22,11 +20,11 @@ TILE_MAX_Z, TILE_MIN_Z, ) +from cython_lib.geo_utils import tile_to_bbox from middlewares.cache_middleware import make_cache_control from models.aed import AED from models.bbox import BBox from models.country import Country -from models.lonlat import LonLat from states.aed_state import AEDState, AEDStateDep from states.country_state import CountryState, CountryStateDep from utils import abbreviate, print_run_time @@ -34,21 +32,6 @@ router = APIRouter() -@njit(fastmath=True) -def _tile_to_lonlat(z: int, x: int, y: int) -> tuple[float, float]: - n = 2.0**z - lon_deg = x / n * 360.0 - 180.0 - lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * y / n))) - lat_deg = math.degrees(lat_rad) - return lon_deg, lat_deg - - -def _tile_to_bbox(z: int, x: int, y: int) -> BBox: - p1_lon, p1_lat = _tile_to_lonlat(z, x, y) - p2_lon, p2_lat = _tile_to_lonlat(z, x + 1, y + 1) - return BBox(LonLat(p1_lon, p2_lat), LonLat(p2_lon, p1_lat)) - - async def _count_aed_in_country(country: Country, aed_state: AEDState, send_stream: MemoryObjectSendStream) -> None: count = await aed_state.count_aeds_by_country_code(country.code) await send_stream.send((country, count)) @@ -174,7 +157,7 @@ async def get_tile( country_state: CountryStateDep, aed_state: AEDStateDep, ): - bbox = _tile_to_bbox(z, x, y) + bbox = tile_to_bbox(z, x, y) assert bbox.p1.lon <= bbox.p2.lon, f'{bbox.p1.lon=} <= {bbox.p2.lon=}' assert bbox.p1.lat <= bbox.p2.lat, f'{bbox.p1.lat=} <= {bbox.p2.lat=}' diff --git a/cython_lib/geo_utils.py b/cython_lib/geo_utils.py new file mode 100644 index 0000000..6d2eaba --- /dev/null +++ b/cython_lib/geo_utils.py @@ -0,0 +1,33 @@ +import cython + +from models.bbox import BBox +from models.lonlat import LonLat + +if cython.compiled: + from cython.cimports.libc.math import atan, pi, sinh + + print(f'{__name__}: 🐇 compiled') +else: + from math import atan, pi, sinh + + print(f'{__name__}: 🐌 not compiled') + + +@cython.cfunc +def _degrees(radians: cython.double) -> cython.double: + return radians * 180 / pi + + +@cython.cfunc +def _tile_to_lonlat(z: cython.int, x: cython.int, y: cython.int) -> tuple[cython.double, cython.double]: + n: cython.double = 2**z + lon_deg: cython.double = x / n * 360.0 - 180.0 + lat_rad: cython.double = atan(sinh(pi * (1 - 2 * y / n))) + lat_deg: cython.double = _degrees(lat_rad) + return lon_deg, lat_deg + + +def tile_to_bbox(z: cython.int, x: cython.int, y: cython.int) -> BBox: + p1_lon, p1_lat = _tile_to_lonlat(z, x, y) + p2_lon, p2_lat = _tile_to_lonlat(z, x + 1, y + 1) + return BBox(LonLat(p1_lon, p2_lat), LonLat(p2_lon, p1_lat)) diff --git a/default.nix b/default.nix index 8e92b05..8a61308 100644 --- a/default.nix +++ b/default.nix @@ -35,11 +35,11 @@ dockerTools.buildLayeredImage { cp "${./.}"/LICENSE . cp "${./.}"/Makefile . cp "${./.}"/*.py . - mkdir -p api/v1 middlewares models states - cp "${./api/v1}"/*.py api/v1 - cp "${./middlewares}"/*.py middlewares - cp "${./models}"/*.py models - cp "${./states}"/*.py states + cp -r "${./.}"/api . + cp -r "${./.}"/cython_lib . + cp -r "${./.}"/middlewares . + cp -r "${./.}"/models . + cp -r "${./.}"/states . ${shell.shellHook} ''; diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..488a4c8 --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +import Cython.Compiler.Options as Options +from Cython.Build import cythonize +from setuptools import Extension, setup + +Options.docstrings = False +Options.annotate = True + +setup( + ext_modules=cythonize( + [ + Extension( + '*', + ['cython_lib/*.py'], + extra_compile_args=[ + '-march=x86-64', + '-ffast-math', + '-fopenmp', + '-flto=auto', + ], + extra_link_args=[ + '-fopenmp', + '-flto=auto', + ], + ), + ], + compiler_directives={ + # https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#compiler-directives + 'language_level': 3, + }, + ), +) diff --git a/utils.py b/utils.py index c734ab7..430dc75 100644 --- a/utils.py +++ b/utils.py @@ -5,12 +5,10 @@ from contextlib import contextmanager from dataclasses import asdict from datetime import timedelta -from math import atan2, cos, pi, radians, sin, sqrt from typing import Any import anyio import httpx -from numba import njit from shapely.geometry import mapping from config import USER_AGENT @@ -90,47 +88,3 @@ def as_dict(data) -> dict: d[k] = mapping(v) return d - - -EARTH_RADIUS = 6371000 -CIRCUMFERENCE = 2 * pi * EARTH_RADIUS - - -@njit(fastmath=True) -def meters_to_lat(meters: float) -> float: - return meters / (CIRCUMFERENCE / 360) - - -@njit(fastmath=True) -def meters_to_lon(meters: float, lat: float) -> float: - return meters / ((CIRCUMFERENCE / 360) * cos(radians(lat))) - - -@njit(fastmath=True) -def lat_to_meters(lat: float) -> float: - return lat * (CIRCUMFERENCE / 360) - - -@njit(fastmath=True) -def lon_to_meters(lon: float, lat: float) -> float: - return lon * ((CIRCUMFERENCE / 360) * cos(radians(lat))) - - -@njit(fastmath=True) -def radians_tuple(p: tuple[float, float]) -> tuple[float, float]: - return (radians(p[0]), radians(p[1])) - - -@njit(fastmath=True) -def haversine_distance(p1: tuple[float, float], p2: tuple[float, float]) -> float: - p1_lat, p1_lon = radians_tuple(p1) - p2_lat, p2_lon = radians_tuple(p2) - - dlat = p2_lat - p1_lat - dlon = p2_lon - p1_lon - - a = sin(dlat / 2) ** 2 + cos(p1_lat) * cos(p2_lat) * sin(dlon / 2) ** 2 - c = 2 * atan2(sqrt(a), sqrt(1 - a)) - - # distance in meters - return c * EARTH_RADIUS