From 3c095f23decead7ef02144636334b44f86c923f8 Mon Sep 17 00:00:00 2001 From: Spoked Date: Wed, 27 Nov 2024 15:43:08 -0500 Subject: [PATCH 1/4] fix: api manual scraping fixes. wip --- src/program/services/downloaders/models.py | 47 +++++++++++++++++----- src/program/services/downloaders/shared.py | 14 ++++--- src/routers/secure/scrape.py | 26 ++++++------ 3 files changed, 59 insertions(+), 28 deletions(-) diff --git a/src/program/services/downloaders/models.py b/src/program/services/downloaders/models.py index c90dbe97..ddcd0864 100644 --- a/src/program/services/downloaders/models.py +++ b/src/program/services/downloaders/models.py @@ -47,21 +47,37 @@ class DebridFile(BaseModel): filesize: Optional[int] = Field(default=None) @classmethod - def create(cls, filename: str, filesize_bytes: int, filetype: Literal["movie", "episode"], file_id: Optional[int] = None) -> Optional["DebridFile"]: + def create( + cls, + filename: str, + filesize_bytes: int, + filetype: Literal["movie", "show", "season", "episode"], + file_id: Optional[int] = None, + limit_filesize: bool = True + + ) -> Optional["DebridFile"]: """Factory method to validate and create a DebridFile""" if not any(filename.endswith(ext) for ext in VIDEO_EXTENSIONS) and not "sample" in filename.lower(): return None - - filesize_mb = filesize_bytes / 1_000_000 - if filetype == "movie": - if not (FILESIZE_MOVIE_CONSTRAINT[0] <= filesize_mb <= FILESIZE_MOVIE_CONSTRAINT[1]): - return None - elif filetype == "episode": - if not (FILESIZE_EPISODE_CONSTRAINT[0] <= filesize_mb <= FILESIZE_EPISODE_CONSTRAINT[1]): - return None + + if limit_filesize: + filesize_mb = filesize_bytes / 1_000_000 + if filetype == "movie": + if not (FILESIZE_MOVIE_CONSTRAINT[0] <= filesize_mb <= FILESIZE_MOVIE_CONSTRAINT[1]): + return None + elif filetype in ["show", "season", "episode"]: + if not (FILESIZE_EPISODE_CONSTRAINT[0] <= filesize_mb <= FILESIZE_EPISODE_CONSTRAINT[1]): + return None return cls(filename=filename, filesize=filesize_bytes, file_id=file_id) + def to_dict(self) -> dict: + """Convert the DebridFile to a dictionary""" + return { + "file_id": self.file_id, + "filename": self.filename, + "filesize": self.filesize + } class ParsedFileData(BaseModel): """Represents a parsed file from a filename""" @@ -85,6 +101,12 @@ def file_ids(self) -> List[int]: """Get the file ids of the cached files""" return [file.file_id for file in self.files if file.file_id is not None] + def to_dict(self) -> dict: + """Convert the TorrentContainer to a dictionary""" + return { + "infohash": self.infohash, + "files": [file.to_dict() for file in self.files] + } class TorrentInfo(BaseModel): """Torrent information from a debrid service""" @@ -105,6 +127,13 @@ def size_mb(self) -> float: """Convert bytes to megabytes""" return self.bytes / 1_000_000 + def to_dict(self) -> dict: + """Convert the TorrentInfo to a dictionary""" + files = [file.to_dict() for file in self.files] + return { + **self.model_dump(), + "files": files + } class DownloadedTorrent(BaseModel): """Represents the result of a download operation""" diff --git a/src/program/services/downloaders/shared.py b/src/program/services/downloaders/shared.py index c71ee9f4..1fd96c4c 100644 --- a/src/program/services/downloaders/shared.py +++ b/src/program/services/downloaders/shared.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from datetime import datetime -from typing import Optional +from typing import Optional, Union from RTN import ParsedData, parse @@ -41,7 +41,7 @@ def get_instant_availability(self, infohash: str, item_type: str) -> Optional[To pass @abstractmethod - def add_torrent(self, infohash: str) -> int: + def add_torrent(self, infohash: str) -> Union[int, str]: """ Add a torrent and return its information @@ -49,17 +49,21 @@ def add_torrent(self, infohash: str) -> int: infohash: The hash of the torrent to add Returns: - str: The ID of the added torrent + Union[int, str]: The ID of the added torrent + + Notes: + The return type changes depending on the downloader """ pass @abstractmethod - def select_files(self, request: list[int]) -> None: + def select_files(self, torrent_id: str, file_ids: list[int]) -> None: """ Select which files to download from the torrent Args: - request: File selection details including torrent ID and file IDs + torrent_id: ID of the torrent to select files for + file_ids: IDs of the files to select """ pass diff --git a/src/routers/secure/scrape.py b/src/routers/secure/scrape.py index 132ef2fc..e801071b 100644 --- a/src/routers/secure/scrape.py +++ b/src/routers/secure/scrape.py @@ -19,6 +19,7 @@ from program.services.scrapers import Scraping from program.services.scrapers.shared import rtn from program.types import Event +from program.services.downloaders.models import TorrentContainer, TorrentInfo class Stream(BaseModel): @@ -218,13 +219,10 @@ def scrape_item(request: Request, id: str) -> ScrapeItemResponse: .unique() .scalar_one_or_none() ) - streams = scraper.scrape(item) - stream_containers = downloader.get_instant_availability([stream for stream in streams.keys()]) - for stream in streams.keys(): - if len(stream_containers.get(stream, [])) > 0: - streams[stream].is_cached = True - else: - streams[stream].is_cached = False + streams: Dict[str, Stream] = scraper.scrape(item) + for stream in streams.values(): + container = downloader.get_instant_availability(stream.infohash, item.type) + stream.is_cached = bool(container and container.cached) log_string = item.log_string return { @@ -278,10 +276,10 @@ def get_info_hash(magnet: str) -> str: session = session_manager.create_session(item_id or imdb_id, info_hash) try: - torrent_id = downloader.add_torrent(info_hash) - torrent_info = downloader.get_torrent_info(torrent_id) - containers = downloader.get_instant_availability([session.magnet]).get(session.magnet, None) - session_manager.update_session(session.id, torrent_id=torrent_id, torrent_info=torrent_info, containers=containers) + torrent_id: Union[int, str] = downloader.add_torrent(info_hash) + torrent_info: TorrentInfo = downloader.get_torrent_info(torrent_id) + container: Optional[TorrentContainer] = downloader.get_instant_availability(info_hash, item.type) + session_manager.update_session(session.id, torrent_id=torrent_id, torrent_info=torrent_info, containers=container) except Exception as e: background_tasks.add_task(session_manager.abort_session, session.id) raise HTTPException(status_code=500, detail=str(e)) @@ -290,8 +288,8 @@ def get_info_hash(magnet: str) -> str: "message": "Started manual scraping session", "session_id": session.id, "torrent_id": torrent_id, - "torrent_info": torrent_info, - "containers": containers, + "torrent_info": torrent_info.model_dump_json() if torrent_info else None, + "containers": [container.model_dump_json()] if container else None, "expires_at": session.expires_at.isoformat() } @@ -307,7 +305,7 @@ def manual_select_files(request: Request, session_id, files: Container) -> Selec raise HTTPException(status_code=404, detail="Session not found or expired") if not session.torrent_id: session_manager.abort_session(session_id) - raise HTTPException(status_code=500, detail="") + raise HTTPException(status_code=500, detail="No torrent ID found") download_type = "uncached" if files.model_dump() in session.containers: From 3f74ee5915cb67737f112aeba8079630f73f52fa Mon Sep 17 00:00:00 2001 From: Spoked Date: Wed, 27 Nov 2024 16:56:00 -0500 Subject: [PATCH 2/4] fix: manual scraping updated for downloader rework --- src/program/services/downloaders/models.py | 13 ++++++------- src/program/services/downloaders/shared.py | 6 +++--- src/routers/secure/scrape.py | 22 +++++++++++----------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/program/services/downloaders/models.py b/src/program/services/downloaders/models.py index ddcd0864..485eb7ea 100644 --- a/src/program/services/downloaders/models.py +++ b/src/program/services/downloaders/models.py @@ -104,8 +104,11 @@ def file_ids(self) -> List[int]: def to_dict(self) -> dict: """Convert the TorrentContainer to a dictionary""" return { - "infohash": self.infohash, - "files": [file.to_dict() for file in self.files] + file.file_id or str(i): { + "filename": file.filename, + "filesize": file.filesize + } + for i, file in enumerate(self.files) } class TorrentInfo(BaseModel): @@ -129,11 +132,7 @@ def size_mb(self) -> float: def to_dict(self) -> dict: """Convert the TorrentInfo to a dictionary""" - files = [file.to_dict() for file in self.files] - return { - **self.model_dump(), - "files": files - } + return self.model_dump() class DownloadedTorrent(BaseModel): """Represents the result of a download operation""" diff --git a/src/program/services/downloaders/shared.py b/src/program/services/downloaders/shared.py index 1fd96c4c..5fc08f12 100644 --- a/src/program/services/downloaders/shared.py +++ b/src/program/services/downloaders/shared.py @@ -57,7 +57,7 @@ def add_torrent(self, infohash: str) -> Union[int, str]: pass @abstractmethod - def select_files(self, torrent_id: str, file_ids: list[int]) -> None: + def select_files(self, torrent_id: Union[int, str], file_ids: list[int]) -> None: """ Select which files to download from the torrent @@ -68,7 +68,7 @@ def select_files(self, torrent_id: str, file_ids: list[int]) -> None: pass @abstractmethod - def get_torrent_info(self, torrent_id: str) -> TorrentInfo: + def get_torrent_info(self, torrent_id: Union[int, str]) -> TorrentInfo: """ Get information about a specific torrent using its ID @@ -81,7 +81,7 @@ def get_torrent_info(self, torrent_id: str) -> TorrentInfo: pass @abstractmethod - def delete_torrent(self, torrent_id: str) -> None: + def delete_torrent(self, torrent_id: Union[int, str]) -> None: """ Delete a torrent from the service diff --git a/src/routers/secure/scrape.py b/src/routers/secure/scrape.py index e801071b..8e794e40 100644 --- a/src/routers/secure/scrape.py +++ b/src/routers/secure/scrape.py @@ -103,10 +103,10 @@ def __init__(self, id: str, item_id: str, magnet: str): self.id = id self.item_id = item_id self.magnet = magnet - self.torrent_id: Optional[str] = None - self.torrent_info: Optional[dict] = None - self.containers: Optional[list] = None - self.selected_files: Optional[dict] = None + self.torrent_id: Optional[Union[int, str]] = None + self.torrent_info: Optional[TorrentInfo] = None + self.containers: Optional[TorrentContainer] = None + self.selected_files: Optional[Dict[str, Dict[str, Union[str, int]]]] = None self.created_at: datetime = datetime.now() self.expires_at: datetime = datetime.now() + timedelta(minutes=5) @@ -288,8 +288,8 @@ def get_info_hash(magnet: str) -> str: "message": "Started manual scraping session", "session_id": session.id, "torrent_id": torrent_id, - "torrent_info": torrent_info.model_dump_json() if torrent_info else None, - "containers": [container.model_dump_json()] if container else None, + "torrent_info": torrent_info.to_dict(), + "containers": [container.to_dict()] if container else None, "expires_at": session.expires_at.isoformat() } @@ -334,7 +334,7 @@ async def manual_update_attributes(request: Request, session_id, data: Union[Con raise HTTPException(status_code=404, detail="Session not found or expired") if not session.item_id: session_manager.abort_session(session_id) - raise HTTPException(status_code=500, detail="") + raise HTTPException(status_code=500, detail="No item ID found") with db.Session() as db_session: if str(session.item_id).startswith("tt") and not db_functions.get_item_by_external_id(imdb_id=session.item_id) and not db_functions.get_item_by_id(session.item_id): @@ -355,8 +355,8 @@ async def manual_update_attributes(request: Request, session_id, data: Union[Con item.reset() item.file = data.filename item.folder = data.filename - item.alternative_folder = session.torrent_info["original_filename"] - item.active_stream = {"infohash": session.magnet, "id": session.torrent_info["id"]} + item.alternative_folder = session.torrent_info.alternative_filename + item.active_stream = {"infohash": session.magnet, "id": session.torrent_info.id} torrent = rtn.rank(session.magnet, session.magnet) item.streams.append(ItemStream(torrent)) item_ids_to_submit.append(item.id) @@ -375,8 +375,8 @@ async def manual_update_attributes(request: Request, session_id, data: Union[Con item_episode.reset() item_episode.file = episode_data.filename item_episode.folder = episode_data.filename - item_episode.alternative_folder = session.torrent_info["original_filename"] - item_episode.active_stream = {"infohash": session.magnet, "id": session.torrent_info["id"]} + item_episode.alternative_folder = session.torrent_info.alternative_filename + item_episode.active_stream = {"infohash": session.magnet, "id": session.torrent_info.id} torrent = rtn.rank(session.magnet, session.magnet) item_episode.streams.append(ItemStream(torrent)) item_ids_to_submit.append(item_episode.id) From e318f9351d98d6f6cfe9b9c8f66cc1c1c1a2a211 Mon Sep 17 00:00:00 2001 From: Spoked Date: Sun, 1 Dec 2024 01:19:47 -0500 Subject: [PATCH 3/4] fix: add strong typed response to scrape api endpoint --- src/program/services/downloaders/models.py | 19 ------------------- src/routers/secure/scrape.py | 14 ++++++++------ 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/src/program/services/downloaders/models.py b/src/program/services/downloaders/models.py index 485eb7ea..5d09181b 100644 --- a/src/program/services/downloaders/models.py +++ b/src/program/services/downloaders/models.py @@ -71,13 +71,6 @@ def create( return cls(filename=filename, filesize=filesize_bytes, file_id=file_id) - def to_dict(self) -> dict: - """Convert the DebridFile to a dictionary""" - return { - "file_id": self.file_id, - "filename": self.filename, - "filesize": self.filesize - } class ParsedFileData(BaseModel): """Represents a parsed file from a filename""" @@ -101,15 +94,6 @@ def file_ids(self) -> List[int]: """Get the file ids of the cached files""" return [file.file_id for file in self.files if file.file_id is not None] - def to_dict(self) -> dict: - """Convert the TorrentContainer to a dictionary""" - return { - file.file_id or str(i): { - "filename": file.filename, - "filesize": file.filesize - } - for i, file in enumerate(self.files) - } class TorrentInfo(BaseModel): """Torrent information from a debrid service""" @@ -130,9 +114,6 @@ def size_mb(self) -> float: """Convert bytes to megabytes""" return self.bytes / 1_000_000 - def to_dict(self) -> dict: - """Convert the TorrentInfo to a dictionary""" - return self.model_dump() class DownloadedTorrent(BaseModel): """Represents the result of a download operation""" diff --git a/src/routers/secure/scrape.py b/src/routers/secure/scrape.py index 8e794e40..3df265c6 100644 --- a/src/routers/secure/scrape.py +++ b/src/routers/secure/scrape.py @@ -39,8 +39,8 @@ class StartSessionResponse(BaseModel): message: str session_id: str torrent_id: str - torrent_info: dict - containers: Optional[List[dict]] + torrent_info: TorrentInfo + containers: Optional[List[TorrentContainer]] expires_at: str class SelectFilesResponse(BaseModel): @@ -276,7 +276,7 @@ def get_info_hash(magnet: str) -> str: session = session_manager.create_session(item_id or imdb_id, info_hash) try: - torrent_id: Union[int, str] = downloader.add_torrent(info_hash) + torrent_id: str = downloader.add_torrent(info_hash) torrent_info: TorrentInfo = downloader.get_torrent_info(torrent_id) container: Optional[TorrentContainer] = downloader.get_instant_availability(info_hash, item.type) session_manager.update_session(session.id, torrent_id=torrent_id, torrent_info=torrent_info, containers=container) @@ -284,15 +284,17 @@ def get_info_hash(magnet: str) -> str: background_tasks.add_task(session_manager.abort_session, session.id) raise HTTPException(status_code=500, detail=str(e)) - return { + data = { "message": "Started manual scraping session", "session_id": session.id, "torrent_id": torrent_id, - "torrent_info": torrent_info.to_dict(), - "containers": [container.to_dict()] if container else None, + "torrent_info": torrent_info, + "containers": [container] if container else None, "expires_at": session.expires_at.isoformat() } + return StartSessionResponse(**data) + @router.post( "/scrape/select_files/{session_id}", summary="Select files for torrent id, for this to be instant it requires files to be one of /manual/instant_availability response containers", From c779b6830be397c2c3c96b0b659d7908c95cc1d7 Mon Sep 17 00:00:00 2001 From: Spoked Date: Thu, 5 Dec 2024 17:24:46 -0500 Subject: [PATCH 4/4] fix: add alldebrid as option in mediafusion --- src/program/services/scrapers/mediafusion.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/program/services/scrapers/mediafusion.py b/src/program/services/scrapers/mediafusion.py index 5a5753b6..b0b018e6 100644 --- a/src/program/services/scrapers/mediafusion.py +++ b/src/program/services/scrapers/mediafusion.py @@ -66,6 +66,9 @@ def validate(self) -> bool: elif self.app_settings.downloaders.torbox.enabled: self.api_key = self.app_settings.downloaders.torbox.api_key self.downloader = "torbox" + elif self.app_settings.downloaders.all_debrid.enabled: + self.api_key = self.app_settings.downloaders.all_debrid.api_key + self.downloader = "alldebrid" else: logger.error("No downloader enabled, please enable at least one.") return False