feat(search): search all repositories (#9949)

docs/cli.md

@@ -809,6 +809,14 @@ This command searches for packages on a remote index.
 poetry search requests pendulum
 ```
 
+{{% note %}}
+PyPI no longer allows for the search of packages without a browser. Please use https://pypi.org/search
+(via a browser) instead.
+
+For more information please see [warehouse documentation](https://warehouse.pypa.io/api-reference/xml-rpc.html#deprecated-methods)
+and this [discussion](https://discuss.python.org/t/fastly-interfering-with-pypi-search/73597/6).
+{{% /note %}}
+
 ## lock
 
 This command locks (without installing) the dependencies specified in `pyproject.toml`.

src/poetry/console/commands/search.py

@@ -21,19 +21,39 @@ class SearchCommand(Command):
     ]
 
     def handle(self) -> int:
-        from poetry.repositories.pypi_repository import PyPiRepository
-
-        results = PyPiRepository().search(self.argument("tokens"))
-
-        for result in results:
-            self.line("")
-            name = f"<info>{result.name}</>"
-
-            name += f" (<comment>{result.version}</>)"
-
-            self.line(name)
-
-            if result.description:
-                self.line(f" {result.description}")
+        seen = set()
+
+        table = self.table(style="compact")
+        table.set_headers(
+            ["<b>Package</>", "<b>Version</>", "<b>Source</>", "<b>Description</>"]
+        )
+
+        rows = []
+
+        for repository in self.poetry.pool.repositories:
+            for result in repository.search(self.argument("tokens")):
+                key = f"{repository.name}::{result.pretty_string}"
+                if key in seen:
+                    continue
+                seen.add(key)
+                rows.append((result, repository.name))
+
+        if not rows:
+            self.line("<info>No matching packages were found.</>")
+            return 0
+
+        for package, source in sorted(
+            rows, key=lambda x: (x[0].name, x[0].version, x[1])
+        ):
+            table.add_row(
+                [
+                    f"<c1>{package.name}</>",
+                    f"<b>{package.version}</b>",
+                    f"<fg=yellow;options=bold>{source}</>",
+                    str(package.description),
+                ]
+            )
+
+        table.render()
 
         return 0

src/poetry/repositories/abstract_repository.py

@@ -23,7 +23,7 @@ def name(self) -> str:
     def find_packages(self, dependency: Dependency) -> list[Package]: ...
 
     @abstractmethod
-    def search(self, query: str) -> list[Package]: ...
+    def search(self, query: str | list[str]) -> list[Package]: ...
 
     @abstractmethod
     def package(

src/poetry/repositories/legacy_repository.py

@@ -148,7 +148,7 @@ def root_page(self) -> SimpleRepositoryRootPage:
 
         return SimpleRepositoryRootPage(response.text)
 
-    def search(self, query: str) -> list[Package]:
+    def search(self, query: str | list[str]) -> list[Package]:
         results: list[Package] = []
 
         for candidate in self.root_page.search(query):

src/poetry/repositories/link_sources/html.py

@@ -81,12 +81,13 @@ def __init__(self, content: str | None = None) -> None:
         parser.feed(content or "")
         self._parsed = parser.anchors
 
-    def search(self, query: str) -> list[str]:
+    def search(self, query: str | list[str]) -> list[str]:
         results: list[str] = []
+        tokens = query if isinstance(query, list) else [query]
 
         for anchor in self._parsed:
             href = anchor.get("href")
-            if href and query in href:
+            if href and any(token in href for token in tokens):
                 results.append(href.rstrip("/"))
 
         return results

src/poetry/repositories/pypi_repository.py

@@ -55,7 +55,7 @@ def __init__(
         self._base_url = url
         self._fallback = fallback
 
-    def search(self, query: str) -> list[Package]:
+    def search(self, query: str | list[str]) -> list[Package]:
         results = []
 
         response = requests.get(

src/poetry/repositories/repository.py

@@ -73,11 +73,12 @@ def has_package(self, package: Package) -> bool:
     def add_package(self, package: Package) -> None:
         self._packages.append(package)
 
-    def search(self, query: str) -> list[Package]:
+    def search(self, query: str | list[str]) -> list[Package]:
         results: list[Package] = []
+        tokens = query if isinstance(query, list) else [query]
 
         for package in self.packages:
-            if query in package.name:
+            if any(token in package.name for token in tokens):
                 results.append(package)
 
         return results

src/poetry/repositories/repository_pool.py

@@ -182,7 +182,7 @@ def find_packages(self, dependency: Dependency) -> list[Package]:
             packages += repo.find_packages(dependency)
         return packages
 
-    def search(self, query: str) -> list[Package]:
+    def search(self, query: str | list[str]) -> list[Package]:
         results: list[Package] = []
         for repo in self.repositories:
             results += repo.search(query)

tests/console/commands/test_search.py

@@ -1,95 +1,150 @@
 from __future__ import annotations
 
+import re
+
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
 
+from poetry.repositories.pypi_repository import PyPiRepository
+
 
 if TYPE_CHECKING:
     import httpretty
 
     from cleo.testers.command_tester import CommandTester
 
+    from poetry.poetry import Poetry
+    from poetry.repositories.legacy_repository import LegacyRepository
     from tests.types import CommandTesterFactory
 
 TESTS_DIRECTORY = Path(__file__).parent.parent.parent
 FIXTURES_DIRECTORY = (
     TESTS_DIRECTORY / "repositories" / "fixtures" / "pypi.org" / "search"
 )
+SQLALCHEMY_SEARCH_OUTPUT_PYPI = """\
+ Package                  Version Source Description
+ broadway-sqlalchemy      0.0.1   PyPI   A broadway extension wrapping Flask-SQLAlchemy
+ cherrypy-sqlalchemy      0.5.3   PyPI   Use SQLAlchemy with CherryPy
+ graphene-sqlalchemy      2.2.2   PyPI   Graphene SQLAlchemy integration
+ jsonql-sqlalchemy        1.0.1   PyPI   Simple JSON-Based CRUD Query Language for SQLAlchemy
+ paginate-sqlalchemy      0.3.0   PyPI   Extension to paginate.Page that supports SQLAlchemy queries
+ sqlalchemy               1.3.10  PyPI   Database Abstraction Library
+ sqlalchemy-audit         0.1.0   PyPI   sqlalchemy-audit provides an easy way to set up revision tracking for your data.
+ sqlalchemy-dao           1.3.1   PyPI   Simple wrapper for sqlalchemy.
+ sqlalchemy-diff          0.1.3   PyPI   Compare two database schemas using sqlalchemy.
+ sqlalchemy-equivalence   0.1.1   PyPI   Provides natural equivalence support for SQLAlchemy declarative models.
+ sqlalchemy-filters       0.10.0  PyPI   A library to filter SQLAlchemy queries.
+ sqlalchemy-nav           0.0.2   PyPI   SQLAlchemy-Nav provides SQLAlchemy Mixins for creating navigation bars compatible with Bootstrap
+ sqlalchemy-plus          0.2.0   PyPI   Create Views and Materialized Views with SqlAlchemy
+ sqlalchemy-repr          0.0.1   PyPI   Automatically generates pretty repr of a SQLAlchemy model.
+ sqlalchemy-schemadisplay 1.3     PyPI   Turn SQLAlchemy DB Model into a graph
+ sqlalchemy-sqlany        1.0.3   PyPI   SAP Sybase SQL Anywhere dialect for SQLAlchemy
+ sqlalchemy-traversal     0.5.2   PyPI   UNKNOWN
+ sqlalchemy-utcdatetime   1.0.4   PyPI   Convert to/from timezone aware datetimes when storing in a DBMS
+ sqlalchemy-wrap          2.1.7   PyPI   Python wrapper for the CircleCI API
+ transmogrify-sqlalchemy  1.0.2   PyPI   Feed data from SQLAlchemy into a transmogrifier pipeline
+"""
 
 
 @pytest.fixture
 def tester(command_tester_factory: CommandTesterFactory) -> CommandTester:
     return command_tester_factory("search")
 
 
-def test_search(tester: CommandTester, http: type[httpretty.httpretty]) -> None:
+def clean_output(text: str) -> str:
+    return re.sub(r"\s+\n", "\n", text)
+
+
+def test_search(
+    tester: CommandTester, http: type[httpretty.httpretty], poetry: Poetry
+) -> None:
+    # we expect PyPI in the default behaviour
+    poetry.pool.add_repository(PyPiRepository())
+
     tester.execute("sqlalchemy")
 
-    expected = """
-sqlalchemy (1.3.10)
- Database Abstraction Library
+    output = clean_output(tester.io.fetch_output())
+
+    assert output == SQLALCHEMY_SEARCH_OUTPUT_PYPI
 
-sqlalchemy-dao (1.3.1)
- Simple wrapper for sqlalchemy.
 
-graphene-sqlalchemy (2.2.2)
- Graphene SQLAlchemy integration
+def test_search_empty_results(
+    tester: CommandTester,
+    http: type[httpretty.httpretty],
+    poetry: Poetry,
+    legacy_repository: LegacyRepository,
+) -> None:
+    poetry.pool.add_repository(legacy_repository)
 
-sqlalchemy-utcdatetime (1.0.4)
- Convert to/from timezone aware datetimes when storing in a DBMS
+    tester.execute("does-not-exist")
 
-paginate-sqlalchemy (0.3.0)
- Extension to paginate.Page that supports SQLAlchemy queries
+    output = tester.io.fetch_output()
+    assert output.strip() == "No matching packages were found."
 
-sqlalchemy-audit (0.1.0)
- sqlalchemy-audit provides an easy way to set up revision tracking for your data.
 
-transmogrify-sqlalchemy (1.0.2)
- Feed data from SQLAlchemy into a transmogrifier pipeline
+def test_search_with_legacy_repository(
+    tester: CommandTester,
+    http: type[httpretty.httpretty],
+    poetry: Poetry,
+    legacy_repository: LegacyRepository,
+) -> None:
+    poetry.pool.add_repository(PyPiRepository())
+    poetry.pool.add_repository(legacy_repository)
 
-sqlalchemy-schemadisplay (1.3)
- Turn SQLAlchemy DB Model into a graph
+    tester.execute("sqlalchemy")
 
-sqlalchemy-traversal (0.5.2)
- UNKNOWN
+    line_before = " sqlalchemy-filters       0.10.0  PyPI   A library to filter SQLAlchemy queries."
+    additional_line = " sqlalchemy-legacy        4.3.4   legacy"
+    expected = SQLALCHEMY_SEARCH_OUTPUT_PYPI.replace(
+        line_before, f"{line_before}\n{additional_line}"
+    )
 
-sqlalchemy-filters (0.10.0)
- A library to filter SQLAlchemy queries.
+    output = clean_output(tester.io.fetch_output())
 
-sqlalchemy-wrap (2.1.7)
- Python wrapper for the CircleCI API
+    assert output == expected
 
-sqlalchemy-nav (0.0.2)
- SQLAlchemy-Nav provides SQLAlchemy Mixins for creating navigation bars compatible with\
- Bootstrap
 
-sqlalchemy-repr (0.0.1)
- Automatically generates pretty repr of a SQLAlchemy model.
+def test_search_only_legacy_repository(
+    tester: CommandTester,
+    http: type[httpretty.httpretty],
+    poetry: Poetry,
+    legacy_repository: LegacyRepository,
+) -> None:
+    poetry.pool.add_repository(legacy_repository)
 
-sqlalchemy-diff (0.1.3)
- Compare two database schemas using sqlalchemy.
+    tester.execute("ipython")
 
-sqlalchemy-equivalence (0.1.1)
- Provides natural equivalence support for SQLAlchemy declarative models.
+    expected = """\
+ Package Version Source Description
+ ipython 5.7.0   legacy
+ ipython 7.5.0   legacy
+"""
 
-broadway-sqlalchemy (0.0.1)
- A broadway extension wrapping Flask-SQLAlchemy
+    output = clean_output(tester.io.fetch_output())
+    assert output == expected
 
-jsonql-sqlalchemy (1.0.1)
- Simple JSON-Based CRUD Query Language for SQLAlchemy
 
-sqlalchemy-plus (0.2.0)
- Create Views and Materialized Views with SqlAlchemy
+def test_search_multiple_queries(
+    tester: CommandTester,
+    http: type[httpretty.httpretty],
+    poetry: Poetry,
+    legacy_repository: LegacyRepository,
+) -> None:
+    poetry.pool.add_repository(legacy_repository)
 
-cherrypy-sqlalchemy (0.5.3)
- Use SQLAlchemy with CherryPy
+    tester.execute("ipython isort")
 
-sqlalchemy-sqlany (1.0.3)
- SAP Sybase SQL Anywhere dialect for SQLAlchemy
+    expected = """\
+ Package        Version Source Description
+ ipython        5.7.0   legacy
+ ipython        7.5.0   legacy
+ isort          4.3.4   legacy
+ isort-metadata 4.3.4   legacy
 """
 
-    output = tester.io.fetch_output()
+    output = clean_output(tester.io.fetch_output())
 
-    assert output == expected
+    # we use a set here to avoid ordering issues
+    assert set(output.split("\n")) == set(expected.split("\n"))

tests/repositories/fixtures/legacy/sqlalchemy-legacy.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Links for sqlalchemy-legacy</title>
+  </head>
+  <body>
+    <h1>Links for sqlalchemy-legacy</h1>
+    <a href="https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/sqlalchemy-legacy-4.3.4-py2-none-any.whl#sha256=383c39c10b5db83e8d150ac5b84d74bda96e3a1b06a30257f022dcbcd21f54b9">sqlalchemy-legacy-4.3.4-py2-none-any.whl</a><br/>
+    </body>
+</html>
+<!--SERIAL 3575149-->