Add backend support for PostgreSQL

.github/workflows/main.yml

@@ -30,6 +30,13 @@ jobs:
       max-parallel: 4
       matrix:
         python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+    services:
+      postgresql:
+        image: postgres:16
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -59,6 +66,13 @@ jobs:
           ls -l dist
   documentation:
     runs-on: ubuntu-latest
+    services:
+      postgresql:
+        image: postgres:16
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v3
       - name: Setup python

.gitignore

@@ -113,6 +113,9 @@ ENV/
 env.bak/
 venv.bak/
 
+# PyCharm project settings
+.idea
+
 # Spyder project settings
 .spyderproject
 .spyproject

README.rst

@@ -79,6 +79,23 @@ Usage
   Please access http://localhost:8000/search.html
 
 
+Development
+===========
+
+Install package in development mode::
+
+    pip install --editable='.[cli,docs,test]' --prefer-binary
+
+Start PostgreSQL server::
+
+    docker run --rm -it --publish=5432:5432 --env "POSTGRES_HOST_AUTH_METHOD=trust" postgres:16 postgres -c log_statement=all
+
+Invoke software tests::
+
+    export POSTGRES_LOG_STATEMENT=all
+    pytest -vvv
+
+
 .. _atsphinx-sqlite3fts: https://pypi.org/project/atsphinx-sqlite3fts/
 .. _Kazuya Takei: https://github.com/attakei
 .. _readthedocs-sphinx-search: https://github.com/readthedocs/readthedocs-sphinx-search

docs/conf.py

@@ -32,6 +32,7 @@
 }
 # atsphinx-sqlite3fts
 sqlite3fts_use_search_html = True
+sqlite3fts_database_url = "postgresql://postgres@localhost:5432"
 
 
 def setup(app):  # noqa: D103

pyproject.toml

@@ -35,6 +35,7 @@ dynamic = ["version", "description"]
 dependencies = [
   "docutils",
   "peewee",
+  "psycopg2[binary]",
   "Sphinx",
 ]
 

src/atsphinx/sqlite3fts/__init__.py

@@ -1,4 +1,4 @@
-"""Sphinx document searcher using SQLite3."""
+"""Sphinx document searcher using SQL database."""
 from sphinx.application import Sphinx
 
 from . import builders, events
@@ -10,9 +10,10 @@ def setup(app: Sphinx):
     """Entrypoint as Sphinx extension."""
     app.add_config_value("sqlite3fts_exclude_pages", [], "env")
     app.add_config_value("sqlite3fts_use_search_html", False, "env")
+    app.add_config_value("sqlite3fts_database_url", None, "env")
     app.add_builder(builders.SqliteBuilder)
     app.connect("config-inited", events.setup_search_html)
-    app.connect("builder-inited", events.configure_database)
+    app.connect("config-inited", events.configure_database)
     app.connect("html-page-context", events.register_document)
     app.connect("build-finished", events.save_database)
     return {

src/atsphinx/sqlite3fts/events.py

@@ -33,12 +33,23 @@ def _generate_search_html(app: Sphinx):
     app.connect("html-collect-pages", _generate_search_html)
 
 
-def configure_database(app: Sphinx):
-    """Connect database for project output."""
+def configure_database(app: Sphinx, config: Config):
+    """
+    Connect database for project output.
+
+    TODO: Add support for multiple database backends?
+    """
+    # SQLite
+    """
     db_path = Path(app.outdir) / "db.sqlite"
     if db_path.exists():
         db_path.unlink()
-    models.initialize(db_path)
+    models.initialize("sqlite", db_path)
+    """
+    # PostgreSQL
+    if not app.config.sqlite3fts_database_url:
+        raise ValueError("Configuring database failed")
+    models.initialize("postgresql", app.config.sqlite3fts_database_url)
 
 
 def register_document(

src/atsphinx/sqlite3fts/models.py

@@ -6,47 +6,49 @@
 
 TODO: Add support for multiple database backends?
 """
+import os
 from pathlib import Path
 from typing import Iterable
 
-from playhouse import sqlite_ext
+from playhouse import postgres_ext as ext
 
-db_proxy = sqlite_ext.DatabaseProxy()
+db_proxy = ext.DatabaseProxy()
 
 
-class Document(sqlite_ext.Model):
+class Document(ext.Model):
     """Document main model."""
 
-    page = sqlite_ext.TextField(null=False, unique=True)
-    title = sqlite_ext.TextField(null=False)
+    page = ext.TextField(null=False, unique=True)
+    title = ext.TextField(null=False)
 
     class Meta:  # noqa: D106
         database = db_proxy
 
 
-class Section(sqlite_ext.Model):
+class Section(ext.Model):
     """Section unit of document."""
 
-    document = sqlite_ext.ForeignKeyField(Document)
-    root = sqlite_ext.BooleanField(default=False, null=False)
-    ref = sqlite_ext.TextField(null=False)
-    title = sqlite_ext.TextField(null=False)
-    body = sqlite_ext.TextField(null=False)
+    document = ext.ForeignKeyField(Document)
+    root = ext.BooleanField(default=False, null=False)
+    ref = ext.TextField(null=False)
+    title = ext.TextField(null=False)
+    body = ext.TextField(null=False)
 
     class Meta:  # noqa: D106
         database = db_proxy
 
 
-class Content(sqlite_ext.FTS5Model):
+class Content(ext.Model):
     """Searching model."""
 
-    rowid = sqlite_ext.RowIDField()
-    title = sqlite_ext.SearchField()
-    body = sqlite_ext.SearchField()
+    rowid = ext.IntegerField()
+    title = ext.TextField()
+    body = ext.TextField()
 
     class Meta:  # noqa: D106
         database = db_proxy
-        options = {"tokenize": "trigram"}
+        # TODO: This is an option from SQLite, it does not work on other DBMS.
+        # options = {"tokenize": "trigram"}
 
 
 def store_document(document: Document, sections: Iterable[Section]):
@@ -74,16 +76,25 @@ def search_documents(keyword: str) -> Iterable[Section]:
     )
 
 
-def bind(db_path: Path):
+def bind(db_type: str, db_path: Path):
     """Bind connection.
 
     This works only set db into proxy, not included creating tables.
     """
-    db = sqlite_ext.SqliteExtDatabase(db_path)
+    if db_type == "sqlite":
+        db = ext.SqliteExtDatabase(db_path)
+    elif db_type == "postgresql":
+        db = ext.PostgresqlExtDatabase(db_path)
+        if "POSTGRES_LOG_STATEMENT" in os.environ:
+            db.execute_sql(
+                f"SET log_statement='{os.environ['POSTGRES_LOG_STATEMENT']}';"
+            )
+    else:
+        raise ValueError(f"Unknown database type: {db_type}")
     db_proxy.initialize(db)
 
 
-def initialize(db_path: Path):
+def initialize(db_type: str, db_path: Path):
     """Bind connection and create tables."""
-    bind(db_path)
+    bind(db_type, db_path)
     db_proxy.create_tables([Document, Section, Content])

tests/__init__.py

@@ -0,0 +1 @@
+"""Test package."""

tests/conftest.py

@@ -1,11 +1,40 @@
 """Configuration for pytest."""
+import os
+
 import pytest
 from sphinx.testing.path import path
 
+from tests.util import Database
+
 pytest_plugins = "sphinx.testing.fixtures"
 collect_ignore = ["roots"]
 
 
+@pytest.fixture(scope="session")
+def database_dsn():
+    """Pytest fixture providing the database connection string for software tests."""
+    return "postgresql://postgres@localhost:5432"
+
+
+@pytest.fixture(scope="session")
+def database(database_dsn):
+    """Pytest fixture returning a database wrapper object."""
+    return Database(database_dsn)
+
+
+@pytest.fixture
+def conn(database):
+    """
+    Pytest fixture returning a database wrapper object, with content cleared.
+
+    This is intended to provide each test case with a blank slate.
+    """
+    if "POSTGRES_LOG_STATEMENT" in os.environ:
+        database.execute(f"SET log_statement='{os.environ['POSTGRES_LOG_STATEMENT']}';")
+    database.reset()
+    return database
+
+
 @pytest.fixture(scope="session")
 def rootdir():
     """Set root directory to use testing sphinx project."""

tests/roots/test-default/conf.py

@@ -3,6 +3,8 @@
     "atsphinx.sqlite3fts",
 ]
 
+sqlite3fts_database_url = "postgresql://postgres@localhost:5432"
+
 # To skip toctree
 rst_prolog = """
 :orphan:

tests/test_builders.py

@@ -1,17 +1,11 @@
 """Test cases for custom builders."""
-import sqlite3
-from pathlib import Path
-
 import pytest
 from sphinx.testing.util import SphinxTestApp
 
 
 @pytest.mark.sphinx("sqlite", testroot="default")
-def test___work_builder(app: SphinxTestApp, status, warning):  # noqa
+def test___work_builder(app: SphinxTestApp, status, warning, conn):  # noqa
     app.build()
-    db_path = Path(app.outdir) / "db.sqlite"
-    assert db_path.exists()
-    conn = sqlite3.connect(db_path)
     assert len(conn.execute("SELECT * FROM document").fetchall()) > 0
     assert len(conn.execute("SELECT * FROM section").fetchall()) > 0
     assert len(conn.execute("SELECT * FROM content").fetchall()) > 0

tests/test_events.py

@@ -1,15 +1,9 @@
 """Test cases for result of events."""
-import sqlite3
-from pathlib import Path
-
 import pytest
 from sphinx.testing.util import SphinxTestApp
 
 
 @pytest.mark.sphinx("html", testroot="default")
-def test___work_builder(app: SphinxTestApp, status, warning):  # noqa
+def test___work_builder(app: SphinxTestApp, status, warning, conn):  # noqa
     app.build()
-    db_path = Path(app.outdir) / "db.sqlite"
-    assert db_path.exists()
-    conn = sqlite3.connect(db_path)
     assert len(conn.execute("SELECT * FROM document").fetchall()) > 0

tests/util.py

@@ -0,0 +1,54 @@
+"""Database wrapper utility."""
+import psycopg2
+
+
+class Database:
+    """
+    Wrap connection to the database in a test context.
+
+    TODO: Re-add adapter for SQLite.
+    TODO: Add adapters for other databases.
+    """
+
+    def __init__(self, dsn: str):
+        """Object constructor."""
+        self.dsn = dsn
+        self.conn = None
+        self.connect()
+
+    def connect(self):
+        """Connect to database, optionally disconnecting when already connected."""
+        if self.conn is not None:
+            self.conn.close()
+        self.conn = psycopg2.connect(self.dsn)
+        self.conn.autocommit = True
+
+    def execute(self, query):
+        """Invoke an SQL statement."""
+        cursor = self.conn.cursor()
+        cursor.execute(query)
+        return Result(cursor=cursor)
+
+    def reset(self):
+        """Clear database content, to provide each test case with a blank slide."""
+        cursor = self.conn.cursor()
+        cursor.execute("DELETE FROM content;")
+        cursor.execute("DELETE FROM section;")
+        cursor.execute("DELETE FROM document;")
+        cursor.close()
+
+
+class Result:
+    """Wrap SQLAlchemy result object."""
+
+    def __init__(self, cursor):
+        """Object constructor."""
+        self.cursor = cursor
+
+    def fetchall(self):
+        """Fetch all records, exhaustively."""
+        return self.cursor.fetchall()
+
+    def __del__(self):
+        """When object is destroyed, also close the cursor."""
+        self.cursor.close()