diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 847f7337..16bfa827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,34 +11,38 @@ on: workflow_dispatch: jobs: - ci-venv: - name: CI (venv) + ci: + name: "CI ${{ matrix.python-version }} (conda: ${{ matrix.conda }})" runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - run: ./run-tests.sh - - ci-conda: - name: CI (conda) - runs-on: ubuntu-latest + conda: [true, false] + python-version: ["3.11"] defaults: run: # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell shell: bash -el {0} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v2 + if: ${{ matrix.conda == true }} with: activate-environment: scitt environment-file: environment.yml - - run: python -m pytest + python-version: ${{ matrix.python-version }} + - name: Run tests with conda + if: ${{ matrix.conda == true }} + run: | + python -m pip install -e . + python -m pytest + - name: Set up Python ${{ matrix.python-version }} + if: ${{ matrix.conda == false }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Run tests with venv + if: ${{ matrix.conda == false }} + run: ./run-tests.sh ci-cd-build-and-push-image-container: name: CI/CD (container) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 00000000..6080ef15 --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,23 @@ +# Your .github/workflows/coverity.yml file. +name: Coverity Scan + +# We only want to test official release code, not every pull request. +on: + push: + branches: + - '**' + +permissions: + contents: read + +jobs: + coverity: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: vapier/coverity-scan-action@cae3c096a2eb21c431961a49375ac17aea2670ce # v1.7.0 + with: + email: ${{ secrets.COVERITY_SCAN_EMAIL }} + token: ${{ secrets.COVERITY_SCAN_TOKEN }} + build_language: 'other' + command: '--no-command --fs-capture-search ./ --fs-capture-search-exclude-regex /cov-analysis/.*' diff --git a/Dockerfile b/Dockerfile index d27b78da..3ddd4c72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # Virtual CCF (non-SGX) build and run: # $ docker build -t ghcr.io/scitt-community/scitt-api-emulator:main --progress plain . # $ docker run --rm -ti -w /src/src/scitt-api-emulator -v $PWD:/src/src/scitt-api-emulator -p 8000:8000 ghcr.io/scitt-community/scitt-api-emulator:main -FROM python:3.8 +FROM python:3.11 WORKDIR /usr/src/scitt-api-emulater @@ -15,4 +15,8 @@ COPY . . RUN pip install --no-cache-dir -e . +# CWE-269 Configure alternate docker user +RUN useradd scitt +USER scitt + CMD scitt-emulator server --workspace workspace/ --tree-alg CCF diff --git a/README.md b/README.md index 7bf84e9d..13f61e3e 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,14 @@ They can be used with the built-in server or an external service implementation. ```sh ./scitt-emulator.sh client create-claim \ - --issuer did:web:example.com \ --content-type application/json \ + --subject 'solar' \ --payload '{"sun": "yellow"}' \ --out claim.cose ``` - _**Note:** The emulator generates an ad-hoc key pair to sign the claim and does not verify claim signatures upon submission._ + _**Note:** The emulator generates an ad-hoc key pair to sign the claim if +``--issuer`` and ``--public-key-pem`` are not given. See [Registration Policies](docs/registration_policies.md) docs for more deatiled examples_ 2. View the signed claim by uploading `claim.cose` to one of the [CBOR or COSE Debugging Tools](#cose-and-cbor-debugging) diff --git a/demos/apple/alice.sh b/demos/apple/alice.sh new file mode 100644 index 00000000..bc2f12e6 --- /dev/null +++ b/demos/apple/alice.sh @@ -0,0 +1,8 @@ +jq < ${HOME}/Documents/fediverse/scitt_federation_alice/config.json \ +&& sleep 2 \ +&& scitt-emulator server \ + --workspace ${HOME}/Documents/fediverse/scitt_federation_alice/workspace_alice/ \ + --tree-alg CCF \ + --port 7000 \ + --middleware scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ + --middleware-config-path ${HOME}/Documents/fediverse/scitt_federation_alice/config.json diff --git a/demos/apple/bob-webhook.sh b/demos/apple/bob-webhook.sh new file mode 100644 index 00000000..47350e28 --- /dev/null +++ b/demos/apple/bob-webhook.sh @@ -0,0 +1,4 @@ +gh webhook forward \ + --repo=pdxjohnny/scitt-api-emulator \ + --events=push \ + --url=https://scitt.bob.chadig.com/github-webhook-notary/ diff --git a/demos/apple/bob.sh b/demos/apple/bob.sh new file mode 100644 index 00000000..d023a157 --- /dev/null +++ b/demos/apple/bob.sh @@ -0,0 +1,12 @@ +jq < ${HOME}/Documents/fediverse/scitt_federation_bob/config.json \ +&& sleep 2 \ +&& scitt-emulator server \ + --workspace ${HOME}/Documents/fediverse/scitt_federation_bob/workspace_bob/ \ + --tree-alg CCF \ + --port 6000 \ + --middleware \ + scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ + scitt_emulator.github_webhook_notary:GitHubWebhookNotaryMiddleware \ + --middleware-config-path \ + ${HOME}/Documents/fediverse/scitt_federation_bob/config.json \ + - diff --git a/demos/apple/get_statement_from_alice.sh b/demos/apple/get_statement_from_alice.sh new file mode 100644 index 00000000..8d2101e4 --- /dev/null +++ b/demos/apple/get_statement_from_alice.sh @@ -0,0 +1,3 @@ +curl -sfL https://github.com/scitt-community/scitt-api-emulator/archive/$(git log -n 1 --format=%H).tar.gz | sha384sum - | awk '{print $1}' + +scitt-emulator client retrieve-claim --entry-id sha384:fe1952f763cf8947b6bc49902d7ec5f4a006c9358d2c6349b07896bf0967ebb7395eba7b30c9b7896b4096bc140a5f42 --url https://scitt.unstable.chadig.com --out webhook.push.cose diff --git a/docs/federation_activitypub.md b/docs/federation_activitypub.md new file mode 100644 index 00000000..d1c26f68 --- /dev/null +++ b/docs/federation_activitypub.md @@ -0,0 +1,198 @@ +# Federation via ActivityPub + +- Federation of SCITT events enables near real-time communication between supply + chains. + - Acceptance of claims to SCITT where payload data contains VEX, CSAF, VSA, + SBOM, VDR, VRF, S2C2F alignment attestations, etc. has the side effect of + enabling a consistent pattern for notification of new vulnerability + and other Software Supply Chain Security data. +- References + - [SCITT Architecture: 7. Federation](https://www.ietf.org/archive/id/draft-ietf-scitt-architecture-02.html#name-federation) + - https://www.w3.org/TR/activitypub/ + - [OpenSSF Stream 8](https://openssf.org/oss-security-mobilization-plan/): + Coordinate Industry-Wide Data Sharing to Improve the Research That Helps + Determine the Most Critical OSS Components + +```mermaid +flowchart LR + subgraph alice[Alice] + subgraph aliceSCITT[SCITT] + alice_submit_claim[Submit Statement] + alice_receipt_created[Receipt Created] + + alice_submit_claim --> alice_receipt_created + end + end + subgraph bob[Bob] + subgraph bobSCITT[SCITT] + bob_submit_claim[Submit Statement] + bob_receipt_created[Receipt Created] + bob_make_statement_available_created[Serve Statement] + + bob_submit_claim --> bob_receipt_created + bob_submit_claim --> bob_make_statement_available_created + end + subgraph bobActivityPubActor[ActivityPub Actor] + bob_inbox[Inbox] + end + + bob_inbox --> bob_submit_claim + end + subgraph eve[Eve] + subgraph eve_client[Submit to Alice, Retrieve from Bob and verify] + eve_submit_claim[Submit Statement] + eve_retrieve_statement[Retrieve Statement] + eve_retrieve_receipt[Retrieve Receipt] + eve_verify_receipt[Verify Receipt] + end + end + + eve_submit_claim --> alice_submit_claim + + eve_retrieve_statement --> eve_verify_receipt + eve_retrieve_receipt --> eve_verify_receipt + bob_make_statement_available_created --> eve_retrieve_statement + bob_receipt_created --> eve_retrieve_receipt + + alice_receipt_created --> bob_inbox +``` + +> Below links to recording of IETF 118 SCITT Meeting, Corresponding asciinema link: https://asciinema.org/a/619517 + +[![asciicast-federation-activitypub-bovine](https://asciinema.org/a/619517.svg)](https://www.youtube.com/watch?v=zEGob4oqca4&t=5354s) + +## Dependencies + +Install the SCITT API Emulator with the `federation-activitypub-bovine` extra. + +- https://bovine-herd.readthedocs.io/en/latest/deployment.html + - Bovine and associated libraries **require Python 3.11 or greater!!!** + +```console +$ pip install -e .[federation-activitypub-bovine] +``` + +## Example of Federating Statements / Receipts Across SCITT Instances + +> Please refer to the [Registration Policies](registration_policies.md) doc for +> more information about claim insert policies. + +In this example Alice and Bob each have their own instance of SCITT. Alice's +insert policy differs from Bob's slightly. Alice and Bob's instances federate +with each other. This means when claims are inserted into one instance and are +given and entry ID and a receipt at notification is sent to the other instance. +The other instance decides if it wants to create a corresponding entry ID and +receipt local to it. + +Federation can be helpful when some aspects of insert policy validation are +shared. By federating with entities an instance trusts for those aspects of +insert policy and instance and it's owner(s) may be able to reduce investment in +compute or other activities required for claim validation. + +As a more specific example, entities may share a common set of insert policy +criteria defined in a collaborative manner (such as a working group). +Attestations of alignment to the [S2C2F](https://github.com/ossf/s2c2f/blob/main/specification/framework.md#appendix-relation-to-scitt) +are one such example. In addition to the requirements / evaluation criteria +defined by the OpenSSF's Supply Chain Integrity Working Group an entity may +desire to evaluate attestations of alignment with added requirements appropriate +to their usage/deployment context and it's threat model. + +By the end of this tutorial you will have four terminals open. + +- One for the ActivityPub Server +- One for Bob's SCITT Instance +- One for Alice's SCITT Instance +- One for submitting claims to Bob and Alice's SCITT instances and querying + their ActivityPub Actors. + +### S2C2F Notes + +- ING-4: Mirror a copy of all OSS source code to an internal location + - One might also want to mirror trust attestations, integrity data, etc. to + ensure availability. Federation could assist with keeping mirrors as up to + date as possible. + +### Bring up Bob's SCITT Instance + +Populate Bob's federation config + +**~/Documents/fediverse/scitt_federation_bob/config.json** + +```json +{ + "handle_name": "bob", + "fqdn": "scitt.bob.chadig.com", + "workspace": "~/Documents/fediverse/scitt_federation_bob/", + "bovine_db_url": "~/Documents/fediverse/scitt_federation_bob/bovine.sqlite3", + "following": { + "alice": { + "actor_id": "alice@scitt.alice.chadig.com" + } + } +} +``` + +Start the server + +```console +$ rm -rf workspace_bob/ +$ mkdir -p workspace_bob/storage/operations +$ scitt-emulator server \ + --workspace ${HOME}/Documents/fediverse/scitt_federation_bob/workspace_bob/ --tree-alg CCF --port 6000 \ + --middleware scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ + --middleware-config-path ${HOME}/Documents/fediverse/scitt_federation_bob/config.json +``` + +### Bring up Alice's SCITT Instance + +Populate Alice's federation config + +**~/Documents/fediverse/scitt_federation_alice/config.json** + +```json +{ + "handle_name": "alice", + "fqdn": "scitt.alice.chadig.com", + "workspace": "~/Documents/fediverse/scitt_federation_alice/", + "bovine_db_url": "~/Documents/fediverse/scitt_federation_alice/bovine.sqlite3", + "following": { + "bob": { + "actor_id": "bob@scitt.bob.chadig.com" + } + } +} +``` + +Start the server + +```console +$ rm -rf workspace_alice/ +$ mkdir -p workspace_alice/storage/operations +$ scitt-emulator server \ + --workspace ${HOME}/Documents/fediverse/scitt_federation_alice/workspace_alice/ --tree-alg CCF --port 7000 \ + --middleware scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ + --middleware-config-path ${HOME}/Documents/fediverse/scitt_federation_alice/config.json +``` + +### Create and Submit Statement to Alice's Instance + +```console +$ scitt-emulator client create-claim --issuer did:web:example.org --content-type application/json --subject solar --payload '{"sun": "yellow"}' --out claim.cose +Claim written to claim.cose +$ scitt-emulator client submit-claim --url http://localhost:7000 --claim claim.cose --out claim.receipt.cbor +Claim registered with entry ID sha384:76303a87c3ff728578d1e941ec4422193367e31fd37ab178257536cba79724d6411c457cd3c47654975dc924ff023123 +Receipt written to claim.receipt.cbor +``` + +### Download Receipt from Bob's Instance + +```console +$ scitt-emulator client retrieve-claim --url http://localhost:6000 --out federated.claim.cose --entry-id sha384:76303a87c3ff728578d1e941ec4422193367e31fd37ab178257536cba79724d6411c457cd3c47654975dc924ff023123 +Claim written to federated.claim.cose +$ scitt-emulator client retrieve-receipt --url http://localhost:6000 --out federated.claim.receipt.cbor --entry-id sha384:76303a87c3ff728578d1e941ec4422193367e31fd37ab178257536cba79724d6411c457cd3c47654975dc924ff023123 +Receipt written to federated.claim.receipt.cbor +$ scitt-emulator client verify-receipt --claim federated.claim.cose --receipt federated.claim.receipt.cbor --service-parameters workspace_alice/service_parameters.json +Leaf hash: 7d8501f1aea9b095b9730dab05f8866c0c9d0e33e6f3f2c7131ff4a3ca1ddf61 +Root: fceb0aa5ac260542753b5086d512fe3bb074ef39ac3becc5d9ce857b020b85fb +Receipt verified +``` diff --git a/docs/registration_policies.md b/docs/registration_policies.md index 61c63a7a..ed1bb3c3 100644 --- a/docs/registration_policies.md +++ b/docs/registration_policies.md @@ -12,14 +12,14 @@ The SCITT API emulator can deny entry based on presence of This is a simple way to enable evaluation of claims prior to submission by arbitrary policy engines which watch the workspace (fanotify, inotify, etc.). -[![asciicast-of-simple-decoupled-file-based-policy-engine](https://asciinema.org/a/572766.svg)](https://asciinema.org/a/572766) +[![asciicast-of-simple-decoupled-file-based-policy-engine](https://asciinema.org/a/620587.svg)](https://asciinema.org/a/620587) Start the server ```console $ rm -rf workspace/ $ mkdir -p workspace/storage/operations -$ scitt-emulator server --workspace workspace/ --tree-alg CCF --use-lro +$ timeout 1s scitt-emulator server --workspace workspace/ --tree-alg CCF --use-lro Service parameters: workspace/service_parameters.json ^C ``` @@ -84,43 +84,58 @@ import os import sys import json import pathlib -import traceback +import unittest -import cbor2 +import cwt import pycose +from pycose.messages import Sign1Message from jsonschema import validate, ValidationError -from pycose.messages import CoseMessage, Sign1Message - -from scitt_emulator.scitt import ClaimInvalidError, COSE_Headers_Issuer -claim = sys.stdin.buffer.read() +from scitt_emulator.scitt import ClaimInvalidError, CWTClaims +from scitt_emulator.verify_statement import verify_statement -msg = CoseMessage.decode(claim) -if pycose.headers.ContentType not in msg.phdr: - raise ClaimInvalidError("Claim does not have a content type header parameter") -if COSE_Headers_Issuer not in msg.phdr: - raise ClaimInvalidError("Claim does not have an issuer header parameter") +def main(): + claim = sys.stdin.buffer.read() -if not msg.phdr[pycose.headers.ContentType].startswith("application/json"): - raise TypeError( - f"Claim content type does not start with application/json: {msg.phdr[pycose.headers.ContentType]!r}" - ) + msg = Sign1Message.decode(claim, tag=True) -SCHEMA = json.loads(pathlib.Path(os.environ["SCHEMA_PATH"]).read_text()) + if pycose.headers.ContentType not in msg.phdr: + raise ClaimInvalidError("Claim does not have a content type header parameter") + if not msg.phdr[pycose.headers.ContentType].startswith("application/json"): + raise TypeError( + f"Claim content type does not start with application/json: {msg.phdr[pycose.headers.ContentType]!r}" + ) -try: - validate( - instance={ - "$schema": "https://schema.example.com/scitt-policy-engine-jsonschema.schema.json", - "issuer": msg.phdr[COSE_Headers_Issuer], - "claim": json.loads(msg.payload.decode()), - }, - schema=SCHEMA, + cwt_cose_key, _pycose_cose_key = verify_statement(msg) + unittest.TestCase().assertTrue( + cwt_cose_key, + "Failed to verify signature on statement", ) -except ValidationError as error: - print(str(error), file=sys.stderr) - sys.exit(1) + + cwt_protected = cwt.decode(msg.phdr[CWTClaims], cwt_cose_key) + issuer = cwt_protected[1] + subject = cwt_protected[2] + + SCHEMA = json.loads(pathlib.Path(os.environ["SCHEMA_PATH"]).read_text()) + + try: + validate( + instance={ + "$schema": "https://schema.example.com/scitt-policy-engine-jsonschema.schema.json", + "issuer": issuer, + "subject": subject, + "claim": json.loads(msg.payload.decode()), + }, + schema=SCHEMA, + ) + except ValidationError as error: + print(str(error), file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() ``` We'll create a small wrapper to serve in place of a more fully featured policy @@ -140,21 +155,110 @@ echo ${CLAIM_PATH} Example running allowlist check and enforcement. ```console -npm install -g nodemon -nodemon -e .cose --exec 'find workspace/storage/operations -name \*.cose -exec nohup sh -xe policy_engine.sh $(cat workspace/service_parameters.json | jq -r .insertPolicy) {} \;' +$ npm install nodemon && \ + node_modules/.bin/nodemon -e .cose --exec 'find workspace/storage/operations -name \*.cose -exec nohup sh -xe policy_engine.sh $(cat workspace/service_parameters.json | jq -r .insertPolicy) {} \;' ``` Also ensure you restart the server with the new config we edited. ```console -scitt-emulator server --workspace workspace/ --tree-alg CCF --use-lro +$ scitt-emulator server --workspace workspace/ --tree-alg CCF --use-lro +``` + +The current emulator notary (create-statement) implementation will sign +statements using a generated ephemeral key or a key we provide via the +`--private-key-pem` argument. + +Since we need to export the key for verification by the policy engine, we will +first generate it using `ssh-keygen`. + +```console +$ export ISSUER_PORT="9000" \ + && export ISSUER_URL="http://localhost:${ISSUER_PORT}" \ + && ssh-keygen -q -f /dev/stdout -t ecdsa -b 384 -N '' -I $RANDOM <</dev/null | python -c 'import sys; from cryptography.hazmat.primitives import serialization; print(serialization.load_ssh_private_key(sys.stdin.buffer.read(), password=None).private_bytes(encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()).decode().rstrip())' > private-key.pem \ + && scitt-emulator client create-claim \ + --private-key-pem private-key.pem \ + --issuer "${ISSUER_URL}" \ + --subject "solar" \ + --content-type application/json \ + --payload '{"sun": "yellow"}' \ + --out claim.cose ``` -Create claim from allowed issuer (`.org`) and from non-allowed (`.com`). +The core of policy engine we implemented in `jsonschema_validator.py` will +verify the COSE message generated using the public portion of the notary's key. +We've implemented two possible styles of key resolution. Both of them require +resolution of public keys via an HTTP server. + +Let's start the HTTP server now, we'll populate the needed files in the +sections corresponding to each resolution style. + +```console +$ python -m http.server "${ISSUER_PORT}" & +$ python_http_server_pid=$! +``` + +### SSH `authorized_keys` style notary public key resolution + +Keys are discovered via making an HTTP GET request to the URL given by the +`issuer` parameter via the `web` DID method and de-serializing the SSH +public keys found within the response body. + +GitHub exports a users authentication keys at https://github.com/username.keys +Leveraging this URL as an issuer `did:web:github.com:username.keys` with the +following pattern would enable a GitHub user to act as a SCITT notary. + +Start an HTTP server with an SSH public key served at the root. + +```console +$ cat private-key.pem | ssh-keygen -f /dev/stdin -y | tee index.html +``` + +### OpenID Connect token style notary public key resolution + +Keys are discovered two part resolution of HTTP paths relative to the issuer + +`/.well-known/openid-configuration` path is requested via HTTP GET. The +response body is parsed as JSON and the value of the `jwks_uri` key is +requested via HTTP GET. + +`/.well-known/jwks` (is typically the value of `jwks_uri`) path is requested +via HTTP GET. The response body is parsed as JSON. Public keys are loaded +from the value of the `keys` key which stores an array of JSON Web Key (JWK) +style serializations. + +```console +$ mkdir -p .well-known/ +$ cat > .well-known/openid-configuration < @@ -174,10 +278,27 @@ Failed validating 'enum' in schema['properties']['issuer']: On instance['issuer']: 'did:web:example.com' +``` + +Modify the allowlist to ensure that our issuer, aka our local HTTP server with +our keys, is set to be the allowed issuer. + +```console +$ export allowlist="$(cat allowlist.schema.json)" && \ + jq '.properties.issuer.enum[0] = env.ISSUER_URL' <(echo "${allowlist}") \ + | tee allowlist.schema.json +``` + +Submit the statement from the issuer we just added to the allowlist. -$ scitt-emulator client create-claim --issuer did:web:example.org --content-type application/json --payload '{"sun": "yellow"}' --out claim.cose -A COSE signed Claim was written to: claim.cose +```console $ scitt-emulator client submit-claim --claim claim.cose --out claim.receipt.cbor Claim registered with entry ID 1 Receipt written to claim.receipt.cbor ``` + +Stop the server that serves the public keys + +```console +$ kill $python_http_server_pid +``` diff --git a/environment.yml b/environment.yml index 90e61fa8..b9696e33 100644 --- a/environment.yml +++ b/environment.yml @@ -38,3 +38,13 @@ dependencies: - jsonschema==4.17.3 - jwcrypto==1.5.0 - PyJWT==2.8.0 + - werkzeug==2.2.2 + - cwt==2.7.1 + - py-multibase==1.0.3 + - py-multicodec==0.2.1 + - tomli-w==1.0.0 + - aiohttp==3.9.0 + - bovine==0.5.3 + - bovine-store==0.5.3 + - bovine-herd==0.5.3 + - bovine-pubsub==0.5.3 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..8efa2302 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +# https://docs.pytest.org/en/7.1.x/how-to/doctest.html#using-doctest-options +doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL +# Alternatively, options can be enabled by an inline comment in the doc test itself: +# >>> something_that_raises() # doctest: +IGNORE_EXCEPTION_DETAIL +# Traceback (most recent call last): +# ValueError: ... +addopts = --doctest-modules diff --git a/run-tests.sh b/run-tests.sh index de8eadb1..cc67c0a3 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -10,7 +10,7 @@ if [ ! -f "venv/bin/activate" ]; then . ./venv/bin/activate pip install -q -U pip setuptools wheel pip install -q -r dev-requirements.txt - pip install -q -e .[oidc] + pip install -q -e .[oidc,federation-activitypub-bovine] else . ./venv/bin/activate fi diff --git a/scitt_emulator/ccf.py b/scitt_emulator/ccf.py index 825c7eea..62a6a4a4 100644 --- a/scitt_emulator/ccf.py +++ b/scitt_emulator/ccf.py @@ -18,15 +18,20 @@ from cryptography.hazmat.primitives import hashes from scitt_emulator.scitt import SCITTServiceEmulator +from scitt_emulator.signals import SCITTSignals +from scitt_emulator.federation import SCITTFederation class CCFSCITTServiceEmulator(SCITTServiceEmulator): tree_alg = "CCF" def __init__( - self, service_parameters_path: Path, storage_path: Optional[Path] = None + self, + signals: SCITTSignals, + service_parameters_path: Path, + storage_path: Optional[Path] = None, ): - super().__init__(service_parameters_path, storage_path) + super().__init__(signals, service_parameters_path, storage_path) if storage_path is not None: self._service_private_key_path = ( self.storage_path / "service_private_key.pem" diff --git a/scitt_emulator/client.py b/scitt_emulator/client.py index b4ff35ee..eb9f9ee9 100644 --- a/scitt_emulator/client.py +++ b/scitt_emulator/client.py @@ -8,8 +8,9 @@ import httpx -import scitt_emulator.scitt as scitt +from scitt_emulator import create_statement from scitt_emulator.tree_algs import TREE_ALGS +from scitt_emulator.signals import SCITTSignals DEFAULT_URL = "http://127.0.0.1:8000" CONNECT_RETRIES = 3 @@ -72,10 +73,6 @@ def post(self, *args, **kwargs): return self._request("POST", *args, **kwargs) -def create_claim(issuer: str, content_type: str, payload: str, claim_path: Path): - scitt.create_claim(claim_path, issuer, content_type, payload) - - def submit_claim( url: str, claim_path: Path, @@ -161,7 +158,7 @@ def verify_receipt(cose_path: Path, receipt_path: Path, service_parameters_path: service_parameters = json.load(f) clazz = TREE_ALGS[service_parameters["treeAlgorithm"]] - service = clazz(service_parameters_path=service_parameters_path) + service = clazz(signals=SCITTSignals(), service_parameters_path=service_parameters_path) service.verify_receipt(cose_path, receipt_path) print("Receipt verified") @@ -170,16 +167,7 @@ def cli(fn): parser = fn(description="Execute client commands") sub = parser.add_subparsers(dest="cmd", help="Command to execute", required=True) - p = sub.add_parser("create-claim", description="Create a fake SCITT claim") - p.add_argument("--out", required=True, type=Path) - p.add_argument("--issuer", required=True, type=str) - p.add_argument("--content-type", required=True, type=str) - p.add_argument("--payload", required=True, type=str) - p.set_defaults( - func=lambda args: scitt.create_claim( - args.out, args.issuer, args.content_type, args.payload - ) - ) + create_statement.cli(sub.add_parser) p = sub.add_parser( "submit-claim", description="Submit a SCITT claim and retrieve the receipt" diff --git a/scitt_emulator/create_statement.py b/scitt_emulator/create_statement.py new file mode 100644 index 00000000..02d6ec78 --- /dev/null +++ b/scitt_emulator/create_statement.py @@ -0,0 +1,220 @@ +# Copyright (c) SCITT Authors +# Licensed under the MIT License. +import pathlib +import argparse +from typing import Union, Optional + +import cwt +import pycose +import pycose.headers +import pycose.messages +import pycose.keys.ec2 +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +# NOTE These are unmaintained but the +# https://github.com/hashberg-io/multiformats stuff and base58 modules don't +# produce the same results: +# https://grotto-networking.com/blog/posts/DID_Key.html#bug-in-multibase-library +import multibase +import multicodec + +# TODO jwcrypto is LGPLv3, is there another option with a permissive licence? +import jwcrypto.jwk + +from scitt_emulator.did_helpers import DID_KEY_METHOD, MULTICODEC_HEX_P384_PUBLIC_KEY + + +@pycose.headers.CoseHeaderAttribute.register_attribute() +class CWTClaims(pycose.headers.CoseHeaderAttribute): + identifier = 14 + fullname = "CWT_CLAIMS" + + +@pycose.headers.CoseHeaderAttribute.register_attribute() +class RegInfo(pycose.headers.CoseHeaderAttribute): + identifier = 393 + fullname = "REG_INFO" + + +@pycose.headers.CoseHeaderAttribute.register_attribute() +class Receipt(pycose.headers.CoseHeaderAttribute): + identifier = 394 + fullname = "RECEIPT" + + +@pycose.headers.CoseHeaderAttribute.register_attribute() +class TBD(pycose.headers.CoseHeaderAttribute): + identifier = 395 + fullname = "TBD" + + +def create_claim( + claim_path: pathlib.Path, + issuer: Union[str, None], + subject: str, + content_type: str, + payload: str, + private_key_pem_path: Optional[str] = None, +): + # https://ietf-wg-scitt.github.io/draft-ietf-scitt-architecture/draft-ietf-scitt-architecture.html#name-signed-statement-envelope + + # Registration Policy (label: TBD, temporary: 393): A map containing + # key/value pairs set by the Issuer which are sealed on Registration and + # non-opaque to the Transparency Service. The key/value pair semantics are + # specified by the Issuer or are specific to the CWT_Claims iss and + # CWT_Claims sub tuple. + # Examples: the sequence number of signed statements + # on a CWT_Claims Subject, Issuer metadata, or a reference to other + # Transparent Statements (e.g., augments, replaces, new-version, CPE-for) + # Reg_Info = { + reg_info = { + # ? "register_by": uint .within (~time), + "register_by": 1000, + # ? "sequence_no": uint, + "sequence_no": 0, + # ? "issuance_ts": uint .within (~time), + "issuance_ts": 1000, + # ? "no_replay": null, + "no_replay": None, + # * tstr => any + } + # } + + # Create COSE_Sign1 structure + # https://python-cwt.readthedocs.io/en/stable/algorithms.html + alg = "ES384" + # Create an ad-hoc key + # oct: size(int) + # RSA: public_exponent(int), size(int) + # EC: crv(str) (one of P-256, P-384, P-521, secp256k1) + # OKP: crv(str) (one of Ed25519, Ed448, X25519, X448) + key = jwcrypto.jwk.JWK() + if private_key_pem_path and private_key_pem_path.exists(): + key.import_from_pem(private_key_pem_path.read_bytes()) + else: + key = key.generate(kty="EC", crv="P-384") + kid = key.thumbprint() + key_as_pem_bytes = key.export_to_pem(private_key=True, password=None) + # cwt_cose_key = cwt.COSEKey.generate_symmetric_key(alg=alg, kid=kid) + cwt_cose_key = cwt.COSEKey.from_pem(key_as_pem_bytes, kid=kid) + # cwt_cose_key_to_cose_key = cwt.algs.ec2.EC2Key.to_cose_key(cwt_cose_key) + cwt_cose_key_to_cose_key = cwt_cose_key.to_dict() + sign1_message_key = pycose.keys.ec2.EC2Key.from_dict(cwt_cose_key_to_cose_key) + + # If issuer was not given used did:key of public key + if issuer is None: + multicodec_prefix_p_384 = "p384-pub" + multicodec.constants.NAME_TABLE[multicodec_prefix_p_384] = MULTICODEC_HEX_P384_PUBLIC_KEY + issuer = ( + DID_KEY_METHOD + + multibase.encode( + "base58btc", + multicodec.add_prefix( + multicodec_prefix_p_384, + load_pem_private_key(key_as_pem_bytes, password=None) + .public_key() + .public_bytes(Encoding.X962, PublicFormat.CompressedPoint), + ), + ).decode() + ) + + # CWT_Claims (label: 14 pending [CWT_CLAIM_COSE]): A CWT representing + # the Issuer (iss) making the statement, and the Subject (sub) to + # correlate a collection of statements about an Artifact. Additional + # [CWT_CLAIMS] MAY be used, while iss and sub MUST be provided + # CWT_Claims = { + cwt_claims = { + # iss (CWT_Claim Key 1): The Identifier of the signer, as a string + # Example: did:web:example.com + # 1 => tstr; iss, the issuer making statements, + 1: issuer, + # sub (CWT_Claim Key 2): The Subject to which the Statement refers, + # chosen by the Issuer + # Example: github.com/opensbom-generator/spdx-sbom-generator/releases/tag/v0.0.13 + # 2 => tstr; sub, the subject of the statements, + 2: subject, + # * tstr => any + } + # } + cwt_token = cwt.encode(cwt_claims, cwt_cose_key) + + # Protected_Header = { + protected = { + # algorithm (label: 1): Asymmetric signature algorithm used by the + # Issuer of a Signed Statement, as an integer. + # Example: -35 is the registered algorithm identifier for ECDSA with + # SHA-384, see COSE Algorithms Registry [IANA.cose]. + # 1 => int ; algorithm identifier, + # https://www.iana.org/assignments/cose/cose.xhtml#algorithms + # pycose.headers.Algorithm: "ES256", + pycose.headers.Algorithm: getattr(cwt.enums.COSEAlgs, alg), + # Key ID (label: 4): Key ID, as a bytestring + # 4 => bstr ; Key ID, + pycose.headers.KID: kid.encode("ascii"), + # 14 => CWT_Claims ; CBOR Web Token Claims, + CWTClaims: cwt_token, + # 393 => Reg_Info ; Registration Policy info, + RegInfo: reg_info, + # 3 => tstr ; payload type + pycose.headers.ContentType: content_type, + } + # } + + # Unprotected_Header = { + unprotected = { + # ; TBD, Labels are temporary, + TBD: "TBD", + # ? 394 => [+ Receipt] + Receipt: None, + } + # } + + # https://github.com/TimothyClaeys/pycose/blob/e527e79b611f6cc6673bbb694056a7468c2eef75/pycose/messages/cosemessage.py#L84-L91 + msg = pycose.messages.Sign1Message( + phdr=protected, + uhdr=unprotected, + payload=payload.encode("utf-8"), + ) + + # Sign + msg.key = sign1_message_key + # https://github.com/TimothyClaeys/pycose/blob/e527e79b611f6cc6673bbb694056a7468c2eef75/pycose/messages/cosemessage.py#L143 + claim = msg.encode(tag=True) + claim_path.write_bytes(claim) + + # Write out private key in PEM format if argument given and not exists + if private_key_pem_path and not private_key_pem_path.exists(): + private_key_pem_path.write_bytes(key_as_pem_bytes) + + +def cli(fn): + p = fn("create-claim", description="Create a fake SCITT claim") + p.add_argument("--out", required=True, type=pathlib.Path) + p.add_argument("--issuer", required=False, type=str, default=None) + p.add_argument("--subject", required=True, type=str) + p.add_argument("--content-type", required=True, type=str) + p.add_argument("--payload", required=True, type=str) + p.add_argument("--private-key-pem", required=False, type=pathlib.Path) + p.set_defaults( + func=lambda args: create_claim( + args.out, + args.issuer, + args.subject, + args.content_type, + args.payload, + private_key_pem_path=args.private_key_pem, + ) + ) + + return p + + +def main(argv=None): + parser = cli(argparse.ArgumentParser) + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scitt_emulator/did_helpers.py b/scitt_emulator/did_helpers.py new file mode 100644 index 00000000..07a4e927 --- /dev/null +++ b/scitt_emulator/did_helpers.py @@ -0,0 +1,193 @@ +import os +import sys +import inspect +import urllib.parse +from typing import Optional, Callable, Dict, Tuple, Union + +import multibase +import multicodec +import cryptography.hazmat.primitives.asymmetric.ec + + +def did_web_to_url( + did_web_string: str, + *, + scheme: Optional[str] = None, +): + if scheme is None: + scheme = os.environ.get("DID_WEB_ASSUME_SCHEME", "https") + return "/".join( + [ + f"{scheme}:/", + *[urllib.parse.unquote(i) for i in did_web_string.split(":")[2:]], + ] + ) + + +class DIDKeyInvalidPublicKeyLengthError(ValueError): + """ + If the byte length of rawPublicKeyBytes does not match the expected public + key length for the associated multicodecValue, an invalidPublicKeyLength + error MUST be raised. + """ + + +class DIDKeyDecoderNotFoundError(NotImplementedError): + """ + Raised when we don't have a function implemented to decode the given key + """ + + +class DIDKeyDecoderError(Exception): + """ + Raised when we failed to decode a key from a did:key DID method + """ + + +class DIDKeyInvalidPublicKeyError(DIDKeyDecoderError): + """ + Raised when the raw bytes of a key are invalid during decode + """ + + +DID_KEY_METHOD = "did:key:" + + +def did_key_decode_public_key(multibase_value: str) -> Tuple[bytes, bytes]: + # 3.1.2.3 + # Decode multibaseValue using the base58-btc multibase alphabet and set + # multicodecValue to the multicodec header for the decoded value. + multibase_value_decoded = multibase.decode(multibase_value) + # Implementers are cautioned to ensure that the multicodecValue is set to + # the result after performing varint decoding. + multicodec_value = multicodec.extract_prefix(multibase_value_decoded) + # Set the rawPublicKeyBytes to the bytes remaining after the multicodec + # header. + raw_public_key_bytes = multicodec.remove_prefix(multibase_value_decoded) + # Return multicodecValue and rawPublicKeyBytes as the decodedPublicKey. + return multicodec_value, raw_public_key_bytes + + +class _MULTICODEC_VALUE_NOT_FOUND_IN_TABLE: + pass + + +MULTICODEC_VALUE_NOT_FOUND_IN_TABLE = _MULTICODEC_VALUE_NOT_FOUND_IN_TABLE() + +# Multicodec hexadecimal value, public key, byte length, Description +MULTICODEC_HEX_SECP256K1_PUBLIC_KEY = 0xE7 +MULTICODEC_HEX_X25519_PUBLIC_KEY = 0xEC +MULTICODEC_HEX_ED25519_PUBLIC_KEY = 0xED +MULTICODEC_HEX_P256_PUBLIC_KEY = 0x1200 +MULTICODEC_HEX_P384_PUBLIC_KEY = 0x1201 +MULTICODEC_HEX_P521_PUBLIC_KEY = 0x1202 +MULTICODEC_HEX_RSA_PUBLIC_KEY = 0x1205 + +MULTICODEC_VALUE_TABLE = { + MULTICODEC_HEX_SECP256K1_PUBLIC_KEY: 33, # secp256k1-pub - Secp256k1 public key (compressed) + MULTICODEC_HEX_X25519_PUBLIC_KEY: 32, # x25519-pub - Curve25519 public key + MULTICODEC_HEX_ED25519_PUBLIC_KEY: 32, # ed25519-pub - Ed25519 public key + MULTICODEC_HEX_P256_PUBLIC_KEY: 33, # p256-pub - P-256 public key (compressed) + MULTICODEC_HEX_P384_PUBLIC_KEY: 49, # p384-pub - P-384 public key (compressed) + MULTICODEC_HEX_P521_PUBLIC_KEY: None, # p521-pub - P-521 public key (compressed) + MULTICODEC_HEX_RSA_PUBLIC_KEY: None, # rsa-pub - RSA public key. DER-encoded ASN.1 type RSAPublicKey according to IETF RFC 8017 (PKCS #1) +} + + +def did_key_signature_method_creation( + multibase_value: hex, + raw_public_key_bytes: bytes, +) -> Union[cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey]: + # 3.1.2 https://w3c-ccg.github.io/did-method-key/#signature-method-creation-algorithm + # Initialize verificationMethod to an empty object. + verification_method = {} + + # Set multicodecValue and rawPublicKeyBytes to the result of passing + # multibaseValue and options to ยง 3.1.3 Decode Public Key Algorithm. + # Ensure the proper key length of rawPublicKeyBytes based on the + # multicodecValue table + public_key_length_MUST_be = MULTICODEC_VALUE_TABLE.get( + multibase_value, MULTICODEC_VALUE_NOT_FOUND_IN_TABLE + ) + if public_key_length_MUST_be is MULTICODEC_VALUE_NOT_FOUND_IN_TABLE: + raise DIDKeyDecoderNotFoundError( + f"multibase_value {multibase_value!r} not in MULTICODEC_VALUE_NOT_FOUND_IN_TABLE {MULTICODEC_VALUE_NOT_FOUND_IN_TABLE!r}" + ) + + # If the byte length of rawPublicKeyBytes does not match the expected public + # key length for the associated multicodecValue, an invalidPublicKeyLength + # error MUST be raised. + if public_key_length_MUST_be is not None and public_key_length_MUST_be != len( + raw_public_key_bytes + ): + raise DIDKeyInvalidPublicKeyLengthError( + f"public_key_length_MUST_be: {public_key_length_MUST_be } != len(raw_public_key_bytes): {len(raw_public_key_bytes)}" + ) + + # Ensure the rawPublicKeyBytes are a proper encoding of the public key type + # as specified by the multicodecValue. This validation is often done by a + # cryptographic library when importing the public key by, for example, + # ensuring that an Elliptic Curve public key is a specific coordinate that + # exists on the elliptic curve. If an invalid public key value is detected, + # an invalidPublicKey error MUST be raised. + # + # SPEC ISSUE: Request for feedback on implementability: It is not clear if + # this particular check is implementable across all public key types. The + # group is accepting feedback on the implementability of this particular + # feature. + try: + if multibase_value in ( + MULTICODEC_HEX_P256_PUBLIC_KEY, + MULTICODEC_HEX_P384_PUBLIC_KEY, + MULTICODEC_HEX_P521_PUBLIC_KEY, + ): + public_key = cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey.from_encoded_point( + cryptography.hazmat.primitives.asymmetric.ec.SECP384R1(), + raw_public_key_bytes, + ) + else: + raise DIDKeyDecoderNotFoundError( + f"No importer for multibase_value {multibase_value!r}" + ) + except Exception as e: + raise DIDKeyInvalidPublicKeyError( + f"invalid raw_public_key_bytes: {raw_public_key_bytes!r}" + ) from e + + return public_key + + +def did_key_to_cryptography_key( + did_key: str, +) -> Union[cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey]: + """ + References + + - https://w3c-ccg.github.io/did-method-key/#p-384 + - RFC7515: JSON Web Key (JWK): https://www.rfc-editor.org/rfc/rfc7517 + - RFC8037: CFRG Elliptic Curve Diffie-Hellman (ECDH) and Signatures in JSON Object Signing and Encryption (JOSE): https://www.rfc-editor.org/rfc/rfc8037 + + Examples + + - P-384: https://github.com/w3c-ccg/did-method-key/blob/f5abee840c31e92cd1ac11737e0b62103ab99d21/test-vectors/nist-curves.json#L112-L166 + + >>> did_key_to_cryptography_key("did:key:invalid") + Traceback (most recent call last): + DIDKeyDecoderNotFoundError: ... + >>> public_key = did_key_to_cryptography_key("did:key:z82Lm1MpAkeJcix9K8TMiLd5NMAhnwkjjCBeWHXyu3U4oT2MVJJKXkcVBgjGhnLBn2Kaau9") + >>> public_key.__class__ + + """ + try: + multibase_value, raw_public_key_bytes = did_key_decode_public_key( + did_key.replace(DID_KEY_METHOD, "", 1) + ) + except Exception as e: + raise DIDKeyDecoderNotFoundError(did_key) from e + + try: + return did_key_signature_method_creation(multibase_value, raw_public_key_bytes) + except Exception as e: + raise DIDKeyDecoderError(did_key) from e + + raise DIDKeyDecoderNotFoundError(did_key) diff --git a/scitt_emulator/federation.py b/scitt_emulator/federation.py new file mode 100644 index 00000000..0679630e --- /dev/null +++ b/scitt_emulator/federation.py @@ -0,0 +1,19 @@ +import json +import dataclasses +from pathlib import Path +from abc import ABC, abstractmethod +from typing import Optional + +from scitt_emulator.signals import SCITTSignals, SCITTSignalsFederationCreatedEntry + + +class SCITTFederation(ABC): + def __init__(self, app, config_path: Path): + self.app = app + self.asgi_app = app.asgi_app + self.config = {} + if config_path and config_path.exists(): + self.config = json.loads(config_path.read_text()) + + async def __call__(self, scope, receive, send): + return await self.asgi_app(scope, receive, send) diff --git a/scitt_emulator/federation_activitypub_bovine.py b/scitt_emulator/federation_activitypub_bovine.py new file mode 100644 index 00000000..823e4dae --- /dev/null +++ b/scitt_emulator/federation_activitypub_bovine.py @@ -0,0 +1,486 @@ +import os +import sys +import json +import enum +import types +import pprint +import atexit +import base64 +import socket +import inspect +import tomllib +import logging +import asyncio +import pathlib +import tempfile +import functools +import traceback +import contextlib +import subprocess +import dataclasses +import urllib.parse +from pathlib import Path +from typing import Optional + +import tomli_w +import bovine +import aiohttp +from bovine_store import BovineAdminStore +from bovine_herd import BovineHerd +from bovine_pubsub import BovinePubSub +from bovine.clients import lookup_uri_with_webfinger +from bovine.crypto import generate_ed25519_private_key, private_key_to_did_key +from mechanical_bull.handlers import ( + load_handlers, +) + +from scitt_emulator.scitt import SCITTServiceEmulator +from scitt_emulator.federation import SCITTFederation +from scitt_emulator.tree_algs import TREE_ALGS +from scitt_emulator.signals import SCITTSignals, SCITTSignalsFederationCreatedEntry + +logger = logging.getLogger(__name__) + + +class SCITTFederationActivityPubBovine(SCITTFederation): + def __init__(self, app, config_path): + super().__init__(app, config_path) + + self.handle_name = self.config["handle_name"] + self.fqdn = self.config.get("fqdn", None) + # This is the federation middleware workspace, not the same as the + # tree_alg class's workspace + self.workspace = Path(self.config["workspace"]).expanduser() + + self.bovine_db_url = self.config.get( + "bovine_db_url", os.environ.get("BOVINE_DB_URL", None) + ) + if self.bovine_db_url and self.bovine_db_url.startswith("~"): + self.bovine_db_url = "sqlite://" + str(Path(self.bovine_db_url).expanduser()) + # TODO Pass this as variable + if not "BOVINE_DB_URL" in os.environ and self.bovine_db_url: + os.environ["BOVINE_DB_URL"] = self.bovine_db_url + logging.debug(f"Set BOVINE_DB_URL to {self.bovine_db_url}") + + BovinePubSub(app) + BovineHerd(app, db_url=self.bovine_db_url) + + app.while_serving(self.initialize_service) + + async def make_client_session(self): + return aiohttp.ClientSession(trust_env=True) + + async def initialize_service(self): + # TODO Better domain / fqdn building + if self.fqdn: + self.domain = self.fqdn + # TODO netloc remove scheme (http, https) before set to domain + # Use schem to build endpoint_path + else: + self.domain = f'http://localhost:{self.app.config["port"]}' + + config_toml_path = pathlib.Path(self.workspace, "config.toml") + if not config_toml_path.exists(): + logger.info("Actor client config does not exist, creating...") + config_toml_obj = { + self.handle_name: { + "secret": generate_ed25519_private_key(), + "host": self.domain, + "domain": self.domain, + "handlers": { + "mechanical_bull.actions.accept_follow_request": True, + }, + }, + } + config_toml_path.write_text(tomli_w.dumps(config_toml_obj)) + logger.info("Actor client config.toml created") + else: + config_toml_obj = tomllib.loads(config_toml_path.read_text()) + + # Enable handler() function in this file for this actor + config_toml_obj[self.handle_name]["handlers"][ + inspect.getmodule(sys.modules[__name__]).__spec__.name + ] = { + "signals": self.app.signals, + "following": self.config.get("following", {}), + } + + # Extract public key from private key in config file + private_key = config_toml_obj[self.handle_name]["secret"] + did_key = bovine.crypto.private_key_to_did_key(private_key) + + bovine_store = self.app.config["bovine_store"] + _account, actor_url = await bovine_store.get_account_url_for_identity(did_key) + if actor_url: + logger.info("Existing actor found. actor_url is %s", actor_url) + else: + logger.info("Actor not found, creating in database...") + bovine_store = BovineAdminStore(domain=self.domain) + bovine_name = await bovine_store.register( + self.handle_name, + ) + logger.info("Created actor with database name %s", bovine_name) + await bovine_store.add_identity_string_to_actor( + bovine_name, + "key0", + did_key, + ) + _account, actor_url = await self.app.config[ + "bovine_store" + ].get_account_url_for_identity(did_key) + logger.info("Actor key added in database. actor_url is %s", actor_url) + + # async with aiohttp.ClientSession(trust_env=True) as client_session: + async with contextlib.AsyncExitStack() as async_exit_stack: + # await mechanical_bull_loop(config_toml_obj) + self.app.config["bovine_async_exit_stack"] = async_exit_stack + self.app.add_background_task( + mechanical_bull_loop, + config_toml_obj, + add_background_task=self.app.add_background_task, + ) + yield + + +# Begin ActivityPub Actor automation handler code +class HandlerAPIVersion(enum.Enum): + # unstable API version used for development between versions + unstable = enum.auto() + v0_2_5 = enum.auto() + + +class HandlerEvent(enum.Enum): + DATA = enum.auto() + OPENED = enum.auto() + CLOSED = enum.auto() + + +async def handle( + client: bovine.BovineClient, + data: dict, + # config.toml arguments + signals: SCITTSignals = None, + following: dict[str, dict] = None, + raise_on_follow_failure: bool = False, + # handler arguments + handler_event: HandlerEvent = None, + handler_api_version: HandlerAPIVersion = HandlerAPIVersion.unstable, +): + try: + logger.info(f"{__file__}:handle(handler_event={handler_event})") + print(f"{__file__}:handle(handler_event={handler_event})") + match handler_event: + case HandlerEvent.OPENED: + # Listen for events from SCITT + # TODO Do this without using a client, server side + async def federate_created_entries_pass_client( + sender: SCITTServiceEmulator, + created_entry: SCITTSignalsFederationCreatedEntry = None, + ): + nonlocal client + nonlocal signals + await federate_created_entries(client, sender, created_entry) + + client.federate_created_entries = types.MethodType( + signals.federation.created_entry.connect( + federate_created_entries_pass_client + ), + client, + ) + # Preform ActivityPub related init + if following: + try: + async with asyncio.TaskGroup() as tg: + for key, value in following.items(): + logger.info("Following... %r", value) + tg.create_task(init_follow(client, **value)) + except (ExceptionGroup, BaseExceptionGroup) as error: + if raise_on_follow_failure: + raise + else: + logger.error("Failures while following: %r", error) + case HandlerEvent.CLOSED: + return + case HandlerEvent.DATA: + print( + f"Got new data in ActivityPub inbox: {pprint.pformat(data)}" + ) + logger.info( + "Got new data in ActivityPub inbox: %s", pprint.pformat(data) + ) + if data.get("type") != "Create": + return + + obj = data.get("object") + if not isinstance(obj, dict): + return + + # Send federated claim / receipt to SCITT + content_str = obj.get("content") + content = json.loads(content_str) + if not isinstance(content, dict): + return + logger.info("Federation received new receipt: %r", content) + + treeAlgorithm = content["treeAlgorithm"] + _entry_id = content["entry_id"] + claim = base64.b64decode(content["claim"].encode()) + receipt = base64.b64decode(content["receipt"].encode()) + service_parameters = base64.b64decode( + content["service_parameters"].encode() + ) + + with tempfile.TemporaryDirectory() as tempdir: + receipt_path = Path(tempdir, "receipt") + receipt_path.write_bytes(receipt) + cose_path = Path(tempdir, "claim") + cose_path.write_bytes(claim) + service_parameters_path = Path(tempdir, "service_parameters") + service_parameters_path.write_bytes(service_parameters) + + clazz = TREE_ALGS[treeAlgorithm] + service = clazz( + signals=SCITTSignals(), + service_parameters_path=service_parameters_path, + ) + service.verify_receipt(cose_path, receipt_path) + + logger.info("Receipt verified") + + # Send signal to submit federated claim + # TODO Announce that this entry ID was created via + # federation to avoid an infinate loop + await signals.federation.submit_claim.send_async( + client, claim=claim + ) + except Exception as ex: + print(ex) + import traceback + traceback.print_exc() + logger.error(ex) + logger.exception(ex) + logger.error(json.dumps(data)) + + +class WebFingerLookupNotFoundError(Exception): + pass + + +async def _init_follow(client, actor_id: str, domain: str = None, retry: int = 5): + url, _ = await lookup_uri_with_webfinger( + client.session, f"acct:{actor_id}", domain=domain + ) + if not url: + raise WebFingerLookupNotFoundError(f"actor_id: {actor_id}, domain: {domain}") + activity = client.activity_factory.follow( + url, + ).build() + logger.info("Sending follow to %s: %r", actor_id, activity) + await client.send_to_outbox(activity) + + +async def init_follow(client, retry: int = 5, **kwargs): + for i in range(0, retry): + try: + return await _init_follow(client, retry=retry, **kwargs) + except Exception as error: + logger.error(repr(error)) + await asyncio.sleep(2**i) + + +async def federate_created_entries( + client: bovine.BovineClient, + sender: SCITTServiceEmulator, + created_entry: SCITTSignalsFederationCreatedEntry = None, +): + print() + print() + print() + print(client, sender, created_entry) + print() + print() + print() + try: + logger.info("federate_created_entry() created_entry: %r", created_entry) + note = ( + client.object_factory.note( + content=json.dumps( + { + "treeAlgorithm": created_entry.tree_alg, + "service_parameters": base64.b64encode( + created_entry.public_service_parameters + ).decode(), + "entry_id": created_entry.entry_id, + "receipt": base64.b64encode(created_entry.receipt).decode(), + "claim": base64.b64encode(created_entry.claim).decode(), + } + ) + ) + .as_public() + .build() + ) + activity = client.activity_factory.create(note).build() + logger.info("Sending... %r", activity) + await client.send_to_outbox(activity) + + # DEBUG NOTE Dumping outbox + print("client:", client) + outbox = client.outbox() + print("outbox:", outbox) + count_messages = 0 + async for message in outbox: + count_messages += 1 + print(f"Message {count_messages} in outbox:", message) + print(f"End of messages in outbox, total: {count_messages}") + except: + logger.error(traceback.format_exc()) + + +async def call_handler_compat(handler, *args, **kwargs): + """Helper function to call a handler across versions of the handler calling + convention. + """ + # Inspect handler to determine accepted arguments and for logging purposes + handler_module = inspect.getmodule(handler) + handler_name = getattr( + handler, "__name__", getattr(handler, "__qualname__", repr(handler)) + ) + parameters = inspect.signature(handler).parameters + + inspect_args = { + name: parameter + for name, parameter in parameters.items() + if parameter.default is inspect.Parameter.empty + } + inspect_kwargs = { + name: parameter + for name, parameter in parameters.items() + if parameter.default is not inspect.Parameter.empty + } + + # Determine version of handler API in use. Assume lowest version if not + handler_api_version = HandlerAPIVersion.unstable + handler_api_version_parameter = inspect_kwargs.get("handler_api_version", None) + if "handler_api_version" in kwargs: + handler_api_version = kwargs["handler_api_version"] + elif "handler_event" not in inspect_kwargs: + handler_api_version = HandlerAPIVersion.v0_2_5 + elif ( + handler_api_version_parameter.annotation is HandlerAPIVersion + and handler_api_version_parameter.default is not inspect.Parameter.empty + ): + handler_api_version = handler_api_version_parameter.default + + # Pass version of handler API called with if not set explictly + if "handler_api_version" not in kwargs: + kwargs["handler_api_version"] = handler_api_version + + # Handle adaptations across versions + if ( + handler_api_version == HandlerAPIVersion.v0_2_5 + and args[list(inspect_args.keys()).index("data")] is None + ): + return + + # Remove unknown arguments which would break calls to older handlers + if len(args) != len(inspect_args): + logger.info( + "%s:%s does not support arguments: %s", + handler_module, + handler_name, + json.dumps(list(inspect_args.keys())[len(args) :]), + ) + + args = args[: len(inspect_args)] + + remove_kwargs = [keyword for keyword in kwargs if keyword not in inspect_kwargs] + + if remove_kwargs: + logger.info( + "%s:%s does not support keyword arguments: %s", + handler_module, + handler_name, + json.dumps(remove_kwargs), + ) + + for keyword in remove_kwargs: + del kwargs[keyword] + + # Call handler and return result + return await handler(*args, **kwargs) + + +async def handle_connection(client: bovine.BovineClient, handlers: list): + print("handle_connection") + event_source = await client.event_source() + print(event_source) + logger.info("Connected") + for handler in handlers: + await call_handler_compat( + handler, + client, + None, + handler_event=HandlerEvent.OPENED, + ) + async for event in event_source: + if not event: + return + if event and event.data: + data = json.loads(event.data) + + for handler in handlers: + await call_handler_compat( + handler, + client, + data, + handler_event=HandlerEvent.DATA, + ) + for handler in handlers: + await call_handler_compat( + handler, + client, + None, + handler_event=HandlerEvent.CLOSED, + ) + + +async def handle_connection_with_reconnect( + client: bovine.BovineClient, + handlers: list, + client_name: str = "BovineClient", + wait_time: int = 10, +): + while True: + await handle_connection(client, handlers) + logger.info( + "Disconnected from server for %s, reconnecting in %d seconds", + client_name, + wait_time, + ) + await asyncio.sleep(wait_time) + + +async def loop(client_name, client_config, handlers): + i = 1 + while True: + try: + async with bovine.BovineClient(**client_config) as client: + await handle_connection_with_reconnect( + client, handlers, client_name=client_name + ) + except Exception as e: + logger.exception("Something went wrong for %s", client_name) + logger.exception(e) + await asyncio.sleep(2**i) + i += 1 + + +# Run client handlers using call_compat +async def mechanical_bull_loop(config, *, add_background_task=asyncio.create_task): + try: + for client_name, client_config in config.items(): + if isinstance(client_config, dict): + handlers = load_handlers(client_config["handlers"]) + add_background_task(loop, client_name, client_config, handlers) + except Exception as e: + logger.exception(e) diff --git a/scitt_emulator/key_loader_format_did_key.py b/scitt_emulator/key_loader_format_did_key.py new file mode 100644 index 00000000..152965b5 --- /dev/null +++ b/scitt_emulator/key_loader_format_did_key.py @@ -0,0 +1,48 @@ +from typing import List, Tuple + +import cwt +import cwt.algs.ec2 +import pycose +import pycose.keys.ec2 +from cryptography.hazmat.primitives import serialization + +# TODO Remove this once we have a example flow for proper key verification +import jwcrypto.jwk + +from scitt_emulator.did_helpers import DID_KEY_METHOD, did_key_to_cryptography_key + + +def key_loader_format_did_key( + unverified_issuer: str, +) -> List[Tuple[cwt.COSEKey, pycose.keys.ec2.EC2Key]]: + jwk_keys = [] + cwt_cose_keys = [] + pycose_cose_keys = [] + cryptography_keys = [] + + if not unverified_issuer.startswith(DID_KEY_METHOD): + return pycose_cose_keys + + cryptography_keys.append(did_key_to_cryptography_key(unverified_issuer)) + + for cryptography_key in cryptography_keys: + jwk_keys.append( + jwcrypto.jwk.JWK.from_pem( + cryptography_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + ) + + for jwk_key in jwk_keys: + cwt_cose_key = cwt.COSEKey.from_pem( + jwk_key.export_to_pem(), + kid=jwk_key.thumbprint(), + ) + cwt_cose_keys.append(cwt_cose_key) + cwt_ec2_key_as_dict = cwt_cose_key.to_dict() + pycose_cose_key = pycose.keys.ec2.EC2Key.from_dict(cwt_ec2_key_as_dict) + pycose_cose_keys.append((cwt_cose_key, pycose_cose_key)) + + return pycose_cose_keys diff --git a/scitt_emulator/key_loader_format_url_referencing_oidc_issuer.py b/scitt_emulator/key_loader_format_url_referencing_oidc_issuer.py new file mode 100644 index 00000000..76a4547c --- /dev/null +++ b/scitt_emulator/key_loader_format_url_referencing_oidc_issuer.py @@ -0,0 +1,61 @@ +import json +import contextlib +import urllib.parse +import urllib.request +from typing import List, Tuple + +import cwt +import cwt.algs.ec2 +import pycose +import pycose.keys.ec2 + +# TODO Remove this once we have a example flow for proper key verification +import jwcrypto.jwk + +from scitt_emulator.did_helpers import did_web_to_url + + +def key_loader_format_url_referencing_oidc_issuer( + unverified_issuer: str, +) -> List[Tuple[cwt.COSEKey, pycose.keys.ec2.EC2Key]]: + jwk_keys = [] + cwt_cose_keys = [] + pycose_cose_keys = [] + + if unverified_issuer.startswith("did:web:"): + unverified_issuer = did_web_to_url(unverified_issuer) + + if "://" not in unverified_issuer or unverified_issuer.startswith("file://"): + return pycose_cose_keys + + # TODO Logging for URLErrors + # Check if OIDC issuer + unverified_issuer_parsed_url = urllib.parse.urlparse(unverified_issuer) + openid_configuration_url = unverified_issuer_parsed_url._replace( + path="/.well-known/openid-configuration", + ).geturl() + with contextlib.suppress(urllib.request.URLError): + with urllib.request.urlopen(openid_configuration_url) as response: + if response.status == 200: + openid_configuration = json.loads(response.read()) + jwks_uri = openid_configuration["jwks_uri"] + with urllib.request.urlopen(jwks_uri) as response: + if response.status == 200: + jwks = json.loads(response.read()) + for jwk_key_as_dict in jwks["keys"]: + jwk_key_as_string = json.dumps(jwk_key_as_dict) + jwk_keys.append( + jwcrypto.jwk.JWK.from_json(jwk_key_as_string), + ) + + for jwk_key in jwk_keys: + cwt_cose_key = cwt.COSEKey.from_pem( + jwk_key.export_to_pem(), + kid=jwk_key.thumbprint(), + ) + cwt_cose_keys.append(cwt_cose_key) + cwt_ec2_key_as_dict = cwt_cose_key.to_dict() + pycose_cose_key = pycose.keys.ec2.EC2Key.from_dict(cwt_ec2_key_as_dict) + pycose_cose_keys.append((cwt_cose_key, pycose_cose_key)) + + return pycose_cose_keys diff --git a/scitt_emulator/key_loader_format_url_referencing_ssh_authorized_keys.py b/scitt_emulator/key_loader_format_url_referencing_ssh_authorized_keys.py new file mode 100644 index 00000000..aea7e38b --- /dev/null +++ b/scitt_emulator/key_loader_format_url_referencing_ssh_authorized_keys.py @@ -0,0 +1,65 @@ +import contextlib +import urllib.parse +import urllib.request +from typing import List, Tuple + +import cwt +import cwt.algs.ec2 +import pycose +import pycose.keys.ec2 +import cryptography.exceptions +from cryptography.hazmat.primitives import serialization + +# TODO Remove this once we have a example flow for proper key verification +import jwcrypto.jwk + +from scitt_emulator.did_helpers import did_web_to_url + + +def key_loader_format_url_referencing_ssh_authorized_keys( + unverified_issuer: str, +) -> List[Tuple[cwt.COSEKey, pycose.keys.ec2.EC2Key]]: + jwk_keys = [] + cwt_cose_keys = [] + pycose_cose_keys = [] + + cryptography_ssh_keys = [] + + if unverified_issuer.startswith("did:web:"): + unverified_issuer = did_web_to_url(unverified_issuer) + + if "://" not in unverified_issuer or unverified_issuer.startswith("file://"): + return pycose_cose_keys + + # Try loading ssh keys. Example: https://github.com/username.keys + with contextlib.suppress(urllib.request.URLError): + with urllib.request.urlopen(unverified_issuer) as response: + while line := response.readline(): + with contextlib.suppress( + (ValueError, cryptography.exceptions.UnsupportedAlgorithm) + ): + cryptography_ssh_keys.append( + serialization.load_ssh_public_key(line) + ) + + for cryptography_ssh_key in cryptography_ssh_keys: + jwk_keys.append( + jwcrypto.jwk.JWK.from_pem( + cryptography_ssh_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + ) + + for jwk_key in jwk_keys: + cwt_cose_key = cwt.COSEKey.from_pem( + jwk_key.export_to_pem(), + kid=jwk_key.thumbprint(), + ) + cwt_cose_keys.append(cwt_cose_key) + cwt_ec2_key_as_dict = cwt_cose_key.to_dict() + pycose_cose_key = pycose.keys.ec2.EC2Key.from_dict(cwt_ec2_key_as_dict) + pycose_cose_keys.append((cwt_cose_key, pycose_cose_key)) + + return pycose_cose_keys diff --git a/scitt_emulator/key_loader_format_url_referencing_x509.py b/scitt_emulator/key_loader_format_url_referencing_x509.py new file mode 100644 index 00000000..2c92db79 --- /dev/null +++ b/scitt_emulator/key_loader_format_url_referencing_x509.py @@ -0,0 +1,65 @@ +import contextlib +import urllib.parse +import urllib.request +from typing import List, Tuple + +import cwt +import cwt.algs.ec2 +import pycose +import pycose.keys.ec2 +import cryptography.exceptions +from cryptography.hazmat.primitives import serialization + +# TODO Remove this once we have a example flow for proper key verification +import jwcrypto.jwk + +from scitt_emulator.did_helpers import did_web_to_url + + +def key_loader_format_url_referencing_x509( + unverified_issuer: str, +) -> List[Tuple[cwt.COSEKey, pycose.keys.ec2.EC2Key]]: + jwk_keys = [] + cwt_cose_keys = [] + pycose_cose_keys = [] + + cryptography_ssh_keys = [] + + if unverified_issuer.startswith("did:web:"): + unverified_issuer = did_web_to_url(unverified_issuer) + + if "://" not in unverified_issuer or unverified_issuer.startswith("file://"): + return pycose_cose_keys + + with contextlib.suppress(urllib.request.URLError): + with urllib.request.urlopen(unverified_issuer) as response: + contents = response.read() + with contextlib.suppress( + (ValueError, cryptography.exceptions.UnsupportedAlgorithm) + ): + for certificate in cryptography.x509.load_pem_x509_certificates( + contents + ): + cryptography_ssh_keys.append(certificate.public_key()) + + for cryptography_ssh_key in cryptography_ssh_keys: + jwk_keys.append( + jwcrypto.jwk.JWK.from_pem( + cryptography_ssh_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + ) + + for jwk_key in jwk_keys: + cwt_cose_key = cwt.COSEKey.from_pem( + jwk_key.export_to_pem(), + kid=jwk_key.thumbprint(), + ) + cwt_cose_keys.append(cwt_cose_key) + cwt_ec2_key_as_dict = cwt_cose_key.to_dict() + pycose_cose_key = pycose.keys.ec2.EC2Key.from_dict(cwt_ec2_key_as_dict) + pycose_cose_keys.append((cwt_cose_key, pycose_cose_key)) + + return pycose_cose_keys diff --git a/scitt_emulator/oidc.py b/scitt_emulator/oidc.py index 4ca770d2..7394978f 100644 --- a/scitt_emulator/oidc.py +++ b/scitt_emulator/oidc.py @@ -2,9 +2,7 @@ # Licensed under the MIT License. import jwt import json -import jwcrypto.jwk import jsonschema -from flask import jsonify from werkzeug.wrappers import Request from scitt_emulator.client import HttpClient @@ -12,6 +10,7 @@ class OIDCAuthMiddleware: def __init__(self, app, config_path): self.app = app + self.asgi_app = app.asgi_app self.config = {} if config_path and config_path.exists(): self.config = json.loads(config_path.read_text()) @@ -26,12 +25,13 @@ def __init__(self, app, config_path): ).json() self.jwks_clients[issuer] = jwt.PyJWKClient(self.oidc_configs[issuer]["jwks_uri"]) - def __call__(self, environ, start_response): - request = Request(environ) - claims = self.validate_token(request.headers["Authorization"].replace("Bearer ", "")) + async def __call__(self, scope, receive, send): + headers = dict(scope.get("headers", [])) + token = headers.get(b"authorization", "").replace(b"Bearer ", b"").decode() + claims = self.validate_token(token) if "claim_schema" in self.config and claims["iss"] in self.config["claim_schema"]: jsonschema.validate(claims, schema=self.config["claim_schema"][claims["iss"]]) - return self.app(environ, start_response) + return await self.asgi_app(scope, receive, send) def validate_token(self, token): validation_error = Exception(f"Failed to validate against all issuers: {self.jwks_clients.keys()!s}") diff --git a/scitt_emulator/rkvst.py b/scitt_emulator/rkvst.py index 54f9f5ee..6498d64b 100644 --- a/scitt_emulator/rkvst.py +++ b/scitt_emulator/rkvst.py @@ -5,22 +5,25 @@ from typing import Optional from pathlib import Path import json -import cbor2 -from pycose.messages import CoseMessage, Sign1Message +from pycose.messages import Sign1Message import pycose.headers import base64 from os import getenv from . import rkvst_mocks from scitt_emulator.scitt import SCITTServiceEmulator +from scitt_emulator.federation import SCITTFederation class RKVSTSCITTServiceEmulator(SCITTServiceEmulator): tree_alg = "RKVST" def __init__( - self, service_parameters_path: Path, storage_path: Optional[Path] = None + self, + service_parameters_path: Path, + storage_path: Optional[Path] = None, + federation: Optional[SCITTFederation] = None, ): - super().__init__(service_parameters_path, storage_path) + super().__init__(service_parameters_path, storage_path, federation) if storage_path is not None: self._service_private_key_path = ( self.storage_path / "service_private_key.pem" @@ -115,6 +118,9 @@ def _submit_claim_async(self, claim: bytes): #event = rkvst_mocks.mock_event_lro_incomplete operation_id = self._event_id_to_operation_id(event["identity"]) + # TODO Federate created entries when operations complete + # if self.federation: + # self.federation.created_entry(entry_id, receipt) return { "operationId": operation_id, "status": "running" diff --git a/scitt_emulator/scitt.py b/scitt_emulator/scitt.py index 3311b778..c87730e9 100644 --- a/scitt_emulator/scitt.py +++ b/scitt_emulator/scitt.py @@ -4,19 +4,20 @@ from typing import Optional from abc import ABC, abstractmethod from pathlib import Path -import contextlib import time import json import uuid +import hashlib import cbor2 -from pycose.messages import CoseMessage, Sign1Message +from pycose.messages import Sign1Message import pycose.headers -from pycose.keys.ec2 import EC2Key -import pycose.keys.curves -# temporary claim header labels, see draft-birkholz-scitt-architecture -COSE_Headers_Issuer = 391 +from scitt_emulator.create_statement import CWTClaims +from scitt_emulator.verify_statement import verify_statement +from scitt_emulator.federation import SCITTFederation +from scitt_emulator.signals import SCITTSignals, SCITTSignalsFederationCreatedEntry + # temporary receipt header labels, see draft-birkholz-scitt-receipts COSE_Headers_Service_Id = "service_id" @@ -27,6 +28,8 @@ MOST_PERMISSIVE_INSERT_POLICY = "*" DEFAULT_INSERT_POLICY = MOST_PERMISSIVE_INSERT_POLICY +# hash algorithm used to content address claims to entryIDs +DEFAULT_ENTRY_ID_HASH_ALGORITHM = "sha384" class ClaimInvalidError(Exception): pass @@ -46,8 +49,13 @@ class PolicyResultDecodeError(Exception): class SCITTServiceEmulator(ABC): def __init__( - self, service_parameters_path: Path, storage_path: Optional[Path] = None + self, + signals: SCITTSignals, + service_parameters_path: Path, + storage_path: Optional[Path] = None, ): + self.signals = signals + self.connect_signals() self.storage_path = storage_path self.service_parameters_path = service_parameters_path @@ -59,6 +67,17 @@ def __init__( with open(self.service_parameters_path) as f: self.service_parameters = json.load(f) + def connect_signals(self): + self.signal_receiver_submit_claim = self.signals.federation.submit_claim.connect( + self.signal_receiver_submit_claim, + ) + + async def signal_receiver_submit_claim(self, _sender, claim: bytes) -> None: + use_lro = self.service_parameters.get("use_lro", False) + result = await self.submit_claim(claim, long_running=use_lro) + while use_lro and result.get("status", None) == "running": + result = await self._finish_operation(result) + @abstractmethod def initialize_service(self): raise NotImplementedError @@ -71,7 +90,7 @@ def create_receipt_contents(self, countersign_tbi: bytes, entry_id: str): def verify_receipt_contents(receipt_contents: list, countersign_tbi: bytes): raise NotImplementedError - def get_operation(self, operation_id: str) -> dict: + async def get_operation(self, operation_id: str) -> dict: operation_path = self.operations_path / f"{operation_id}.json" try: with open(operation_path, "r") as f: @@ -82,7 +101,7 @@ def get_operation(self, operation_id: str) -> dict: if operation["status"] == "running": # Pretend that the service finishes the operation after # the client having checked the operation status once. - operation = self._finish_operation(operation) + operation = await self._finish_operation(operation) return operation def get_entry(self, entry_id: str) -> dict: @@ -102,9 +121,15 @@ def get_claim(self, entry_id: str) -> bytes: raise EntryNotFoundError(f"Entry {entry_id} not found") return claim - def submit_claim(self, claim: bytes, long_running=True) -> dict: + async def submit_claim(self, claim: bytes, long_running=True) -> dict: insert_policy = self.service_parameters.get("insertPolicy", DEFAULT_INSERT_POLICY) + try: + entry_id = self.get_entry_id(claim) + return self.get_entry(entry_id) + except EntryNotFoundError: + pass + if long_running: return self._create_operation(claim) elif insert_policy != MOST_PERMISSIVE_INSERT_POLICY: @@ -112,21 +137,26 @@ def submit_claim(self, claim: bytes, long_running=True) -> dict: f"non-* insertPolicy only works with long_running=True: {insert_policy!r}" ) else: - return self._create_entry(claim) + return await self._create_entry(claim) - def _create_entry(self, claim: bytes) -> dict: - last_entry_path = self.storage_path / "last_entry_id.txt" - if last_entry_path.exists(): - with open(last_entry_path, "r") as f: - last_entry_id = int(f.read()) - else: - last_entry_id = 0 + def public_service_parameters(self) -> bytes: + # TODO Only export public portion of cert + return json.dumps(self.service_parameters).encode() - entry_id = str(last_entry_id + 1) + def get_entry_id(self, claim: bytes) -> str: + entry_id_hash_alg = self.service_parameters.get( + "entryIdHashAlgorith", + DEFAULT_ENTRY_ID_HASH_ALGORITHM, + ) + entry_id_hash = hashlib.new(entry_id_hash_alg) + entry_id_hash.update(claim) + entry_id = f"{entry_id_hash_alg}:{entry_id_hash.hexdigest()}" + return entry_id - self._create_receipt(claim, entry_id) + async def _create_entry(self, claim: bytes) -> dict: + entry_id = self.get_entry_id(claim) - last_entry_path.write_text(entry_id) + receipt = self._create_receipt(claim, entry_id) claim_path = self.storage_path / f"{entry_id}.cose" claim_path.write_bytes(claim) @@ -134,6 +164,18 @@ def _create_entry(self, claim: bytes) -> dict: print(f"A COSE signed Claim was written to: {claim_path}") entry = {"entryId": entry_id} + + await self.signals.federation.created_entry.send_async( + self, + created_entry=SCITTSignalsFederationCreatedEntry( + tree_alg=self.tree_alg, + entry_id=entry_id, + receipt=receipt, + claim=claim, + public_service_parameters=self.public_service_parameters(), + ) + ) + return entry def _create_operation(self, claim: bytes): @@ -192,7 +234,7 @@ def _sync_policy_result(self, operation: dict): return policy_result - def _finish_operation(self, operation: dict): + async def _finish_operation(self, operation: dict): operation_id = operation["operationId"] operation_path = self.operations_path / f"{operation_id}.json" claim_src_path = self.operations_path / f"{operation_id}.cose" @@ -209,7 +251,7 @@ def _finish_operation(self, operation: dict): return operation claim = claim_src_path.read_bytes() - entry = self._create_entry(claim) + entry = await self._create_entry(claim) claim_src_path.unlink() operation["status"] = "succeeded" @@ -225,7 +267,7 @@ def _create_receipt(self, claim: bytes, entry_id: str): # Note: This emulator does not verify the claim signature and does not apply # registration policies. try: - msg = CoseMessage.decode(claim) + msg = Sign1Message.decode(claim, tag=True) except: raise ClaimInvalidError("Claim is not a valid COSE message") if not isinstance(msg, Sign1Message): @@ -236,10 +278,15 @@ def _create_receipt(self, claim: bytes, entry_id: str): raise ClaimInvalidError( "Claim does not have a content type header parameter" ) - if COSE_Headers_Issuer not in msg.phdr: - raise ClaimInvalidError("Claim does not have an issuer header parameter") - if not isinstance(msg.phdr[COSE_Headers_Issuer], str): - raise ClaimInvalidError("Claim issuer is not a string") + if CWTClaims not in msg.phdr: + raise ClaimInvalidError("Claim does not have a CWTClaims header parameter") + + try: + cwt_cose_key, _pycose_cose_key = verify_statement(msg) + except Exception as e: + raise ClaimInvalidError("Failed to verify signature on statement") from e + if not cwt_cose_key: + raise ClaimInvalidError("Failed to verify signature on statement") # Extract fields of COSE_Sign1 for countersigning outer = cbor2.loads(claim) @@ -270,6 +317,7 @@ def _create_receipt(self, claim: bytes, entry_id: str): with open(receipt_path, "wb") as f: f.write(receipt) print(f"Receipt written to {receipt_path}") + return receipt def get_receipt(self, entry_id: str): receipt_path = self.storage_path / f"{entry_id}.receipt.cbor" @@ -304,28 +352,6 @@ def verify_receipt(self, cose_path: Path, receipt_path: Path): self.verify_receipt_contents(receipt_contents, countersign_tbi) -def create_claim(claim_path: Path, issuer: str, content_type: str, payload: str): - # Create COSE_Sign1 structure - protected = { - pycose.headers.Algorithm: "ES256", - pycose.headers.ContentType: content_type, - COSE_Headers_Issuer: issuer, - } - msg = Sign1Message(phdr=protected, payload=payload.encode("utf-8")) - - # Create an ad-hoc key - # Note: The emulator does not validate signatures, hence the short-cut. - key = EC2Key.generate_key(pycose.keys.curves.P256) - - # Sign - msg.key = key - claim = msg.encode(tag=True) - - with open(claim_path, "wb") as f: - f.write(claim) - print(f"A COSE signed Claim was written to: {claim_path}") - - def create_countersign_to_be_included( body_protected, sign_protected, payload, signature ): diff --git a/scitt_emulator/server.py b/scitt_emulator/server.py index 1ab4e60b..36181238 100644 --- a/scitt_emulator/server.py +++ b/scitt_emulator/server.py @@ -5,16 +5,20 @@ from pathlib import Path from io import BytesIO import random +import logging -from flask import Flask, request, send_file, make_response +from quart import Quart, request, send_file, make_response +from blinker import Namespace from scitt_emulator.tree_algs import TREE_ALGS from scitt_emulator.plugin_helpers import entrypoint_style_load from scitt_emulator.scitt import EntryNotFoundError, ClaimInvalidError, OperationNotFoundError +from scitt_emulator.signals import SCITTSignals +from scitt_emulator.signals import SCITTSignalsFederationCreatedEntry -def make_error(code: str, msg: str, status_code: int): - return make_response( +async def make_error(code: str, msg: str, status_code: int): + return await make_response( { "type": f"urn:ietf:params:scitt:error:{code}", "detail": msg, @@ -23,19 +27,24 @@ def make_error(code: str, msg: str, status_code: int): ) -def make_unavailable_error(): - return make_error("serviceUnavailable", "Service unavailable, try again later", 503) +async def make_unavailable_error(): + return await make_error("serviceUnavailable", "Service unavailable, try again later", 503) def create_flask_app(config): - app = Flask(__name__) + app = Quart(__name__) # See http://flask.pocoo.org/docs/latest/config/ app.config.update(dict(DEBUG=True)) app.config.update(config) - if app.config.get("middleware", None): - app.wsgi_app = app.config["middleware"](app.wsgi_app, app.config.get("middleware_config_path", None)) + # See https://blinker.readthedocs.io/en/stable/#blinker.base.Signal.send + app.signals = SCITTSignals( + add_background_task=app.add_background_task, + ) + + for middleware, middleware_config_path in zip(app.config.get("middleware", []), app.config.get("middleware_config_path", [])): + app.asgi_app = middleware(app, middleware_config_path) error_rate = app.config["error_rate"] use_lro = app.config["use_lro"] @@ -48,7 +57,9 @@ def create_flask_app(config): clazz = TREE_ALGS[app.config["tree_alg"]] app.scitt_service = clazz( - storage_path=storage_path, service_parameters_path=app.service_parameters_path + signals=app.signals, + storage_path=storage_path, + service_parameters_path=app.service_parameters_path, ) app.scitt_service.initialize_service() print(f"Service parameters: {app.service_parameters_path}") @@ -56,60 +67,64 @@ def create_flask_app(config): def is_unavailable(): return random.random() <= error_rate + @app.route("/test", methods=["GET"]) + async def get_test(): + return await make_response({"OK": True}, 200, {}) + @app.route("/entries//receipt", methods=["GET"]) - def get_receipt(entry_id: str): + async def get_receipt(entry_id: str): if is_unavailable(): - return make_unavailable_error() + return await make_unavailable_error() try: receipt = app.scitt_service.get_receipt(entry_id) except EntryNotFoundError as e: - return make_error("entryNotFound", str(e), 404) - return send_file(BytesIO(receipt), download_name=f"{entry_id}.receipt.cbor") + return await make_error("entryNotFound", str(e), 404) + return await send_file(BytesIO(receipt), attachment_filename=f"{entry_id}.receipt.cbor") @app.route("/entries/", methods=["GET"]) - def get_claim(entry_id: str): + async def get_claim(entry_id: str): if is_unavailable(): - return make_unavailable_error() + return await make_unavailable_error() try: claim = app.scitt_service.get_claim(entry_id) except EntryNotFoundError as e: - return make_error("entryNotFound", str(e), 404) - return send_file(BytesIO(claim), download_name=f"{entry_id}.cose") + return await make_error("entryNotFound", str(e), 404) + return await send_file(BytesIO(claim), attachment_filename=f"{entry_id}.cose") @app.route("/entries", methods=["POST"]) - def submit_claim(): + async def submit_claim(): if is_unavailable(): - return make_unavailable_error() + return await make_unavailable_error() try: - if use_lro: - result = app.scitt_service.submit_claim(request.get_data(), long_running=True) + # NOTE This got refactored to support content addressable claims + result = await app.scitt_service.submit_claim(await request.get_data(), long_running=use_lro) + if "operationId" in result: headers = { "Location": f"{request.host_url}/operations/{result['operationId']}", "Retry-After": "1" } status_code = 202 else: - result = app.scitt_service.submit_claim(request.get_data(), long_running=False) headers = { "Location": f"{request.host_url}/entries/{result['entryId']}", } status_code = 201 except ClaimInvalidError as e: - return make_error("invalidInput", str(e), 400) - return make_response(result, status_code, headers) + return await make_error("invalidInput", str(e), 400) + return await make_response(result, status_code, headers) @app.route("/operations/", methods=["GET"]) - def get_operation(operation_id: str): + async def get_operation(operation_id: str): if is_unavailable(): - return make_unavailable_error() + return await make_unavailable_error() try: - operation = app.scitt_service.get_operation(operation_id) + operation = await app.scitt_service.get_operation(operation_id) except OperationNotFoundError as e: - return make_error("operationNotFound", str(e), 404) + return await make_error("operationNotFound", str(e), 404) headers = {} if operation["status"] == "running": headers["Retry-After"] = "1" - return make_response(operation, 200, headers) + return await make_response(operation, 200, headers) return app @@ -117,6 +132,7 @@ def get_operation(operation_id: str): def cli(fn): parser = fn() parser.add_argument("-p", "--port", type=int, default=8000) + parser.add_argument("--log", type=str, default="INFO") parser.add_argument("--error-rate", type=float, default=0.01) parser.add_argument("--use-lro", action="store_true", help="Create operations for submissions") parser.add_argument("--tree-alg", required=True, choices=list(TREE_ALGS.keys())) @@ -124,13 +140,16 @@ def cli(fn): parser.add_argument( "--middleware", type=lambda value: list(entrypoint_style_load(value))[0], - default=None, + nargs="*", + default=[], ) - parser.add_argument("--middleware-config-path", type=Path, default=None) + parser.add_argument("--middleware-config-path", type=Path, nargs="*", default=[]) def cmd(args): + logging.basicConfig(level=getattr(logging, args.log.upper(), "INFO")) app = create_flask_app( { + "port": args.port, "middleware": args.middleware, "middleware_config_path": args.middleware_config_path, "tree_alg": args.tree_alg, diff --git a/scitt_emulator/signals.py b/scitt_emulator/signals.py new file mode 100644 index 00000000..24b3d717 --- /dev/null +++ b/scitt_emulator/signals.py @@ -0,0 +1,31 @@ +import asyncio +from dataclasses import dataclass, field +from typing import Callable + +import blinker + + +@dataclass +class SCITTSignalsFederationCreatedEntry: + tree_alg: str + entry_id: str + receipt: bytes + claim: bytes + public_service_parameters: bytes + + +@dataclass +class SCITTSignalsFederation: + _signal_namespace: blinker.Namespace = field(default_factory=blinker.Namespace) + created_entry: blinker.Signal = field(init=False) + submit_claim: blinker.Signal = field(init=False) + + def __post_init__(self): + self.created_entry = self._signal_namespace.signal("create_entry") + self.submit_claim = self._signal_namespace.signal("submit_claim") + + +@dataclass +class SCITTSignals: + add_background_task: Callable = field(default=asyncio.create_task) + federation: SCITTSignalsFederation = field(default_factory=SCITTSignalsFederation) diff --git a/scitt_emulator/verify_statement.py b/scitt_emulator/verify_statement.py new file mode 100644 index 00000000..f913720e --- /dev/null +++ b/scitt_emulator/verify_statement.py @@ -0,0 +1,66 @@ +import os +import itertools +import contextlib +import urllib.parse +import urllib.request +import importlib.metadata +from typing import Optional, Callable, List, Tuple + +import cwt +import cwt.algs.ec2 +import pycose +import pycose.keys.ec2 +from pycose.messages import Sign1Message + +from scitt_emulator.did_helpers import did_web_to_url +from scitt_emulator.create_statement import CWTClaims + + +ENTRYPOINT_KEY_LOADERS = "scitt_emulator.verify_signature.key_loaders" + + +def verify_statement( + msg: Sign1Message, + *, + key_loaders: Optional[ + List[Callable[[str], List[Tuple[cwt.COSEKey, pycose.keys.ec2.EC2Key]]]] + ] = None, +) -> bool: + """ + Resolve keys for statement issuer and verify signature on COSESign1 + statement and embedded CWT + """ + if key_loaders is None: + key_loaders = [] + # There is some difference in the return value of entry_points across + # Python versions/envs (conda vs. non-conda). Python 3.8 returns a dict. + entrypoints = importlib.metadata.entry_points() + if isinstance(entrypoints, dict): + for entrypoint in entrypoints.get(ENTRYPOINT_KEY_LOADERS, []): + key_loaders.append(entrypoint.load()) + elif isinstance(entrypoints, getattr(importlib.metadata, "EntryPoints", list)): + for entrypoint in entrypoints: + if entrypoint.group == ENTRYPOINT_KEY_LOADERS: + key_loaders.append(entrypoint.load()) + else: + raise TypeError(f"importlib.metadata.entry_points returned unknown type: {type(entrypoints)}: {entrypoints!r}") + + # Figure out what the issuer is + cwt_cose_loads = cwt.cose.COSE()._loads + cwt_unverified_protected = cwt_cose_loads( + cwt_cose_loads(msg.phdr[CWTClaims]).value[2] + ) + unverified_issuer = cwt_unverified_protected[1] + + # Load keys from issuer and attempt verification. Return keys used to verify + # as tuple of cwt.COSEKey and pycose.keys formats + for cwt_cose_key, pycose_cose_key in itertools.chain( + *[key_loader(unverified_issuer) for key_loader in key_loaders] + ): + msg.key = pycose_cose_key + with contextlib.suppress(Exception): + verify_signature = msg.verify_signature() + if verify_signature: + return cwt_cose_key, pycose_cose_key + + return None, None diff --git a/setup.py b/setup.py index 466dd6fc..000ae83a 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,26 @@ entry_points = { 'console_scripts': [ 'scitt-emulator=scitt_emulator.cli:main' - ] + ], + 'scitt_emulator.verify_signature.key_loaders': [ + 'did_key=scitt_emulator.key_loader_format_did_key:key_loader_format_did_key', + 'url_referencing_oidc_issuer=scitt_emulator.key_loader_format_url_referencing_oidc_issuer:key_loader_format_url_referencing_oidc_issuer', + 'url_referencing_ssh_authorized_keys=scitt_emulator.key_loader_format_url_referencing_ssh_authorized_keys:key_loader_format_url_referencing_ssh_authorized_keys', + 'url_referencing_x509=scitt_emulator.key_loader_format_url_referencing_x509:key_loader_format_url_referencing_x509', + ], }, python_requires=">=3.8", install_requires=[ "cryptography", "cbor2", + "cwt", + "py-multicodec", + "py-multibase", + "jwcrypto", "pycose", "httpx", "flask", + "quart", "rkvst-archivist" ], extras_require={ @@ -26,6 +37,15 @@ "PyJWT", "jwcrypto", "jsonschema", - ] + ], + "federation-activitypub-bovine": [ + "tomli-w", + "aiohttp", + "bovine", + "bovine-store", + "bovine-herd", + "bovine-pubsub", + "mechanical-bull", + ], }, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 95319901..72872294 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,50 +1,193 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. import os +import io import json +import types +import socket +import pathlib +import asyncio +import aiohttp +import functools import threading +import traceback +import contextlib +import unittest.mock +import multiprocessing import pytest import jwt import jwcrypto -from flask import Flask, jsonify -from werkzeug.serving import make_server + +from quart import Quart, jsonify, send_file +import hypercorn.config + +import bovine.utils from scitt_emulator import cli, server from scitt_emulator.oidc import OIDCAuthMiddleware -issuer = "did:web:example.com" +import logging + +logger = logging.getLogger(__name__) + content_type = "application/json" payload = '{"foo": "bar"}' +subject = "repo:scitt-community/scitt-api-emulator:ref:refs/heads/main" + +old_socket_getaddrinfo = socket.getaddrinfo +old_create_sockets = hypercorn.config.Config.create_sockets +old_webfinger_response_json = bovine.utils.webfinger_response_json + + +def load_services_from_services_path(services, host): + if isinstance(services, (str, pathlib.Path)): + services_path = pathlib.Path(services) + if not services_path.exists(): + raise socket.gaierror(f"{host} has not bound yet") + services_content = services_path.read_text() + services_dict = json.loads(services_content) + services = { + handle_name: types.SimpleNamespace(**service_dict) + for handle_name, service_dict in services_dict.items() + } + print("services:", services) + return services + +def socket_getaddrinfo_map_service_ports(services, host, *args, **kwargs): + # Map f"scitt.{handle_name}.example.com" to various local ports + if "scitt." not in host: + return old_socket_getaddrinfo(host, *args, **kwargs) + _, handle_name, _, _ = host.split(".") + services = load_services_from_services_path(services, host) + if handle_name not in services: + raise socket.gaierror(f"{host} has not bound yet") + return [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + 6, + "", + ("127.0.0.1", services[handle_name].port), + ) + ] + + +# TODO Remvoe, no need to mock if we set set scheme in domain on store.register +def http_webfinger_response_json(*args, **kwargs): + webfinger_response_json = old_webfinger_response_json(*args, **kwargs) + return webfinger_response_json + # webfinger_response_json["links"][0]["href"] = webfinger_response_json["links"][0]["href"].replace("https://", "http://") + + +def make_MockClientRequest(services): + class MockClientRequest(aiohttp.ClientRequest): + def __init__(self, method, url, *args, **kwargs): + nonlocal services + if "scitt." in url.host: + # uri = urllib.parse.urlparse(url) + # host = uri.hostname + host = url.host + _, handle_name, _, _ = host.split(".") + services = load_services_from_services_path(services, host) + if handle_name not in services: + raise socket.gaierror(f"{host} has not bound yet") + url = url.with_host("127.0.0.1") + url = url.with_port(services[handle_name].port) + kwargs.setdefault("headers", {}) + kwargs["headers"]["Host"] = f"http://{host}" + super().__init__(method, url, *args, **kwargs) + print("Is SSL?", self.is_ssl()) + return MockClientRequest def execute_cli(argv): return cli.main([str(v) for v in argv]) class Service: - def __init__(self, config, create_flask_app=None): + def __init__(self, config, create_flask_app=None, services=None): self.config = config self.create_flask_app = ( create_flask_app if create_flask_app is not None else server.create_flask_app ) + self.services = services def __enter__(self): - app = self.create_flask_app(self.config) - if hasattr(app, "service_parameters_path"): - self.service_parameters_path = app.service_parameters_path + self.app = self.create_flask_app(self.config) + if hasattr(self.app, "service_parameters_path"): + self.service_parameters_path = self.app.service_parameters_path self.host = "127.0.0.1" - self.server = make_server(self.host, 0, app) - port = self.server.port - self.url = f"http://{self.host}:{port}" - app.url = self.url - self.thread = threading.Thread(name="server", target=self.server.serve_forever) - self.thread.start() + addr_queue = multiprocessing.Queue() + self.process = multiprocessing.Process(name="server", target=self.server_process, + args=(self.app, addr_queue, + self.services)) + self.process.start() + self.host = addr_queue.get(True) + self.port = addr_queue.get(True) + self.url = f"http://{self.host}:{self.port}" + self.app.url = self.url return self def __exit__(self, *args): - self.server.shutdown() - self.thread.join() + self.process.terminate() + self.process.join() + + @staticmethod + def server_process(app, addr_queue, services): + # os.environ["BUTCHER_ALLOW_HTTP"] = "1" + try: + with contextlib.ExitStack() as exit_stack: + MockClientRequest = make_MockClientRequest(services) + exit_stack.enter_context( + unittest.mock.patch( + "aiohttp.ClientRequest", + side_effect=MockClientRequest, + ) + ) + class MockConfig(hypercorn.config.Config): + def create_sockets(self, *args, **kwargs): + sockets = old_create_sockets(self, *args, **kwargs) + server_name, server_port = sockets.insecure_sockets[0].getsockname() + addr_queue.put(server_name) + addr_queue.put(server_port) + app.host = server_name + app.port = server_port + app.url = f"http://{app.host}:{app.port}" + # Ensure that connect calls to them resolve as we want + exit_stack.enter_context( + unittest.mock.patch( + "socket.getaddrinfo", + wraps=functools.partial( + socket_getaddrinfo_map_service_ports, + services, + ) + ) + ) + return sockets + + exit_stack.enter_context( + unittest.mock.patch( + "quart.app.HyperConfig", + side_effect=MockConfig, + ) + ) + exit_stack.enter_context( + unittest.mock.patch( + "bovine.utils.webfinger_response_json", + wraps=http_webfinger_response_json, + ) + ) + exit_stack.enter_context( + unittest.mock.patch( + "bovine_herd.server.wellknown.webfinger_response_json", + wraps=http_webfinger_response_json, + ) + ) + app.run(port=0) + except: + # traceback.print_exc() + pass @pytest.mark.parametrize( "use_lro", [True, False], @@ -71,8 +214,8 @@ def test_client_cli(use_lro: bool, tmp_path): "create-claim", "--out", claim_path, - "--issuer", - issuer, + "--subject", + "test", "--content-type", content_type, "--payload", @@ -157,11 +300,27 @@ def test_client_cli(use_lro: bool, tmp_path): def create_flask_app_oidc_server(config): - app = Flask("oidc_server") + app = Quart("oidc_server") app.config.update(dict(DEBUG=True)) app.config.update(config) + # TODO For testing ssh key style issuers, not OIDC related needs to be moved + @app.route("/", methods=["GET"]) + def ssh_public_keys(): + from cryptography.hazmat.primitives import serialization + return send_file( + io.BytesIO( + serialization.load_pem_public_key( + app.config["key"].export_to_pem(), + ).public_bytes( + encoding=serialization.Encoding.OpenSSH, + format=serialization.PublicFormat.OpenSSH, + ) + ), + mimetype="text/plain", + ) + @app.route("/.well-known/openid-configuration", methods=["GET"]) def openid_configuration(): return jsonify( @@ -203,7 +362,6 @@ def test_client_cli_token(tmp_path): key = jwcrypto.jwk.JWK.generate(kty="RSA", size=2048) algorithm = "RS256" audience = "scitt.example.org" - subject = "repo:scitt-community/scitt-api-emulator:ref:refs/heads/main" with Service( {"key": key, "algorithms": [algorithm]}, @@ -232,8 +390,8 @@ def test_client_cli_token(tmp_path): ) with Service( { - "middleware": OIDCAuthMiddleware, - "middleware_config_path": middleware_config_path, + "middleware": [OIDCAuthMiddleware], + "middleware_config_path": [middleware_config_path], "tree_alg": "CCF", "workspace": workspace_path, "error_rate": 0.1, @@ -246,8 +404,8 @@ def test_client_cli_token(tmp_path): "create-claim", "--out", claim_path, - "--issuer", - issuer, + "--subject", + "test", "--content-type", content_type, "--payload", diff --git a/tests/test_docs.py b/tests/test_docs.py index ea3d92d9..b1e6948f 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -7,44 +7,53 @@ import copy import types import pathlib +import logging import tempfile import textwrap import threading import itertools +import traceback import subprocess import contextlib -import unittest.mock -import pytest +import urllib.parse + import myst_parser.parsers.docutils_ import docutils.nodes import docutils.utils +import jwcrypto + +from scitt_emulator.tree_algs import TREE_ALGS +from scitt_emulator.signals import SCITTSignals from scitt_emulator.client import ClaimOperationError from .test_cli import ( Service, content_type, payload, + subject, execute_cli, + create_flask_app_oidc_server, ) +logger = logging.getLogger(__name__) + repo_root = pathlib.Path(__file__).parents[1] docs_dir = repo_root.joinpath("docs") -allowlisted_issuer = "did:web:example.org" -non_allowlisted_issuer = "did:web:example.com" +non_allowlisted_issuer = "did:web:denied.example.com" CLAIM_DENIED_ERROR = {"type": "denied", "detail": "content_address_of_reason"} CLAIM_DENIED_ERROR_BLOCKED = { "type": "denied", "detail": textwrap.dedent( """ - 'did:web:example.com' is not one of ['did:web:example.org'] + 'did:web:denied.example.com' is not one of ['did:web:example.org'] Failed validating 'enum' in schema['properties']['issuer']: {'enum': ['did:web:example.org'], 'type': 'string'} On instance['issuer']: - 'did:web:example.com' + 'did:web:denied.example.com' """ ).lstrip(), } @@ -84,7 +93,14 @@ def poll_workspace(config, stop_event): while running: for cose_path in operations_path.glob("*.cose"): denial = copy.deepcopy(CLAIM_DENIED_ERROR) - with open(cose_path, "rb") as stdin_fileobj: + with contextlib.ExitStack() as exit_stack: + try: + stdin_fileobj = exit_stack.enter_context( + open(cose_path, "rb"), + ) + except: + logger.error(traceback.format_exc()) + continue env = { **os.environ, "SCHEMA_PATH": str(config["schema_path"].resolve()), @@ -152,6 +168,15 @@ def docutils_find_code_samples(nodes): samples[node.astext()] = nodes[i + 3].astext() return samples +def url_to_did_web(url_string): + url = urllib.parse.urlparse(url_string) + return ":".join( + [ + urllib.parse.quote(i) + for i in ["did", "web", url.netloc, *filter(bool, url.path.split("/"))] + ] + ) + def test_docs_registration_policies(tmp_path): workspace_path = tmp_path / "workspace" @@ -159,6 +184,7 @@ def test_docs_registration_policies(tmp_path): receipt_path = tmp_path / "claim.receipt.cbor" entry_id_path = tmp_path / "claim.entry_id.txt" retrieved_claim_path = tmp_path / "claim.retrieved.cose" + private_key_pem_path = tmp_path / "notary-private-key.pem" # Grab code samples from docs # TODO Abstract into abitrary docs testing code @@ -170,40 +196,78 @@ def test_docs_registration_policies(tmp_path): for name, content in docutils_find_code_samples(nodes).items(): tmp_path.joinpath(name).write_text(content) + key = jwcrypto.jwk.JWK.generate(kty="EC", crv="P-384") + # cwt_cose_key = cwt.COSEKey.generate_symmetric_key(alg=alg, kid=kid) + private_key_pem_path.write_bytes( + key.export_to_pem(private_key=True, password=None), + ) + algorithm = "ES384" + audience = "scitt.example.org" + + # tell jsonschema_validator.py that we want to assume non-TLS URLs for tests + os.environ["DID_WEB_ASSUME_SCHEME"] = "http" + + # ensure we use the policy engine + storage_path = workspace_path / "storage" + storage_path.mkdir(parents=True) + service_parameters_path = workspace_path / "service_parameters.json" + tree_alg = "CCF" + TREE_ALGS[tree_alg]( + signals=SCITTSignals(), + storage_path=storage_path, + service_parameters_path=service_parameters_path, + ).initialize_service() + service_parameters = json.loads(service_parameters_path.read_text()) + service_parameters["insertPolicy"] = "external" + service_parameters_path.write_text(json.dumps(service_parameters)) + with Service( + {"key": key, "algorithms": [algorithm]}, + create_flask_app=create_flask_app_oidc_server, + ) as oidc_service, Service( { - "tree_alg": "CCF", + "tree_alg": tree_alg, "workspace": workspace_path, "error_rate": 0.1, "use_lro": True, } ) as service, SimpleFileBasedPolicyEngine( { - "storage_path": service.server.app.scitt_service.storage_path, + "storage_path": storage_path, "enforce_policy": tmp_path.joinpath("enforce_policy.py"), "jsonschema_validator": tmp_path.joinpath("jsonschema_validator.py"), "schema_path": tmp_path.joinpath("allowlist.schema.json"), } ) as policy_engine: - # set the policy to enforce - service.server.app.scitt_service.service_parameters["insertPolicy"] = "external" + # set the issuer to the did:web version of the OIDC / SSH keys service + issuer = url_to_did_web(oidc_service.url) - # create denied claim + # create claim command = [ "client", "create-claim", "--out", claim_path, "--issuer", - non_allowlisted_issuer, + issuer, + "--subject", + subject, "--content-type", content_type, "--payload", payload, + "--private-key-pem", + private_key_pem_path, ] execute_cli(command) assert os.path.exists(claim_path) + # replace example issuer with test OIDC service issuer (URL) in error + claim_denied_error_blocked = CLAIM_DENIED_ERROR_BLOCKED + claim_denied_error_blocked["detail"] = claim_denied_error_blocked["detail"].replace( + "did:web:denied.example.com", issuer, + ) + # submit denied claim command = [ "client", @@ -224,27 +288,39 @@ def test_docs_registration_policies(tmp_path): check_error = error assert check_error assert "error" in check_error.operation - assert check_error.operation["error"] == CLAIM_DENIED_ERROR_BLOCKED + assert check_error.operation["error"] == claim_denied_error_blocked assert not os.path.exists(receipt_path) assert not os.path.exists(entry_id_path) - # create accepted claim + # replace example issuer with test OIDC service issuer in allowlist + allowlist_schema_json_path = tmp_path.joinpath("allowlist.schema.json") + allowlist_schema_json_path.write_text( + allowlist_schema_json_path.read_text().replace( + "did:web:example.org", issuer, + ) + ) + + # submit accepted claim using SSH authorized_keys lookup command = [ "client", - "create-claim", - "--out", + "submit-claim", + "--claim", claim_path, - "--issuer", - allowlisted_issuer, - "--content-type", - content_type, - "--payload", - payload, + "--out", + receipt_path, + "--out-entry-id", + entry_id_path, + "--url", + service.url ] execute_cli(command) - assert os.path.exists(claim_path) + assert os.path.exists(receipt_path) + receipt_path.unlink() + assert os.path.exists(entry_id_path) + receipt_path.unlink(entry_id_path) - # submit accepted claim + # TODO Switch back on the OIDC routes + # submit accepted claim using OIDC -> jwks lookup command = [ "client", "submit-claim", @@ -259,4 +335,6 @@ def test_docs_registration_policies(tmp_path): ] execute_cli(command) assert os.path.exists(receipt_path) + receipt_path.unlink() assert os.path.exists(entry_id_path) + receipt_path.unlink(entry_id_path) diff --git a/tests/test_federation_activitypub_bovine.py b/tests/test_federation_activitypub_bovine.py new file mode 100644 index 00000000..535112df --- /dev/null +++ b/tests/test_federation_activitypub_bovine.py @@ -0,0 +1,337 @@ +# Copyright (c) SCITT Authors +# Licensed under the MIT License. +import os +import sys +import time +import json +import copy +import types +import socket +import asyncio +import pathlib +import tempfile +import textwrap +import threading +import functools +import itertools +import subprocess +import contextlib +import unittest.mock + +import aiohttp +import pytest +import tomllib +import myst_parser.parsers.docutils_ +import docutils.nodes +import docutils.utils +import bovine + +from scitt_emulator.tree_algs import TREE_ALGS +from scitt_emulator.signals import SCITTSignals +from scitt_emulator.client import ClaimOperationError +from scitt_emulator.federation_activitypub_bovine import ( + SCITTFederationActivityPubBovine, +) + +from .test_cli import ( + Service, + content_type, + payload, + subject, + execute_cli, + socket_getaddrinfo_map_service_ports, + make_MockClientRequest, +) +from .test_docs import ( + docutils_recursively_extract_nodes, + docutils_find_code_samples, + SimpleFileBasedPolicyEngine, +) + + +repo_root = pathlib.Path(__file__).parents[1] +docs_dir = repo_root.joinpath("docs") + + +@pytest.mark.parametrize('anyio_backend', ['asyncio']) +async def test_docs_federation_activitypub_bovine(anyio_backend, tmp_path): + claim_path = tmp_path / "claim.cose" + receipt_path = tmp_path / "claim.receipt.cbor" + entry_id_path = tmp_path / "claim.entry_id.txt" + retrieved_claim_path = tmp_path / "claim.retrieved.cose" + + # Grab code samples from docs + # TODO Abstract into abitrary docs testing code + doc_path = docs_dir.joinpath("registration_policies.md") + markdown_parser = myst_parser.parsers.docutils_.Parser() + document = docutils.utils.new_document(str(doc_path.resolve())) + parsed = markdown_parser.parse(doc_path.read_text(), document) + nodes = docutils_recursively_extract_nodes(document) + for name, content in docutils_find_code_samples(nodes).items(): + tmp_path.joinpath(name).write_text(content) + + # Allow any issuers + allowlist_schema_json_path = tmp_path.joinpath("allowlist.schema.json") + allowlist_schema_json = json.loads(allowlist_schema_json_path.read_text()) + del allowlist_schema_json["properties"]["issuer"]["enum"] + allowlist_schema_json_path.write_text(json.dumps(allowlist_schema_json)) + + services = {} + policy_engines = {} + bovine_clients = {} + services_path = tmp_path / "services.json" + + MockClientRequest = make_MockClientRequest(services_path) + + class TestSCITTFederationActivityPubBovine(SCITTFederationActivityPubBovine): + async def make_client_session(self): + nonlocal MockClientRequest + return aiohttp.ClientSession(trust_env=True, + request_class=MockClientRequest) + + for handle_name, following in { + "bob": { + "alice": { + "actor_id": "alice@scitt.alice.example.com", + "domain": "http://scitt.alice.example.com", + }, + }, + "alice": { + "bob": { + "actor_id": "bob@scitt.bob.example.com", + "domain": "http://scitt.bob.example.com", + }, + }, + }.items(): + middleware_config_path = ( + tmp_path + / handle_name + / "federation-activitypub-bovine-middleware-config.json" + ) + middleware_config_path.parent.mkdir() + middleware_config_path.write_text( + json.dumps( + { + "handle_name": handle_name, + "fqdn": f"http://scitt.{handle_name}.example.com", + "workspace": str(tmp_path / handle_name), + "bovine_db_url": f"sqlite://{(tmp_path / handle_name / 'bovine.sqlite3').resolve()}", + "following": following, + } + ) + ) + + # ensure service parameters include methods service can federate by + workspace_path = tmp_path / handle_name / "workspace" + storage_path = workspace_path / "storage" + storage_path.mkdir(parents=True) + service_parameters_path = workspace_path / "service_parameters.json" + tree_alg = "CCF" + TREE_ALGS[tree_alg]( + signals=SCITTSignals(), + storage_path=storage_path, + service_parameters_path=service_parameters_path, + ).initialize_service() + service_parameters = json.loads(service_parameters_path.read_text()) + service_parameters["use_lro"] = True + service_parameters["insertPolicy"] = "external" + # TODO Decide on how we offer extensions for more federation protocols + # and declare which version is in use. We would need an extension doc + # which describes the format of this blob and how to intrepret it + # https://github.com/ietf-wg-scitt/draft-ietf-scitt-architecture/issues/79#issuecomment-1797016940 + service_parameters["federation"] = [ + { + "protocol": "https://github.com/w3c/activitypub", + "version": "https://github.com/w3c/activitypub/commit/cda0c902317f194daeeb50b2df0225bca5b06f52", + "activitypub": { + "actors": { + handle_name: { + # SCITT_ALL_SUBJECTS would be a special value + # We'd want to have extension docs explain more + "subjects": "SCITT_ALL_SUBJECTS", + } + } + } + } + ] + service_parameters_path.write_text(json.dumps(service_parameters)) + services[handle_name] = Service( + { + "middleware": [TestSCITTFederationActivityPubBovine], + "middleware_config_path": [middleware_config_path], + "tree_alg": "CCF", + "workspace": workspace_path, + "error_rate": 0, + "use_lro": True, + }, + services=services_path, + ) + policy_engines[handle_name] = SimpleFileBasedPolicyEngine( + { + "storage_path": storage_path, + "enforce_policy": tmp_path.joinpath("enforce_policy.py"), + "jsonschema_validator": tmp_path.joinpath("jsonschema_validator.py"), + "schema_path": tmp_path.joinpath("allowlist.schema.json"), + } + ) + + + # TODO __aexit__ + async_exit_stack = await contextlib.AsyncExitStack().__aenter__() + + with contextlib.ExitStack() as exit_stack: + # Ensure that connect calls to them resolve as we want + exit_stack.enter_context( + unittest.mock.patch( + "socket.getaddrinfo", + wraps=functools.partial( + socket_getaddrinfo_map_service_ports, + services, + ) + ) + ) + # Start all the services + for handle_name, service in services.items(): + services[handle_name] = exit_stack.enter_context(service) + # Test of resolution + assert ( + socket.getaddrinfo(f"scitt.{handle_name}.example.com", 0)[0][-1][-1] + == services[handle_name].port + ) + # Start the policy engine for the service + policy_engines[handle_name] = exit_stack.enter_context(policy_engines[handle_name]) + + # Serialize services + services_path.write_text( + json.dumps( + { + handle_name: {"port": service.port} + for handle_name, service in services.items() + } + ) + ) + exit_stack.enter_context( + unittest.mock.patch( + "aiohttp.client_reqrep.ClientRequest", + side_effect=make_MockClientRequest(services), + ) + ) + exit_stack.enter_context( + unittest.mock.patch( + "aiohttp.client.ClientRequest", + side_effect=make_MockClientRequest(services), + ) + ) + + # Ensure we have a client for each service + for handle_name, service in services.items(): + config_toml_path = tmp_path / handle_name / "config.toml" + config_toml_obj = {} + while not config_toml_path.exists() or len(config_toml_obj) == 0: + await asyncio.sleep(0.1) + if config_toml_path.exists(): + config_toml_obj = tomllib.loads(config_toml_path.read_text()) + bovine_clients[handle_name] = await async_exit_stack.enter_async_context( + bovine.BovineClient(**config_toml_obj[handle_name]) + ) + + # Poll following endpoints until all services are following each other + for handle_name, client in bovine_clients.items(): + count_accepts = 0 + while count_accepts != (len(bovine_clients) - 1): + count_accepts = 0 + async for message in client.outbox(): + if message["type"] == "Accept": + count_accepts += 1 + + # Create claims in each instance + claims = [] + for handle_name, service in services.items(): + # create claim + command = [ + "client", + "create-claim", + "--out", + claim_path, + "--subject", + subject, + "--content-type", + content_type, + "--payload", + payload, + ] + execute_cli(command) + assert os.path.exists(claim_path) + + # submit claim + command = [ + "client", + "submit-claim", + "--claim", + claim_path, + "--out", + receipt_path, + "--out-entry-id", + entry_id_path, + "--url", + service.url, + ] + execute_cli(command) + claim = claim_path.read_bytes() + claim_path.unlink() + assert os.path.exists(receipt_path) + receipt_path.unlink() + assert os.path.exists(entry_id_path) + entry_id = entry_id_path.read_text() + entry_id_path.unlink() + + claims.append( + { + "entry_id": entry_id, + "claim": claim, + "service.handle_name": handle_name, + } + ) + + # await asyncio.sleep(100) + + # Test that we can download claims from all instances federated with + for handle_name, service in services.items(): + for claim in claims: + entry_id = claim["entry_id"] + original_handle_name = claim["service.handle_name"] + # Do not test claim retrieval from submission service here, only + # services federated with + if original_handle_name == handle_name: + continue + their_claim_path = claim_path.with_suffix( + f".federated.{original_handle_name}.to.{handle_name}" + ) + command = [ + "client", + "retrieve-claim", + "--entry-id", + entry_id, + "--out", + their_claim_path, + "--url", + service.url, + ] + # TODO Retry with backoff with cap + # TODO Remove try except, fix federation + error = None + for i in range(0, 10): + try: + execute_cli(command) + break + except Exception as e: + if "urn:ietf:params:scitt:error:entryNotFound" in str(e): + error = e + time.sleep(1) + else: + raise + if error: + raise error + assert os.path.exists(their_claim_path) + assert their_claim_path.read_bytes() == claim["claim"] + their_claim_path.unlink()