Merge pull request #13 from sunbeam-labs/12-containerize-and-version

12 containerize and version

.github/workflows/dockerhub.yml

@@ -0,0 +1,36 @@
+name: Push to DockerHub
+
+on:
+  workflow_call:
+
+  workflow_dispatch:
+
+jobs:
+    build-and-push-to-dockerhub:
+        name: Push Docker image to Docker Hub
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Log in to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                username: ${{ secrets.DOCKERHUB_USERNAME }}
+                password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Extract metadata (tags, labels) for Docker
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                images: sunbeamlabs/sbx_template
+
+            - name: Build and push Docker image
+              uses: docker/build-push-action@v5
+              with:
+                context: .
+                file: envs/sbx_template_env.Dockerfile
+                push: true
+                tags: ${{ steps.meta.outputs.tags }}
+                labels: ${{ steps.meta.outputs.labels }}

.github/workflows/pr.yml

@@ -0,0 +1,14 @@
+name: Tests
+
+on:
+    pull_request:
+      branches:
+        - main
+    push:
+      branches:
+        - main
+
+jobs:
+    run-tests:
+        uses: ./.github/workflows/tests.yml
+        secrets: inherit

.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: Release
+
+on:
+    release:
+      types: [published]
+
+    workflow_dispatch:
+
+jobs:
+    run-tests:
+        uses: ./.github/workflows/tests.yml
+        secrets: inherit
+
+    check-version:
+        runs-on: ubuntu-latest
+        steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+
+          - name: Get sbx version
+            shell: bash
+            run: |
+              SBX_VER=$(cat VERSION)
+              echo "SBX_VER=$SBX_VER" >> $GITHUB_ENV
+
+          - id: get_version
+            uses: battila7/get-version-action@v2
+
+          - name: Check version
+            shell: bash
+            run: |
+              RELEASE_VERSION=${{ steps.get_version.outputs.version-without-v }}
+              echo "Release version: ${RELEASE_VERSION}"
+              echo "Sbx version: ${{ env.SBX_VER }}"
+
+              if [[ $RELEASE_VERSION == ${{ env.SBX_VER }} ]]; then
+                  echo "Versions match, continuing..."
+              else
+                  echo "Versions don't match, exiting..."
+                  exit 1
+              fi
+
+    push-to-dockerhub:
+        uses: ./.github/workflows/docker.yml
+        secrets: inherit
+        needs:
+          - run-tests
+          - check-version
+
+    test-apptainer:
+      name: Apptainer Test
+      runs-on: ubuntu-latest
+      needs: push-to-dockerhub
+
+      steps:
+        - name: Checkout Code
+          uses: actions/checkout@v4
+
+        - name: Set test env
+          run: echo "SUNBEAM_TEST_PROFILE=apptainer" >> $GITHUB_ENV
+
+        - uses: eWaterCycle/setup-apptainer@v2
+          with:
+            apptainer-version: 1.1.2
+
+        - name: Test with Sunbeam
+          uses: sunbeam-labs/sbx_test_action@v1
+          with:
+            test-directory: ".tests/e2e/"
+
+        - name: Dump Logs
+          shell: bash
+          if: always()
+          run: tail -n +1 logs/*
+
+        - name: Dump Stats
+          shell: bash
+          if: always()
+          run: cat stats/*

.github/workflows/tests.yml

@@ -5,32 +5,71 @@ on:
     branches: [ master, main ]
   push:
     branches: [ master, main ]
+  workflow_call:
+  workflow_dispatch:
   schedule:
     - cron: "0 13 * * 1"
 
-jobs:  
-  test-e2e:
-    name: Test Extension with Sunbeam
+jobs:
+  lint:
+    name: Lint Code
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
-
-      - name: Test with Sunbeam
-        uses: sunbeam-labs/sbx_test_action@v1
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+            python-version: 3.12
 
-  test-units:
-    name: Run Unittests
+      - name: Install Dependencies
+        run: pip install black snakefmt
+
+      - name: Run Linter
+        run: |
+          black --check .
+          snakefmt --check *.smk
+
+  test-unit:
+    name: Run Extension Unit Tests
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
-
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+            python-version: 3.12
+
       - name: Install Dependencies
-        run: python -m pip install pytest
+        run: pip install pytest
+
+      - name: Run Unit Tests
+        run: true #pytest .tests/unit/
+        # This'll require having a lib within scripts with internal tests
+
+  test-e2e:
+    name: Test Extension with Sunbeam
+    runs-on: ubuntu-latest
+    needs:
+      - test-unit
+      - lint
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Test with Sunbeam
+        uses: sunbeam-labs/sbx_test_action@v1
+
+      - name: Dump Logs
+        shell: bash
+        if: always()
+        run: tail -n +1 logs/*
 
-      # TODO: Add demo unit tests
-      #- name: Run Tests
-      #  run: pytest .tests/unit/
+      - name: Dump Stats
+        shell: bash
+        if: always()
+        run: cat stats/*

.tests/e2e/test_full_run.py

@@ -1,8 +1,7 @@
-import csv
-import os
 import pytest
 import shutil
 import subprocess as sp
+import sys
 import tempfile
 from pathlib import Path
 
@@ -41,19 +40,30 @@ def setup():
 @pytest.fixture
 def run_sunbeam(setup):
     temp_dir, project_dir = setup
-
-    # Run the test job.
-    sp.check_output(
-        [
-            "sunbeam",
-            "run",
-            "--profile",
-            project_dir,
-            "all_template",
-            "--directory",
-            temp_dir,
-        ]
-    )
+    output_fp = project_dir / "sunbeam_output"
+    log_fp = output_fp / "logs"
+    stats_fp = project_dir / "stats"
+
+    # Run the test job
+    try:
+        sp.check_output(
+            [
+                "sunbeam",
+                "run",
+                "--profile",
+                project_dir,
+                "all_template",
+                "--directory",
+                temp_dir,
+            ]
+        )
+    except sp.CalledProcessError as e:
+        shutil.copytree(log_fp, "logs/")
+        shutil.copytree(stats_fp, "stats/")
+        sys.exit(e)
+
+    shutil.copytree(log_fp, "logs/")
+    shutil.copytree(stats_fp, "stats/")
 
     output_fp = project_dir / "sunbeam_output"
     benchmarks_fp = project_dir / "stats/"

README.md

@@ -32,7 +32,7 @@ The dependency .yml file can be named whatever you want, as long as you refer to
 
 ### .github/
 
-This directory contains CI workflows for GitHub to run automatically on PRs, including tests and linting. If the linter raises errors, you can fix them by running `snakefmt` on any snakemake files and `black` on any python files.
+This directory contains CI workflows for GitHub to run automatically on PRs, including tests and linting. If the linter raises errors, you can fix them by running `snakefmt` on any snakemake files and `black` on any python files. The release workflow will build and push a docker image for each environment in the extension.
 
 ### .tests/
 
@@ -42,6 +42,10 @@ This directory contains tests, broken down into types such as end-to-end (e2e) a
 
 This directory contains scripts that can be run by rules. Use this for any rules that need to run python, R, etc code.
 
+### envs/*.Dockerfile
+
+The Dockerfiles provided with conda env specifications allow for containerized runs of sunbeam (meaning they use docker containers to run each rule rather than conda envs).
+
 (You can delete everything above this line)
 -----------------------------------------------------------------
 

VERSION

@@ -0,0 +1 @@
+0.0.0

config.yml

@@ -1,2 +1,2 @@
 sbx_template:
-  example_rule_options: ''
+  example_rule_options: 'DUMMY STRING'

envs/sbx_template_env.Dockerfile

@@ -0,0 +1,17 @@
+FROM condaforge/mambaforge:latest
+
+# Setup
+WORKDIR /home/sbx_template_env
+
+COPY envs/sbx_template_env.yml ./
+
+# Install environment
+RUN conda env create --file sbx_template_env.yml --name sbx_template
+
+ENV PATH="/opt/conda/envs/sbx_template/bin/:${PATH}"
+
+# "Activate" the environment
+SHELL ["conda", "run", "-n", "sbx_template", "/bin/bash", "-c"]
+
+# Run
+CMD "bash"

envs/sbx_template_env.yml

@@ -2,3 +2,5 @@ name: sbx_template
 channels:
   - conda-forge
   - bioconda
+dependencies:
+  - samtools