diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b4202be0..040587d5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -49,7 +49,7 @@ jobs: shell: bash run: | cd $GITHUB_WORKSPACE - ls -l . | grep "slurm_test_" + ls -l .snakemake/slurm_logs/ | grep "rule_test" test-apptainer: name: Apptainer Test @@ -77,4 +77,4 @@ jobs: sunbeam init --data_fp tests/data/reads/ --profile apptainer projects/test/ - sunbeam run --profile projects/test/ \ No newline at end of file + sunbeam run --profile projects/test/ --docker_tag latest \ No newline at end of file diff --git a/docs/commands.rst b/docs/commands.rst index d570ab61..d1f89589 100755 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -46,7 +46,7 @@ Sunbeam Commands Executes the Sunbeam pipeline by calling Snakemake. .. code-block:: shell - sunbeam run [-h] [-m] [-s PATH] [--target_list [TARGETS, ...]] [--include [INCLUDES, ...]] [--exclude [EXCLUDE, ...]] -- + sunbeam run [-h] [-m] [-s PATH] [--target_list [TARGETS, ...]] [--include [INCLUDES, ...]] [--exclude [EXCLUDE, ...]] [--docker_tag TAG] .. tip:: The ``--target_list`` option is deprecated. Pass the targets directly to ``sunbeam run`` instead. @@ -66,7 +66,8 @@ Sunbeam Commands --target_list: A list of targets to run successively. (DEPRECATED) --include: List of extensions to include in run. --exclude: List of extensions to exclude from run, use 'all' to exclude all extensions. - : You can pass further arguments to Snakemake after ``--``, e.g: ``$ sunbeam run -- --cores 12``. See http://snakemake.readthedocs.io for more information. + --docker_tag: Tag to use for internal environment docker images. Try 'latest' if the default tag doesn't work. + : You can pass further arguments to Snakemake, e.g: ``$ sunbeam run --cores 12``. See http://snakemake.readthedocs.io for more information. .. tip:: The ``--profile`` option is a snakemake option but should be used whenever using ``sunbeam run``. The main sunbeam snakefile requires a config object to be defined and the profile created by ``sunbeam init`` will always specify a config file to get that from. diff --git a/docs/examples.rst b/docs/examples.rst index 3b6d8286..dada3697 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -54,7 +54,7 @@ Then you submit the job: sbatch run_sunbeam.sh -Once this run completes, you will have a directory called ``/projects/my_project/sunbeam_output/`` that contains all of the output from the run and ``slurm_*`` files wherever you ran the main script from that contains logs for each job. Look in ``/projects/my_project/sunbeam_output/assembly/contigs/`` for the assembled contigs. +Once this run completes, you will have a directory called ``/projects/my_project/sunbeam_output/`` that contains all of the output from the run and ``.snakemake/slurm_logs/rule_*/`` directories wherever you ran the main script from that contain logs for each job. Look in ``/projects/my_project/sunbeam_output/assembly/contigs/`` for the assembled contigs. Using Containerized Environments -------------------------------- diff --git a/pyproject.toml b/pyproject.toml index b9f06f12..81b601f9 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,7 +104,6 @@ classifiers = [ # Optional # For an analysis of this field vs pip's requirements files see: # https://packaging.python.org/discussions/install-requires-vs-requirements/ dependencies = [ # Optional - "docker", "more-itertools", "pyyaml", ] diff --git a/src/sunbeamlib/__init__.py b/src/sunbeamlib/__init__.py index cad829a9..b1dd6c71 100755 --- a/src/sunbeamlib/__init__.py +++ b/src/sunbeamlib/__init__.py @@ -37,20 +37,9 @@ def __str__(self) -> str: def get_docker_str(repo: str, user: str = "sunbeamlabs") -> str: - # Docker import needs to live here to avoid circular imports - # pyproject.toml needs __version__ to be defined before installing dependencies - import docker - - client = docker.from_env() - image_name = f"{user}/{repo}:{__version__}" - try: - client.images.get_registry_data(image_name) - return f"docker://{image_name}" - except docker.errors.NotFound: - sys.stderr.write( - f"WARNING: {image_name} not found on DockerHub, using latest tag instead.\n" - ) - return f"docker://{user}/{repo}:latest" + docker_tag = os.environ.get("SUNBEAM_DOCKER_TAG", __version__) + + return f"docker://{user}/{repo}:{docker_tag}" def load_sample_list( diff --git a/src/sunbeamlib/script_run.py b/src/sunbeamlib/script_run.py index 91f561f6..036fdbf5 100644 --- a/src/sunbeamlib/script_run.py +++ b/src/sunbeamlib/script_run.py @@ -4,6 +4,8 @@ import subprocess from pathlib import Path +from sunbeamlib import __version__ + def main(argv=sys.argv): epilog_str = ( @@ -51,6 +53,11 @@ def main(argv=sys.argv): default=[], help="List of extensions to exclude from run, use 'all' to exclude all extensions", ) + parser.add_argument( + "--docker_tag", + default=__version__, + help="The tag to use when pulling docker images for the core pipeline environments, defaults to sunbeam's current version ($SUNBEAM_VER), a good alternative is 'latest' for the latest stable release", + ) # The remaining args (after --) are passed to Snakemake args, remaining = parser.parse_known_args(argv) @@ -83,6 +90,8 @@ def main(argv=sys.argv): if args.exclude: os.environ["SUNBEAM_EXTS_EXCLUDE"] = ", ".join(args.exclude) + os.environ["SUNBEAM_DOCKER_TAG"] = args.docker_tag + snakemake_args = ( [ "snakemake", diff --git a/workflow/Snakefile b/workflow/Snakefile index 99ccd7a5..e78c6674 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -179,17 +179,6 @@ localrules: samples, -for rule_name in rules._rules: - rule_obj = getattr(rules, rule_name).rule - wcs = rule_obj._wildcard_names - if wcs: - rule_obj.resources["slurm_extra"] = ( - lambda wc, rule_name=rule_name: f"--output=slurm_{rule_name}_{'_'.join(wc)}_%j" - ) - else: - rule_obj.resources["slurm_extra"] = f"--output=slurm_{rule_name}_%j" - - onstart: try: shutil.rmtree(BENCHMARK_FP) diff --git a/workflow/rules/qc.smk b/workflow/rules/qc.smk index 237b6944..3944991f 100755 --- a/workflow/rules/qc.smk +++ b/workflow/rules/qc.smk @@ -112,7 +112,7 @@ rule trimmomatic_unpaired: TRAILING:{Cfg[qc][trailing]} \ SLIDINGWINDOW:{params.sw_start}:{params.sw_end} \ MINLEN:{Cfg[qc][minlen]} \ - > >(tee {log}) 2> >(tee {log} >&2) + 2>&1 | tee {log} """ @@ -156,7 +156,7 @@ rule trimmomatic_paired: TRAILING:{Cfg[qc][trailing]} \ SLIDINGWINDOW:{params.sw_start}:{params.sw_end} \ MINLEN:{Cfg[qc][minlen]} \ - > >(tee {log}) 2> >(tee {log} >&2) + 2>&1 | tee {log} """