Merge pull request #264 from NationalGenomicsInfrastructure/monthly

Monthly merge Jul 2024

env_vars/site_sthlm_env_all.yml

@@ -24,6 +24,7 @@ multiqc_options:
   push_statusdb: True
   save_remote: True
   template: ngi
+  output_fn_name: multiqc_report.html
 
 multiqc_swedac_accredited: True
 multiqc_sshkey: /path/to/ssh_key

host_vars/deploy/main.yml

@@ -16,7 +16,7 @@ tools_path:
 
 # Python versions
 python2_version: 2.7
-python3_version: 3.8
+python3_version: 3.11
 
 #Nextflow defauts
 nextflow_dest: "{{ sw_path }}/nextflow"
@@ -46,10 +46,10 @@ ngi_pipeline_venv_py2: "{{ sw_path }}/anaconda/envs/{{ NGI_venv_py2_name }}"
 ngi_pipeline_conf: "{{ root_path }}/conf/"
 ngi_resources: "{{ root_path }}/resources/"
 
-sarek_tag: "2.7"
+sarek_tag: "3.4.2"
 sarek_dest: "{{ sw_path }}/sarek/{{ sarek_tag | replace('.', '_') }}"
 
-rnaseq_tag: "3.3"
+rnaseq_tag: "3.8.1"
 rnaseq_dest: "{{ sw_path }}/rnaseq/{{ rnaseq_tag | replace('.', '_') }}"
 
 demultiplex_tag: "1.4.1"

roles/archive-upload-ws/tasks/1_install.yml

@@ -17,13 +17,10 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: install archive-upload
   pip:
     name: .
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
-

roles/archive-verify-ws/tasks/1_install.yml

@@ -17,4 +17,3 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"

roles/arteria-checkqc-ws/defaults/main.yml

@@ -2,4 +2,4 @@ arteria_service_name: arteria-checkqc-ws
 arteria_checkqc_service_log: "{{ arteria_service_log_dir }}/{{ arteria_service_name }}.log"
 
 checkqc_repo: https://github.com/Molmed/checkQC.git
-checkqc_version: v4.0.2
+checkqc_version: v4.0.4

roles/arteria-checkqc-ws/tasks/install.yml

@@ -17,12 +17,10 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: install CheckQC
   pip:
     name: .
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"

roles/arteria-checksum-ws/tasks/1_install.yml

@@ -17,13 +17,10 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: install arteria-checksum
   pip:
     name: .
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
-

roles/arteria-delivery-ws/tasks/install.yml

@@ -16,12 +16,10 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: install arteria-delivery
   pip:
     name: .
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"

roles/arteria-sequencing-report-ws/tasks/install.yml

@@ -17,15 +17,13 @@
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: install sequencing report service
   pip:
     name: .
     chdir: "{{ arteria_service_src_path }}"
     executable: "{{ arteria_service_env_root }}/bin/pip"
     state: present
-    extra_args: "-U"
 
 - name: create temp folder
   tempfile:

roles/arteria-sequencing-report-ws/tasks/install_config.yml

@@ -45,9 +45,9 @@
 - name: deploying {{ arteria_service_name }} nextflow configs for pipelines
   template:
     src: "{{ item }}"
-    dest: "{{ arteria_service_nextflow_config_dir }}/{{ item | basename }}"
+    dest: "{{ arteria_service_nextflow_config_dir }}/{{ item | basename | splitext | first }}"
   with_fileglob:
-    - "../files/nextflow_configs/*.config"
+    - "../templates/nextflow_configs/*.config.j2"
 
 - name: Deploy arteria-delivery wrapper
   template:

roles/arteria-sequencing-report-ws/files/nextflow_configs/demultiplex.config → roles/arteria-sequencing-report-ws/templates/nextflow_configs/demultiplex.config.j2

@@ -10,6 +10,10 @@
 ----------------------------------------------------------------------------------------
 */
 
+plugins {
+    id 'nf-validation@{{ nf_validation_version }}'
+}
+
 process {
 
     withName: BCL2FASTQ {

roles/delivery/tasks/main.yml

@@ -13,7 +13,6 @@
   pip:
     executable: "{{ delivery_env_path }}/bin/pip"
     requirements: "{{ delivery_dest }}/requirements.txt"
-    extra_args: "-U"
     state: present
 
 - name: install delivery into venv

roles/multiqc/defaults/main.yml

@@ -1,7 +1,7 @@
 multiqc_repo: https://github.com/ewels/MultiQC.git
 multiqc_dest: "{{ sw_path }}/multiqc"
-multiqc_version: "v1.21"
+multiqc_version: "v1.22.3"
 
 multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git
 multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi"
-multiqc_ngi_version: "0.7.2"
+multiqc_ngi_version: "0.8.0"

roles/multiqc/templates/multiqc_config.yml.j2

@@ -5,8 +5,11 @@ export_plots: true
 swedac_location: {{ site_full }}
 swedac_accredited: {{ multiqc_swedac_accredited }}
 
+#Restrict megaqc to upps for now
+{% if site == "upps" %}
 megaqc_url: {{ megaqc_url }}
 megaqc_access_token: {{ megaqc_access_token }}
+{% endif %}
 
 {% for key, value in multiqc_options | dictsort %}
 {{ key }}: {{ value }}

roles/nextflow/defaults/main.yml

@@ -1,20 +1,28 @@
 java_home: /sw/comp/java/x86_64/OracleJDK_11.0.9
 nextflow_java: "{{ java_home }}"
-nextflow_version_tag: 22.10.6
-nextflow_download_url: https://github.com/nextflow-io/nextflow/releases/download/v{{ nextflow_version_tag }}/nextflow-{{ nextflow_version_tag }}-all
+nextflow_version_tag: 24.04.1
+nextflow_download_url: https://github.com/nextflow-io/nextflow/releases/download/v{{ nextflow_version_tag }}/nextflow
+nf_validation_version: 1.1.2
 nextflow_local_env:
   NXF_HOME: "{{ nextflow_dest }}/workfiles"
   NXF_OPTS: -Xms1g -Xmx3500m
   NXF_JAVA_HOME: "{{ nextflow_java }}"
+  NXF_PLUGINS_DIR: "{{ nextflow_dest }}/workfiles/plugins"
   PATH: "{{ tools_path.PATH }}"
 nextflow_env:
+  NXF_HOME: "{{ nextflow_dest }}/workfiles"
   NXF_LAUNCHER: "$PWD/work"
   NXF_TEMP: "{{ ngi_pipeline_nobackup }}/nextflow/temp"
   NXF_WORK: "{{ ngi_pipeline_nobackup }}/nextflow/work"
   NXF_OPTS: -Xms1g -Xmx3500m
   NXF_JAVA_HOME: "{{ nextflow_java }}"
+  NXF_PLUGINS_DIR: "{{ nextflow_dest }}/workfiles/plugins"
   NXF_SINGULARITY_HOME_MOUNT: true
   PATH: "{{ tools_path.PATH }}"
 nextflow_plugins:
   - name: nf-validation
-    version: 1.1.0
+    version: "{{ nf_validation_version }}"
+  - name: nf-prov
+    version: 1.2.1
+  - name: nf-tower
+    version: 1.6.3

roles/nextflow/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 # The steps within the comments can be effectively replaced by 'module load Nextflow'
 # Currently the module has problems being automatically initialized
 ###
@@ -38,6 +37,14 @@
     line: "export PATH={{ nextflow_dest }}:$PATH"
     backup: no
 
+- name: Download Nextflow plugins
+  command: "./nextflow plugin install {{ item.name }}@{{ item.version }}"
+  environment: "{{ nextflow_local_env }}"
+  with_items: "{{ nextflow_plugins }}"
+  args:
+    chdir: "{{ nextflow_dest }}"
+  tags: plugins
+
 - name: Create miarka config
   template:
     src: "nextflow_miarka_site.config.j2"
@@ -53,9 +60,12 @@
   with_items:
     - export NXF_LAUNCHER="{{ nextflow_env.NXF_LAUNCHER }}"
     - export NXF_OPTS='{{ nextflow_env.NXF_OPTS }}'
-  # Path to java to be used for nextflow
+    # Path to java to be used for nextflow
     - export NXF_JAVA_HOME={{ nextflow_env.NXF_JAVA_HOME }}
     - export NXF_SINGULARITY_HOME_MOUNT={{ nextflow_env.NXF_SINGULARITY_HOME_MOUNT }}
+    - export NXF_PLUGINS_DIR={{ nextflow_env.NXF_PLUGINS_DIR }}
+    - export NXF_HOME={{ nextflow_env.NXF_HOME }}
+    - export NXF_OFFLINE='true'
 
 - name: Store nextflow version in deployment
   lineinfile:

roles/nf-core/defaults/main.yml

@@ -1,14 +1,16 @@
 nf_core_env_name: nf-core-env
 nf_core_env: "{{ sw_path }}/anaconda/envs/{{ nf_core_env_name }}"
-nf_core_version: 2.13.1
+nf_core_version: 2.14.1
 nf_core_container_repo: docker://nfcore
 nf_core_vars:
   NFCORE_NO_VERSION_CHECK: 1
-  SINGULARITY_CACHEDIR: /scratch/{{ deployment_version }}/singularity_cache
-  SINGULARITY_TMPDIR: /scratch/{{ deployment_version }}/singularity_temp
+  APPTAINER_CACHEDIR: /scratch/{{ deployment_version }}/singularity_cache
+  APPTAINER_TMPDIR: /scratch/{{ deployment_version }}/singularity_temp
   NXF_SINGULARITY_CACHEDIR: "{{ container_dir }}"
   NXF_SINGULARITY_LIBRARYDIR: "{{ container_dir }}"
   NXF_SINGULARITY_HOME_MOUNT: true
+  NXF_PLUGINS_DIR: "{{ nextflow_local_env.NXF_PLUGINS_DIR }}"
+  NXF_DISABLE_CHECK_LATEST: true
   NXF_HOME: "{{ nextflow_local_env.NXF_HOME }}"
   JAVA_HOME: "{{ nextflow_local_env.NXF_JAVA_HOME }}"
   PATH: "{{ nf_core_env }}/bin:{{ tools_path.PATH }}"
@@ -21,20 +23,31 @@ pipelines:
     release: "{{ sarek_tag }}"
     container_dir: "{{ biocontainers_dirname }}"
     pipeline_specific_containers:
-      - sarekvep
-      - sareksnpeff
+      - vep-108.2
+      - snpeff-5.1
     pipeline_genomes:
       - GRCh38
       - GRCh37
       - GRCm38
+      - GRCm39
       - CanFam3.1
+      - WBcel235
   - name: methylseq
     release: 2.3.0
     container_dir: "{{ biocontainers_dirname }}"
+    extra_parameters:
+      references:
+        - name: pUC19
+          accession: L09137.2
+          sha1: 059f13457cb9469cbfb35755197170c99ae96a97
+        - name: lambda
+          accession: NC_001416.1
+          sha1: 2b03cbf38a8e6816914b172f048a2fe896317df7
   - name: ampliseq
     release: 2.5.0
-    silva_base_url: https://www.arb-silva.de/fileadmin/silva_databases/qiime
-    silva_zip: Silva_132_release.zip
+    extra_parameters:
+      silva_base_url: https://www.arb-silva.de/fileadmin/silva_databases/qiime
+      silva_zip: Silva_132_release.zip
     container_dir: "{{ biocontainers_dirname }}"
   - name: atacseq
     release: 2.0
@@ -66,7 +79,7 @@ nf_core_delivery_readmes:
   methylseq:
     - DELIVERY.README.METHYLSEQ.md
 
-awscli_version: 1.27.5
+awscli_version: 2.15.57
 aws_igenomes_repo: https://github.com/ewels/AWS-iGenomes.git
 aws_igenomes_commit: dda1d928fdb0d018aabbf91ac6ce8e153b699991
 aws_igenomes_dest: "{{ sw_path }}/AWS-iGenomes"
@@ -75,3 +88,5 @@ igenomes:
     source: GATK
     build: GRCh38
     type: gatk
+
+ncbi_eutils_url: https://eutils.ncbi.nlm.nih.gov/entrez/eutils

roles/nf-core/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - { role: setup_base_config, tags: setup_base_config }
   - { role: anaconda, tags: anaconda }
   - { role: nextflow, tags: nextflow }
+  - { role: taca, tags: taca }

roles/nf-core/tasks/ampliseq_include.yml

@@ -0,0 +1,20 @@
+
+
+- name: create directory for {{ pipeline }}  resources
+  file:
+    path: "{{ ngi_containers }}/{{ pipeline }}"
+    state: directory
+
+- name: check if SILVA zip exists
+  stat:
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+    path: "{{ ngi_containers }}/{{ pipeline }}/{{ extra_parameters.silva_zip }}"
+  register: silva_stat
+
+- name: download SILVA classifier data
+  get_url:
+    url: "{{ extra_parameters.silva_base_url }}/{{ extra_parameters.silva_zip }}"
+    dest: "{{ ngi_containers }}/{{ pipeline }}/{{ extra_parameters.silva_zip }}"
+  when: not silva_stat.stat.exists

roles/nf-core/tasks/main.yml

@@ -15,8 +15,7 @@
     release: "{{ item.release }}"
     pipeline_specific_containers: "{{ item.pipeline_specific_containers | default([]) }}"
     pipeline_genomes: "{{ item.pipeline_genomes | default([]) }}"
-    silva_base_url: "{{ item.silva_base_url | default('') }}"
-    silva_zip: "{{ item.silva_zip | default('') }}"
+    extra_parameters: "{{ item.extra_parameters | default({}) }}"
     container_dir: "{{ ngi_containers }}/{{ item.container_dir | default(item.name) }}"
   with_items: "{{ pipelines }}"
   tags: pipelines

roles/nf-core/tasks/methylseq_include.yml

@@ -0,0 +1,44 @@
+
+- name: create directory for {{ pipeline }}  resources
+  file:
+    path: "{{ ngi_containers }}/{{ pipeline }}"
+    state: directory
+
+# the eutils service will only tolerate 3 requests / second so make sure not to exceed this
+- name: download FASTA references
+  get_url:
+    url: "{{ ncbi_eutils_url }}/efetch.fcgi?db=nuccore&rettype=fasta&retmode=text&id={{ reference.accession }}"
+    dest: "{{ ngi_containers }}/{{ pipeline }}/{{ reference.name }}_{{ reference.accession }}.fa"
+    checksum: "sha1:{{ reference.sha1 }}"
+  with_items: "{{ extra_parameters.references }}"
+  loop_control:
+    loop_var: reference
+
+# add methylseq-specific options to pipeline nextflow config
+- name: "add specific parameters to {{ site }} {{ pipeline }} nextflow config - 1"
+  lineinfile:
+    dest: "{{ ngi_pipeline_conf }}/{{ pipeline }}_{{ site }}.config"
+    line: |
+      params { 
+        genomes {
+    backup: no
+
+- name: "add specific parameters to {{ site }} {{ pipeline }} nextflow config - 2"
+  lineinfile:
+    dest: "{{ ngi_pipeline_conf }}/{{ pipeline }}_{{ site }}.config"
+    line: |
+      '{{ reference.name }}' { 
+        fasta = "{{ ngi_containers }}/{{ pipeline }}/{{ reference.name }}_{{ reference.accession }}.fa"
+      }
+    backup: no
+  with_items: "{{ extra_parameters.references }}"
+  loop_control:
+    loop_var: reference
+
+- name: "add specific parameters to {{ site }} {{ pipeline }} nextflow config - 3"
+  lineinfile:
+    dest: "{{ ngi_pipeline_conf }}/{{ pipeline }}_{{ site }}.config"
+    line: |
+      {{ '  }' }}
+      {{ '}' }}
+    backup: no