looking for conflicts in atac.wdl

Merge branch 'develop' into aa-peakcall
broadinstitute · Dec 13, 2024 · 7b5a87e · 7b5a87e
2 parents db59231 + 145b6bf
commit 7b5a87e
Show file tree

Hide file tree

Showing 107 changed files with 1,306 additions and 458 deletions.
diff --git a/beta-pipelines/skylab/slidetags/SlideTags.wdl b/beta-pipelines/skylab/slidetags/SlideTags.wdl
@@ -1,6 +1,7 @@
 version 1.0
 
 import "scripts/spatial-count.wdl" as SpatialCount
+import "scripts/positioning.wdl" as Positioning
 
 workflow SlideTags {
 
@@ -10,25 +11,28 @@ workflow SlideTags {
         String id
         Array[String] fastq_paths
         Array[String] pucks
-        Int mem_GiB = 64
-        Int disk_GiB = 128
-        String docker = "us.gcr.io/broad-gotc-prod/slide-tags:1.0.0"
+        Array[String] rna_paths
+        String sb_path
+        String docker = "us.gcr.io/broad-gotc-prod/slide-tags:1.1.0"
      }
 
     parameter_meta {
         fastq_paths: "Array of paths to spatial fastq files"
         pucks: "Array of paths to puck files"
-        mem_GiB: "Memory in GiB to allocate to the task"
-        disk_GiB: "Disk in GiB to allocate to the task"
         docker: "Docker image to use"
     }
 
     call SpatialCount.count as spatial_count {
         input:
             fastq_paths = fastq_paths,
             pucks = pucks,
-            mem_GiB = mem_GiB,
-            disk_GiB = disk_GiB,
+            docker = docker
+     }
+
+    call Positioning.generate_positioning as positioning {
+        input:
+            rna_paths = rna_paths,
+            sb_path = spatial_count.sb_counts,
             docker = docker
      }
 

diff --git a/beta-pipelines/skylab/slidetags/scripts/CREDITS.md b/beta-pipelines/skylab/slidetags/scripts/CREDITS.md
@@ -6,6 +6,10 @@ This project uses code from the following sources:
   URL: [https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/spatial-count/spatial-count.wdl](https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/spatial-count/spatial-count.wdl)  
   This code was adapted and modified from the Macosko Lab's pipeline repository.
 
+- **Positioning Workflow**
+  URL: [https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/positioning/positioning.wdl](https://github.com/MacoskoLab/Macosko-Pipelines/blob/main/positioning/positioning.wdl)  
+  This code was adapted and modified from the Macosko Lab's pipeline repository.
+
 Additional modifications include output handling and script download changes.
 
 Please refer to the original source for the full context of the workflow.
diff --git a/beta-pipelines/skylab/slidetags/scripts/positioning.wdl b/beta-pipelines/skylab/slidetags/scripts/positioning.wdl
@@ -0,0 +1,90 @@
+version 1.0
+
+task generate_positioning {
+  input {
+    Array[String] rna_paths
+    String sb_path
+    Int mem_GiB  = 128
+    Int disk_GiB = 128
+    Int nthreads = 16
+    String docker
+  }
+  command <<<
+    set -euo pipefail
+    set -x
+    echo "<< starting spatial-count >>"
+
+    gcloud config set storage/process_count 16 # is this set by user?
+    gcloud config set storage/thread_count  2 # is this set by user?
+
+    # Download the scripts -- these need to be changed -- also need to add to docker   
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/ee005109446f58764509ee47ff51c212ce8dabe3/positioning/positioning.R
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/6a78716aa08a9f2506c06844f7e3fd491b03aa8b/positioning/load_matrix.R
+    wget https://raw.githubusercontent.com/MacoskoLab/Macosko-Pipelines/a7fc86abbdd3d46461c500e7d024315d88a97e9a/positioning/run-positioning.R
+
+    echo "RNA: ~{sep=' ' rna_paths}"
+    echo "SB: ~{sb_path}"
+
+    # Assert that the RNA files exist
+    rnas=(~{sep=' ' rna_paths})
+    for rna in "${rnas[@]}" ; do
+        if ! gsutil stat "$rna" &> /dev/null ; then
+            echo "ERROR: gsutil stat command failed on file $rna"
+            exit 1
+        fi
+    done
+
+    # Download the RNA
+    echo "Downloading RNA:"
+    mkdir RNA
+    gcloud storage cp ~{sep=' ' rna_paths} RNA
+
+    # Assert that the SB file exists
+    if ! gsutil stat "~{sb_path}" &> /dev/null ; then
+        echo "ERROR: gsutil stat command failed on file ~{sb_path}"
+        exit 1
+    fi
+
+    # Download the SB
+    echo "Downloading SB:"
+    mkdir SB
+    gcloud storage cp ~{sb_path} SB
+
+    # Run the script
+    echo ; echo "Running run-positioning.R"
+    Rscript run-positioning.R RNA SB output
+
+    # Upload the results
+    ls output/* 
+
+    if [[ -f output/seurat.qs ]] ; then
+        echo "true" > DONE
+    else
+        echo ; echo "ERROR: CANNOT FIND: seurat.qs"
+    fi
+
+    echo; echo "Writing logs:"
+    echo; echo "RNA size:"; du -sh RNA
+    echo; echo "SB size:"; du -sh SB
+    echo; echo "output size:"; du -sh output
+    echo; echo "FREE SPACE:"; df -h
+
+    echo "tar files/logs"
+    cat stdout stderr > positioning.log
+    tar -zcvf output.tar.gz output
+    echo "<< completed positioning >>"
+  >>>
+
+  output {
+    File output_file = "output.tar.gz"
+    File positioning_log = "positioning.log"
+  }
+
+  runtime {
+    docker: docker
+    memory: "~{mem_GiB} GB"
+    disks: "local-disk ~{disk_GiB} SSD"
+    cpu: nthreads
+  }
+
+}
diff --git a/beta-pipelines/skylab/slidetags/scripts/spatial-count.wdl b/beta-pipelines/skylab/slidetags/scripts/spatial-count.wdl
@@ -4,8 +4,9 @@ task count {
   input {
     Array[String] fastq_paths
     Array[String] pucks
-    Int mem_GiB
-    Int disk_GiB
+    Int mem_GiB = 64
+    Int disk_GiB = 128
+    Int nthreads = 1
     String docker
   }
   command <<<
@@ -74,7 +75,6 @@ task count {
   >>>
 
   output {
-    Boolean DONE = read_boolean("DONE")
     File sb_counts = "SBcounts.h5"
     File spatial_log = "spatial-count.log"
 
@@ -83,8 +83,7 @@ task count {
     docker: docker
     memory: "~{mem_GiB} GB"
     disks: "local-disk ~{disk_GiB} SSD"
-    cpu: 1
-    preemptible: 0
+    cpu: nthreads
   }
 }
 
diff --git a/deprecated/pipelines/cemba/cemba_methylcseq/README.md b/deprecated/pipelines/cemba/cemba_methylcseq/README.md
@@ -1,6 +1,8 @@
 ## Announcement: CEMBA is Deprecated 9/12/2024
 
-The CEMBA workflow is deprecated and is no longer supported. However, the CEMBA documentation is still available. See [CEMBA Pipeline Overview](https://broadinstitute.github.io/warp/docs/Pipelines/CEMBA_MethylC_Seq_Pipeline/README) on the [WARP documentation site](https://broadinstitute.github.io/warp/)!
+The CEMBA workflow is deprecated and no longer supported. File paths in the JSON file are also no longer supported.
+
+However, the CEMBA documentation is still available. See [CEMBA Pipeline Overview](https://broadinstitute.github.io/warp/docs/Pipelines/CEMBA_MethylC_Seq_Pipeline/README) on the [WARP documentation site](https://broadinstitute.github.io/warp/)! The CEMBA data is also available on the NEMO data portal. The whitelist includes the following barcodes: CTCACG, CAGATC, CGATGT, ACTTGA, TTAGGC, GATCAG, TGACCA, TAGCTT, ACAGTG, GGCTAC, GCCAAT, CTTGTA.
 
 ### CEMBA summary
 

diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -1,40 +1,40 @@
 Pipeline Name	Version	Date of Last Commit
 CheckFingerprint	 1.0.22	2024-10-28 
-RNAWithUMIsPipeline	 1.0.17	2024-09-06 
-AnnotationFiltration	 1.2.6	2024-09-06 
-UltimaGenomicsWholeGenomeGermline	 1.1.1	2024-10-28 
-WholeGenomeGermlineSingleSample	 3.3.2	2024-10-28 
-ExomeGermlineSingleSample	 3.2.2	2024-10-28 
-JointGenotypingByChromosomePartTwo	 1.5.1	2024-09-10 
-JointGenotypingByChromosomePartOne	 1.5.1	2024-09-10 
-ReblockGVCF	 2.3.1	2024-10-28 
-JointGenotyping	 1.7.1	2024-09-10 
-UltimaGenomicsJointGenotyping	 1.2.1	2024-09-10 
-VariantCalling	 2.2.3	2024-10-28 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.22	2024-10-28 
-GDCWholeGenomeSomaticSingleSample	 1.3.3	2024-09-06 
-BroadInternalRNAWithUMIs	 1.0.35	2024-09-06 
-BroadInternalUltimaGenomics	 1.1.1	2024-10-28 
-BroadInternalArrays	 1.1.13	2024-10-28 
-BroadInternalImputation	 1.1.13	2024-09-06 
-Arrays	 2.6.29	2024-10-28 
-ValidateChip	 1.16.6	2024-09-06 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
+UltimaGenomicsWholeGenomeGermline	 1.1.2	2024-11-04 
+WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
+ReblockGVCF	 2.3.2	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
+UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
+VariantCalling	 2.2.4	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
+BroadInternalUltimaGenomics	 1.1.2	2024-11-04 
+BroadInternalArrays	 1.1.14	2024-11-04 
+BroadInternalImputation	 1.1.14	2024-11-04 
+Arrays	 2.6.30	2024-11-04 
+ValidateChip	 1.16.7	2024-11-04 
 MultiSampleArrays	 1.6.2	2024-08-02 
-Imputation	 1.1.14	2024-09-06 
-IlluminaGenotypingArray	 1.12.23	2024-10-28 
-ExternalWholeGenomeReprocessing	 2.3.2	2024-10-28 
-ExternalExomeReprocessing	 3.3.2	2024-10-28 
+Imputation	 1.1.15	2024-11-04 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
+ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
 CramToUnmappedBams	 1.1.3	2024-08-02 
-WholeGenomeReprocessing	 3.3.2	2024-10-28 
-ExomeReprocessing	 3.3.2	2024-10-28 
-BuildIndices	 3.0.0	2023-12-06 
+WholeGenomeReprocessing	 3.3.3	2024-11-04 
+ExomeReprocessing	 3.3.3	2024-11-04 
+BuildIndices	 3.1.0	2024-11-26 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Multiome	 5.8.0	2024-10-23 
-PairedTag	 1.8.0	2024-10-23 
+Multiome	 5.9.3	2024-12-3 
+PairedTag	 1.8.4	2024-12-3 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-MultiSampleSmartSeq2SingleNucleus	 2.0.2	2024-10-23 
-Optimus	 7.8.0	2024-10-23 
-atac	 2.4.0	2024-10-23 
+MultiSampleSmartSeq2SingleNucleus	 2.0.6	2024-11-15 
+Optimus	 7.8.4	2024-12-3 
+atac	 2.5.3	2024-11-22 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 
-SlideSeq	 3.4.3	2024-10-24 
+SlideSeq	 3.4.7	2024-12-3 
diff --git a/pipelines/broad/annotation_filtration/AnnotationFiltration.changelog.md b/pipelines/broad/annotation_filtration/AnnotationFiltration.changelog.md
@@ -1,3 +1,8 @@
+# 1.2.7
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 1.2.6
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/annotation_filtration/AnnotationFiltration.wdl b/pipelines/broad/annotation_filtration/AnnotationFiltration.wdl
@@ -4,7 +4,7 @@ import "../../../tasks/broad/Funcotator.wdl" as Funcotator
 
 workflow AnnotationFiltration {
 
-  String pipeline_version = "1.2.6"
+  String pipeline_version = "1.2.7"
 
   input {
     Array[File] vcfs
@@ -15,7 +15,7 @@ workflow AnnotationFiltration {
     File ref_dict
     File? funcotator_interval_list
 
-    String gatk_docker = "us.gcr.io/broad-gatk/gatk:4.6.0.0"
+    String gatk_docker = "us.gcr.io/broad-gatk/gatk:4.6.1.0"
     File? custom_data_source_tar_gz
   }
 

diff --git a/pipelines/broad/annotation_filtration/test_inputs/Plumbing/hg38.json b/pipelines/broad/annotation_filtration/test_inputs/Plumbing/hg38.json
@@ -10,6 +10,6 @@
   "AnnotationFiltration.ref_fasta_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.fai",
   "AnnotationFiltration.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
 
-  "AnnotationFiltration.gatk_docker": "us.gcr.io/broad-gatk/gatk:4.5.0.0",
+  "AnnotationFiltration.gatk_docker": "us.gcr.io/broad-gatk/gatk:4.6.1.0",
   "AnnotationFiltration.custom_data_source_tar_gz": "gs://broad-public-datasets/funcotator/funcotator_dataSources.v1.6.20190124g.tar.gz"
 }
diff --git a/pipelines/broad/annotation_filtration/test_inputs/Scientific/hg38.json b/pipelines/broad/annotation_filtration/test_inputs/Scientific/hg38.json
@@ -8,6 +8,6 @@
   "AnnotationFiltration.ref_fasta_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.fai",
   "AnnotationFiltration.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
 
-  "AnnotationFiltration.gatk_docker": "us.gcr.io/broad-gatk/gatk:4.5.0.0",
+  "AnnotationFiltration.gatk_docker": "us.gcr.io/broad-gatk/gatk:4.6.1.0",
   "AnnotationFiltration.custom_data_source_tar_gz": "gs://broad-public-datasets/funcotator/funcotator_dataSources.v1.6.20190124g.tar.gz"
 }
diff --git a/pipelines/broad/arrays/imputation/Imputation.changelog.md b/pipelines/broad/arrays/imputation/Imputation.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.15
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 1.1.14
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/arrays/imputation/Imputation.wdl b/pipelines/broad/arrays/imputation/Imputation.wdl
@@ -6,7 +6,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Imputation {
 
-  String pipeline_version = "1.1.14"
+  String pipeline_version = "1.1.15"
 
   input {
     Int chunkLength = 25000000

diff --git a/pipelines/broad/arrays/single_sample/Arrays.changelog.md b/pipelines/broad/arrays/single_sample/Arrays.changelog.md
@@ -1,3 +1,8 @@
+# 2.6.30
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 2.6.29
 2024-10-28 (Date of Last Commit)
 

diff --git a/pipelines/broad/arrays/single_sample/Arrays.wdl b/pipelines/broad/arrays/single_sample/Arrays.wdl
@@ -23,7 +23,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Arrays {
 
-  String pipeline_version = "2.6.29"
+  String pipeline_version = "2.6.30"
 
   input {
     String chip_well_barcode

diff --git a/pipelines/broad/arrays/validate_chip/ValidateChip.changelog.md b/pipelines/broad/arrays/validate_chip/ValidateChip.changelog.md
@@ -1,3 +1,8 @@
+# 1.16.7
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 1.16.6
 2024-09-06 (Date of Last Commit)
 

diff --git a/pipelines/broad/arrays/validate_chip/ValidateChip.wdl b/pipelines/broad/arrays/validate_chip/ValidateChip.wdl
@@ -21,7 +21,7 @@ import "../../../../tasks/broad/InternalArraysTasks.wdl" as InternalTasks
 
 workflow ValidateChip {
 
-  String pipeline_version = "1.16.6"
+  String pipeline_version = "1.16.7"
 
   input {
     String sample_alias

diff --git a/pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.changelog.md b/pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.changelog.md
@@ -1,3 +1,8 @@
+# 1.7.2
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 1.7.1
 2024-09-10 (Date of Last Commit)
 

diff --git a/pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.wdl b/pipelines/broad/dna_seq/germline/joint_genotyping/JointGenotyping.wdl
@@ -7,7 +7,7 @@ import "https://raw.githubusercontent.com/broadinstitute/gatk/4.5.0.0/scripts/vc
 # Joint Genotyping for hg38 Whole Genomes and Exomes (has not been tested on hg19)
 workflow JointGenotyping {
 
-  String pipeline_version = "1.7.1"
+  String pipeline_version = "1.7.2"
 
   input {
     File unpadded_intervals_file
@@ -214,7 +214,7 @@ workflow JointGenotyping {
         extract_extra_args = extract_extra_args,
         resource_args = resource_args,
         output_prefix = callset_name,
-        gatk_docker = "us.gcr.io/broad-gatk/gatk:4.6.0.0"
+        gatk_docker = "us.gcr.io/broad-gatk/gatk:4.6.1.0"
     }
   } 
   if (!run_vets) {

diff --git a/...line/joint_genotyping/UltimaGenomics/UltimaGenomicsJointGenotyping.changelog.md b/...line/joint_genotyping/UltimaGenomics/UltimaGenomicsJointGenotyping.changelog.md
@@ -1,3 +1,8 @@
+# 1.2.2
+2024-11-04 (Date of Last Commit)
+
+* Updated to GATK version 4.6.1.0
+
 # 1.2.1
 2024-09-10 (Date of Last Commit)