From 08fd34a1d656bb0bbd1e44edd265da16eef6d598 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 5 Mar 2024 11:56:15 +0100 Subject: [PATCH 01/28] Prepare release 5.8.2 --- .../cloud/docker/cellbase-builder/Dockerfile | 2 +- .../app/scripts/ensembl-scripts/DB_CONFIG.pm | 8 +- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- .../core/config/DownloadProperties.java | 20 ----- .../src/main/resources/configuration.yml | 74 +++++-------------- cellbase-lib/pom.xml | 6 +- .../org/opencb/cellbase/lib/EtlCommons.java | 1 - .../cellbase/lib/builders/GeneBuilder.java | 4 +- .../lib/builders/OntologyBuilder.java | 9 --- .../lib/download/GeneDownloadManager.java | 26 ------- .../lib/download/GenomeDownloadManager.java | 18 ++--- .../lib/download/OntologyDownloadManager.java | 10 +-- .../lib/download/ProteinDownloadManager.java | 34 ++++----- .../download/RegulationDownloadManager.java | 2 +- .../lib/impl/core/MetaMongoDBAdaptor.java | 3 +- cellbase-server/pom.xml | 2 +- pom.xml | 21 +++--- 19 files changed, 73 insertions(+), 173 deletions(-) diff --git a/cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile b/cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile index 6e1657d1bf..17d5accff4 100644 --- a/cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile +++ b/cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile @@ -28,4 +28,4 @@ RUN cd /opt/ensembl && \ git clone https://github.com/Ensembl/ensembl-compara.git && \ git clone https://github.com/Ensembl/ensembl-io.git -ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase/scripts/ensembl-scripts +ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase diff --git a/cellbase-app/app/scripts/ensembl-scripts/DB_CONFIG.pm b/cellbase-app/app/scripts/ensembl-scripts/DB_CONFIG.pm index 70865465e9..aa22cf10b1 100755 --- a/cellbase-app/app/scripts/ensembl-scripts/DB_CONFIG.pm +++ b/cellbase-app/app/scripts/ensembl-scripts/DB_CONFIG.pm @@ -134,10 +134,10 @@ our $ENSEMBL_GENOMES_PORT = "4157"; our $ENSEMBL_GENOMES_USER = "anonymous"; ## Vertebrates -our $HOMO_SAPIENS_CORE = "homo_sapiens_core_110_38"; -our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_110_38"; -our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_110_38"; -our $HOMO_SAPIENS_COMPARA = "homo_sapiens_compara_110_38"; +our $HOMO_SAPIENS_CORE = "homo_sapiens_core_104_38"; +our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_104_38"; +our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_104_38"; +our $HOMO_SAPIENS_COMPARA = "homo_sapiens_compara_104_38"; #our $HOMO_SAPIENS_CORE = "homo_sapiens_core_78_38"; #our $HOMO_SAPIENS_VARIATION = "homo_sapiens_variation_78_38"; #our $HOMO_SAPIENS_FUNCTIONAL = "homo_sapiens_funcgen_78_38"; diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index d7b8c8a47e..289046b4cc 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 566e1b40e8..ce235ddfa6 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index b628de3b21..3e68b37513 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 ../pom.xml diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/DownloadProperties.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/DownloadProperties.java index 19f1606c91..ee4216f560 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/DownloadProperties.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/DownloadProperties.java @@ -26,7 +26,6 @@ public class DownloadProperties { private EnsemblProperties ensembl; private EnsemblProperties ensemblGenomes; private URLProperties hgnc; - private URLProperties cancerHotspot; private URLProperties refSeq; private URLProperties refSeqFasta; private URLProperties refSeqProteinFasta; @@ -71,7 +70,6 @@ public class DownloadProperties { private URLProperties hpoObo; private URLProperties goObo; private URLProperties doidObo; - private URLProperties mondoObo; private URLProperties goAnnotation; private URLProperties revel; private URLProperties pubmed; @@ -519,24 +517,6 @@ public DownloadProperties setHgnc(URLProperties hgnc) { return this; } - public URLProperties getCancerHotspot() { - return cancerHotspot; - } - - public DownloadProperties setCancerHotspot(URLProperties cancerHotspot) { - this.cancerHotspot = cancerHotspot; - return this; - } - - public URLProperties getMondoObo() { - return mondoObo; - } - - public DownloadProperties setMondoObo(URLProperties mondoObo) { - this.mondoObo = mondoObo; - return this; - } - public static class EnsemblProperties { private DatabaseCredentials database; diff --git a/cellbase-core/src/main/resources/configuration.yml b/cellbase-core/src/main/resources/configuration.yml index f24827532c..0f8d199118 100644 --- a/cellbase-core/src/main/resources/configuration.yml +++ b/cellbase-core/src/main/resources/configuration.yml @@ -62,11 +62,7 @@ download: url: host: ftp://ftp.ensemblgenomes.org/pub hgnc: - host: https://ftp.ebi.ac.uk/pub/databases/genenames/hgnc/archive/monthly/tsv/hgnc_complete_set_2023-11-01.txt - version: 2023-11-01 - cancerHotspot: - host: https://www.cancerhotspots.org/files/hotspots_v2.xls - version: "v2" + host: https://ftp.ebi.ac.uk/pub/databases/genenames/hgnc/archive/monthly/tsv/hgnc_complete_set_2022-01-01.txt refSeq: host: https://ftp.ncbi.nih.gov/refseq/H_sapiens/annotation/GRCh38_latest/refseq_identifiers/GRCh38_latest_genomic.gtf.gz refSeqFasta: @@ -77,15 +73,12 @@ download: host: https://ftp.ncbi.nih.gov/refseq/H_sapiens/annotation/GRCh38_latest/refseq_identifiers/GRCh38_latest_rna.fna.gz maneSelect: # host: https://ftp.ncbi.nlm.nih.gov/refseq/MANE/MANE_human/release_0.93/MANE.GRCh38.v0.93.summary.txt.gz -# host: https://ftp.ncbi.nlm.nih.gov/refseq/MANE/MANE_human/release_1.0/MANE.GRCh38.v1.0.summary.txt.gz - host: https://ftp.ncbi.nlm.nih.gov/refseq/MANE/MANE_human/release_1.1/MANE.GRCh38.v1.1.summary.txt.gz - version: "1.1" + host: https://ftp.ncbi.nlm.nih.gov/refseq/MANE/MANE_human/release_1.0/MANE.GRCh38.v1.0.summary.txt.gz + version: 0.93 lrg: host: http://ftp.ebi.ac.uk/pub/databases/lrgex/list_LRGs_transcripts_xrefs.txt - version: "2021-03-30" geneUniprotXref: host: http://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/ - version: "2023-11-08" geneExpressionAtlas: host: ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/gxa/allgenes_updown_in_organism_part_2.0.14.tab.gz mirbase: @@ -95,49 +88,33 @@ download: targetScan: host: http://hgdownload.cse.ucsc.edu/goldenPath/ miRTarBase: - host: https://mirtarbase.cuhk.edu.cn/~miRTarBase/miRTarBase_2022/cache/download/9.0/hsa_MTI.xlsx - version: "9.0" - - ## Protein Data + host: https://mirtarbase.cuhk.edu.cn/~miRTarBase/miRTarBase_2022/cache/download/8.0/hsa_MTI.xlsx uniprot: - host: https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.xml.gz - version: "2023-11-08" + host: ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.xml.gz uniprotRelNotes: - host: https://ftp.uniprot.org/pub/databases/uniprot/relnotes.txt - version: "2023-11-08" + host: ftp://ftp.uniprot.org/pub/databases/uniprot/relnotes.txt + intact: + host: ftp://ftp.ebi.ac.uk/pub/databases/intact/current/psimitab/intact.txt interpro: - host: https://ftp.ebi.ac.uk/pub/databases/interpro/current_release/protein2ipr.dat.gz - version: "2023-11-08" + host: ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/protein2ipr.dat.gz interproRelNotes: - host: https://ftp.ebi.ac.uk/pub/databases/interpro/current_release/release_notes.txt - intact: - host: https://ftp.ebi.ac.uk/pub/databases/intact/current/psimitab/intact.txt - version: "2023-10-07" - - ## Conservation Scores + host: ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/release_notes.txt conservation: host: https://hgdownload.cse.ucsc.edu/goldenPath/ - version: "2022-08-30" gerp: - host: http://ftp.ensembl.org/pub/release-110/compara/conservation_scores/91_mammals.gerp_conservation_score/gerp_conservation_scores.homo_sapiens.GRCh38.bw - version: "2023-05-17" + host: http://ftp.ensembl.org/pub/release-104/compara/conservation_scores/90_mammals.gerp_conservation_score/gerp_conservation_scores.homo_sapiens.GRCh38.bw clinvar: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-02.xml.gz -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2023-12.xml.gz - version: "2023-12-01" + host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz clinvarVariation: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-02.xml.gz -# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2023-12.xml.gz + host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz clinvarSummary: host: http://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz - version: "2023-12-01" clinvarVariationAllele: host: http://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variation_allele.txt.gz - version: "2023-12-01" clinvarEfoTerms: host: ftp://ftp.ebi.ac.uk/pub/databases/eva/ClinVar/2015/ClinVar_Traits_EFO_Names_260615.csv iarctp53: @@ -155,12 +132,9 @@ download: genomicSuperDups: host: http://hgdownload.cse.ucsc.edu/goldenPath gwasCatalog: -# host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv -# version: "1.0.2 associations_e106_r2022-05-17" - host: ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/2023/12/21/gwas-catalog-associations.tsv - version: "23-12-21" + host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv + version: "1.0.2 associations_e106_r2022-05-17" hpo: - ## Downlaod manually from here now: https://hpo.jax.org/app/data/annotations host: https://ci.monarchinitiative.org/view/hpo/job/hpo.annotations/lastSuccessfulBuild/artifact/rare-diseases/util/annotation/phenotype_to_genes.txt disgenet: host: https://www.disgenet.org/static/disgenet_ap1/files/downloads @@ -168,30 +142,20 @@ download: - all_gene_disease_associations.tsv.gz - readme.txt dgidb: - host: https://old.dgidb.org/data/monthly_tsvs/2022-Feb/interactions.tsv - version: "2022-02-01" + host: https://dgidb.org/data/monthly_tsvs/2021-Jan/interactions.tsv cadd: - ## Nacho: Move to https://krishna.gs.washington.edu/download/CADD/v1.7-pre/GRCh38/whole_genome_SNVs.tsv.gz ASAP! -# host: https://krishna.gs.washington.edu/download/CADD/v1.6/GRCh38/whole_genome_SNVs.tsv.gz - host: https://krishna.gs.washington.edu/download/CADD/v1.7-pre/GRCh38/whole_genome_SNVs.tsv.gz - version: "1.7-pre" + host: https://krishna.gs.washington.edu/download/CADD/v1.6/GRCh38/whole_genome_SNVs.tsv.gz reactome: host: http://www.reactome.org/download/current/biopax.zip gnomadConstraints: host: https://storage.googleapis.com/gcp-public-data--gnomad/release/2.1.1/constraint/gnomad.v2.1.1.lof_metrics.by_transcript.txt.bgz - version: "2.1.1" + version: 2.1.1 hpoObo: host: http://purl.obolibrary.org/obo/hp.obo - version: "2023-12-01" goObo: host: http://purl.obolibrary.org/obo/go/go-basic.obo - version: "2023-12-01" doidObo: host: http://purl.obolibrary.org/obo/doid.obo - version: "2023-12-01" - mondoObo: - host: http://purl.obolibrary.org/obo/mondo.obo - version: "2023-12-01" goAnnotation: host: http://geneontology.org/gene-associations/goa_human.gaf.gz revel: @@ -218,7 +182,7 @@ species: - id: hsapiens scientificName: Homo sapiens assemblies: - - ensemblVersion: '110_38' + - ensemblVersion: '104_38' name: GRCh38 - ensemblVersion: '82_37' name: GRCh37 diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index dd087fb692..ffb98718f8 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 ../pom.xml @@ -137,10 +137,10 @@ com.github.samtools htsjdk - + io.jsonwebtoken jjwt-api diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java index 124ac6e6fc..4396f0c2f1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java @@ -71,7 +71,6 @@ public class EtlCommons { public static final String HPO_FILE = "hp.obo"; public static final String GO_FILE = "go-basic.obo"; public static final String DOID_FILE = "doid.obo"; - public static final String MONDO_FILE = "mondo.obo"; public static final String PFM_DATA = "regulatory_pfm"; // Build specific data options diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/GeneBuilder.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/GeneBuilder.java index cd0863a259..563f76dea7 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/GeneBuilder.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/GeneBuilder.java @@ -90,8 +90,8 @@ public GeneBuilder(Path geneDirectoryPath, Path genomeSequenceFastaFile, Species boolean flexibleGTFParsing, CellBaseSerializer serializer) throws CellBaseException { this(null, geneDirectoryPath.resolve("description.txt"), geneDirectoryPath.resolve("xrefs.txt"), - geneDirectoryPath.resolve("hgnc_complete_set_2023-11-01.txt"), - geneDirectoryPath.resolve("MANE.GRCh38.v1.1.summary.txt.gz"), + geneDirectoryPath.resolve("hgnc_complete_set_2022-01-01.txt"), + geneDirectoryPath.resolve("MANE.GRCh38.v1.0.summary.txt.gz"), geneDirectoryPath.resolve("list_LRGs_transcripts_xrefs.txt"), geneDirectoryPath.resolve("idmapping_selected.tab.gz"), geneDirectoryPath.getParent().resolve("regulation/motif_features.gff.gz"), diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/OntologyBuilder.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/OntologyBuilder.java index 1eabf8975a..8873dd7f93 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/OntologyBuilder.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/OntologyBuilder.java @@ -32,14 +32,12 @@ public class OntologyBuilder extends CellBaseBuilder { private Path hpoFile; private Path goFile; private Path doidFile; - private Path mondoFile; public OntologyBuilder(Path oboDirectoryPath, CellBaseSerializer serializer) { super(serializer); hpoFile = oboDirectoryPath.resolve(EtlCommons.HPO_FILE); goFile = oboDirectoryPath.resolve(EtlCommons.GO_FILE); doidFile = oboDirectoryPath.resolve(EtlCommons.DOID_FILE); - mondoFile = oboDirectoryPath.resolve(EtlCommons.MONDO_FILE); } @Override @@ -66,13 +64,6 @@ public void parse() throws Exception { serializer.serialize(term); } - bufferedReader = FileUtils.newBufferedReader(mondoFile); - terms = parser.parseOBO(bufferedReader, "Mondo Ontology"); - for (OntologyTerm term : terms) { - term.setSource("MONDO"); - serializer.serialize(term); - } - serializer.close(); } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GeneDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GeneDownloadManager.java index 260ff75427..9d2685eadf 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GeneDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GeneDownloadManager.java @@ -82,8 +82,6 @@ public List download() throws IOException, InterruptedException { downloadFiles.addAll(downloadRefSeq(refseqFolder)); downloadFiles.add(downloadMane(geneFolder)); downloadFiles.add(downloadLrg(geneFolder)); - downloadFiles.add(downloadHgnc(geneFolder)); - downloadFiles.add(downloadCancerHotspot(geneFolder)); downloadFiles.add(downloadDrugData(geneFolder)); downloadFiles.addAll(downloadGeneUniprotXref(geneFolder)); downloadFiles.add(downloadGeneExpressionAtlas(geneFolder)); @@ -210,30 +208,6 @@ private DownloadFile downloadLrg(Path geneFolder) throws IOException, Interrupte return null; } - private DownloadFile downloadHgnc(Path geneFolder) throws IOException, InterruptedException { - if (speciesConfiguration.getScientificName().equals("Homo sapiens")) { - logger.info("Downloading HGNC ..."); - String url = configuration.getDownload().getHgnc().getHost(); - saveVersionData(EtlCommons.GENE_DATA, "HGNC_GENE", configuration.getDownload().getHgnc().getVersion(), - getTimeStamp(), Collections.singletonList(url), geneFolder.resolve("hgncVersion.json")); - String[] array = url.split("/"); - return downloadFile(url, geneFolder.resolve(array[array.length - 1]).toString()); - } - return null; - } - - private DownloadFile downloadCancerHotspot(Path geneFolder) throws IOException, InterruptedException { - if (speciesConfiguration.getScientificName().equals("Homo sapiens")) { - logger.info("Downloading Cancer Hotspot ..."); - String url = configuration.getDownload().getCancerHotspot().getHost(); - saveVersionData(EtlCommons.GENE_DATA, "CANCER_HOTSPOT", configuration.getDownload().getHgnc().getVersion(), - getTimeStamp(), Collections.singletonList(url), geneFolder.resolve("cancerHotspotVersion.json")); - String[] array = url.split("/"); - return downloadFile(url, geneFolder.resolve(array[array.length - 1]).toString()); - } - return null; - } - private DownloadFile downloadGO(Path geneFolder) throws IOException, InterruptedException { if (speciesConfiguration.getScientificName().equals("Homo sapiens")) { logger.info("Downloading go annotation..."); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GenomeDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GenomeDownloadManager.java index 0ba9f39db4..5a0609867f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GenomeDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/GenomeDownloadManager.java @@ -47,11 +47,11 @@ public GenomeDownloadManager(String species, String assembly, Path targetDirecto public List download() throws IOException, InterruptedException { List downloadFiles = new ArrayList<>(); downloadFiles.addAll(downloadReferenceGenome()); - downloadFiles.addAll(downloadConservation()); - downloadFiles.addAll(downloadRepeats()); +// downloadFiles.addAll(downloadConservation()); +// downloadFiles.addAll(downloadRepeats()); // cytobands -// runGenomeInfo(); + runGenomeInfo(); return downloadFiles; } @@ -115,16 +115,16 @@ public List downloadConservation() throws IOException, Interrupted List phastconsUrls = new ArrayList<>(chromosomes.length); List phyloPUrls = new ArrayList<>(chromosomes.length); for (String chromosome : chromosomes) { - String phastConsUrl = url + "/phastCons470way/hg38.470way.phastCons/chr" + chromosome - + ".phastCons470way.wigFix.gz"; + String phastConsUrl = url + "/phastCons100way/hg38.100way.phastCons/chr" + chromosome + + ".phastCons100way.wigFix.gz"; downloadFiles.add(downloadFile(phastConsUrl, conservationFolder.resolve("phastCons") - .resolve("chr" + chromosome + ".phastCons470way.wigFix.gz").toString())); + .resolve("chr" + chromosome + ".phastCons100way.wigFix.gz").toString())); phastconsUrls.add(phastConsUrl); - String phyloPUrl = url + "/phyloP470way/hg38.470way.phyloP/chr" + chromosome - + ".phyloP470way.wigFix.gz"; + String phyloPUrl = url + "/phyloP100way/hg38.100way.phyloP100way/chr" + chromosome + + ".phyloP100way.wigFix.gz"; downloadFiles.add(downloadFile(phyloPUrl, conservationFolder.resolve("phylop") - .resolve("chr" + chromosome + ".phyloP470way.wigFix.gz").toString())); + .resolve("chr" + chromosome + ".phyloP100way.wigFix.gz").toString())); phyloPUrls.add(phyloPUrl); } String gerpUrl = configuration.getDownload().getGerp().getHost(); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/OntologyDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/OntologyDownloadManager.java index 522be7b27d..0776354e80 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/OntologyDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/OntologyDownloadManager.java @@ -36,7 +36,7 @@ public OntologyDownloadManager(String species, String assembly, Path targetDirec public List download() throws IOException, InterruptedException { - logger.info("Downloading OBO files ..."); + logger.info("Downloading obo files ..."); List downloadFiles = new ArrayList<>(); Path oboFolder = downloadFolder.resolve("ontology"); @@ -44,22 +44,20 @@ public List download() throws IOException, InterruptedException { String url = configuration.getDownload().getHpoObo().getHost(); downloadFiles.add(downloadFile(url, oboFolder.resolve("hp.obo").toString())); + saveVersionData(EtlCommons.OBO_DATA, "HPO", getTimeStamp(), getTimeStamp(), Collections.singletonList(url), buildFolder.resolve(EtlCommons.HPO_VERSION_FILE)); url = configuration.getDownload().getGoObo().getHost(); downloadFiles.add(downloadFile(url, oboFolder.resolve("go-basic.obo").toString())); + saveVersionData(EtlCommons.OBO_DATA, "GO", getTimeStamp(), getTimeStamp(), Collections.singletonList(url), buildFolder.resolve(EtlCommons.GO_VERSION_FILE)); url = configuration.getDownload().getDoidObo().getHost(); downloadFiles.add(downloadFile(url, oboFolder.resolve("doid.obo").toString())); - saveVersionData(EtlCommons.OBO_DATA, "DO", getTimeStamp(), getTimeStamp(), - Collections.singletonList(url), buildFolder.resolve(EtlCommons.DO_VERSION_FILE)); - url = configuration.getDownload().getMondoObo().getHost(); - downloadFiles.add(downloadFile(url, oboFolder.resolve("mondo.obo").toString())); - saveVersionData(EtlCommons.OBO_DATA, "MONDO", getTimeStamp(), getTimeStamp(), + saveVersionData(EtlCommons.OBO_DATA, "DO", getTimeStamp(), getTimeStamp(), Collections.singletonList(url), buildFolder.resolve(EtlCommons.DO_VERSION_FILE)); return downloadFiles; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ProteinDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ProteinDownloadManager.java index 5a722ed448..08f28cfdad 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ProteinDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ProteinDownloadManager.java @@ -22,6 +22,7 @@ import org.opencb.commons.utils.FileUtils; import java.io.BufferedReader; +import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.nio.file.Files; @@ -33,8 +34,6 @@ public class ProteinDownloadManager extends AbstractDownloadManager { private static final String UNIPROT_NAME = "UniProt"; - private static final String INTERPRO_NAME = "InterPro"; - private static final String INTACT_NAME = "IntAct"; public ProteinDownloadManager(String species, String assembly, Path targetDirectory, CellBaseConfiguration configuration) throws IOException, CellBaseException { @@ -57,7 +56,6 @@ public List download() throws IOException, InterruptedException { Files.createDirectories(proteinFolder); List downloadFiles = new ArrayList<>(); - // Uniprot String url = configuration.getDownload().getUniprot().getHost(); downloadFiles.add(downloadFile(url, proteinFolder.resolve("uniprot_sprot.xml.gz").toString())); Files.createDirectories(proteinFolder.resolve("uniprot_chunks")); @@ -65,25 +63,23 @@ public List download() throws IOException, InterruptedException { String relNotesUrl = configuration.getDownload().getUniprotRelNotes().getHost(); downloadFiles.add(downloadFile(relNotesUrl, proteinFolder.resolve("uniprotRelnotes.txt").toString())); + saveVersionData(EtlCommons.PROTEIN_DATA, UNIPROT_NAME, getLine(proteinFolder.resolve("uniprotRelnotes.txt"), 1), getTimeStamp(), Collections.singletonList(url), proteinFolder.resolve("uniprotVersion.json")); - // Interpro - String interproUrl = configuration.getDownload().getInterpro().getHost(); - downloadFiles.add(downloadFile(interproUrl, proteinFolder.resolve("protein2ipr.dat.gz").toString())); - - relNotesUrl = configuration.getDownload().getInterproRelNotes().getHost(); - downloadFiles.add(downloadFile(relNotesUrl, proteinFolder.resolve("interproRelnotes.txt").toString())); - saveVersionData(EtlCommons.PROTEIN_DATA, INTERPRO_NAME, getLine(proteinFolder.resolve("interproRelnotes.txt"), 5), - getTimeStamp(), Collections.singletonList(interproUrl), proteinFolder.resolve("interproVersion.json")); - - // Intact - String intactUrl = configuration.getDownload().getIntact().getHost(); - downloadFiles.add(downloadFile(intactUrl, proteinFolder.resolve("intact.txt").toString())); - saveVersionData(EtlCommons.PROTEIN_DATA, INTACT_NAME, configuration.getDownload().getIntact().getVersion(), - getTimeStamp(), Collections.singletonList(intactUrl), proteinFolder.resolve("intactVersion.json")); - return downloadFiles; + +// url = configuration.getDownload().getIntact().getHost(); +// downloadFile(url, proteinFolder.resolve("intact.txt").toString()); +// saveVersionData(EtlCommons.PROTEIN_DATA, INTACT_NAME, null, getTimeStamp(), Collections.singletonList(url), +// proteinFolder.resolve("intactVersion.json")); +// +// url = configuration.getDownload().getInterpro().getHost(); +// downloadFile(url, proteinFolder.resolve("protein2ipr.dat.gz").toString()); +// relNotesUrl = configuration.getDownload().getInterproRelNotes().getHost(); +// downloadFile(relNotesUrl, proteinFolder.resolve("interproRelnotes.txt").toString()); +// saveVersionData(EtlCommons.PROTEIN_DATA, INTERPRO_NAME, getLine(proteinFolder.resolve("interproRelnotes.txt"), 5), +// getTimeStamp(), Collections.singletonList(url), proteinFolder.resolve("interproVersion.json")); } private void splitUniprot(Path uniprotFilePath, Path splitOutdirPath) throws IOException { @@ -100,7 +96,7 @@ private void splitUniprot(Path uniprotFilePath, Path splitOutdirPath) throws IOE inEntry = true; beforeEntry = false; if (count % 10000 == 0) { - pw = new PrintWriter(Files.newOutputStream(splitOutdirPath.resolve("chunk_" + chunk + ".xml").toFile().toPath())); + pw = new PrintWriter(new FileOutputStream(splitOutdirPath.resolve("chunk_" + chunk + ".xml").toFile())); pw.println(header.toString().trim()); } count++; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/RegulationDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/RegulationDownloadManager.java index 51152e478d..1abb352fbe 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/RegulationDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/RegulationDownloadManager.java @@ -64,8 +64,8 @@ public List download() throws IOException, InterruptedException, N List downloadFiles = new ArrayList<>(); downloadFiles.addAll(downloadRegulatoryaAndMotifFeatures()); - downloadFiles.add(downloadMiRTarBase()); downloadFiles.add(downloadMirna()); + downloadFiles.add(downloadMiRTarBase()); return downloadFiles; } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java index 5d7dbc65d0..e5cd4d38cc 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java @@ -16,8 +16,6 @@ package org.opencb.cellbase.lib.impl.core; - -import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; import com.mongodb.client.model.Filters; @@ -25,6 +23,7 @@ import org.bson.BsonDocument; import org.bson.Document; import org.bson.conversions.Bson; +import org.codehaus.jackson.map.ObjectMapper; import org.opencb.cellbase.core.api.key.ApiKeyStats; import org.opencb.cellbase.core.api.query.AbstractQuery; import org.opencb.cellbase.core.api.query.ProjectionQueryOptions; diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index d5e995998f..691fc60712 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 ../pom.xml diff --git a/pom.xml b/pom.xml index ebe4aeb5f5..b79658f109 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.0.0 + 5.8.2 pom CellBase project @@ -23,20 +23,18 @@ ${project.version} - 5.0.0 - 3.0.0 + 4.12.0 + 2.12.1 0.1.0 - 9.4.51.v20230217 - - 2.14.3 - 3.14.0 - 1.7.36 - + 2.11.4 + 1.9.13 2.30.1 + 1.7.32 2.17.2 1.5.2 5.5.2 0.8.8 + 9.4.17.v20190418 0.11.5 1.6.5 3.1.0 @@ -52,6 +50,7 @@ 1.48.0 2.4 2.4 + 3.12.0 2.1.6 4.4 1.69 @@ -413,11 +412,11 @@ swagger-annotations ${swagger-annotations.version} - + io.jsonwebtoken jjwt-jackson From 8e79271ec73fcc31c5f8a74c52e2ff9a550c7d73 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 7 Mar 2024 17:13:20 +0100 Subject: [PATCH 02/28] Prepare release 6.0.0 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 289046b4cc..adf59c092a 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index ce235ddfa6..977b4b81cd 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 3e68b37513..8fcd89455f 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index ffb98718f8..07197e3a0a 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 691fc60712..71cdea12ee 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index b79658f109..f007168202 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.2 + 6.0.0-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.12.0 - 2.12.1 + 5.0.0-SNAPSHOT + 3.0.0-SNAPSHOT 0.1.0 2.11.4 1.9.13 From 4120c5cd479439e96c835d764ecb3ecb4dec8ec2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 30 Apr 2024 10:39:21 +0200 Subject: [PATCH 03/28] Prepare next release 5.8.4-SNAPSHOT --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index b561d3cc62..0767c9b05f 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 96f8c1d062..e4fd5d5b7d 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index cd396d0e1e..0b5f87db6c 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 15351ef02b..760c8e8ce0 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 101a284617..8b1b6a44e5 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index e41d354314..bfcc6602cd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 5.8.4-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.12.0 - 2.12.2 + 4.12.1-SNAPSHOT + 2.12.3-SNAPSHOT 0.1.0 2.11.4 1.9.13 From a618d7538f9fbc145794c0858eee38218a85dc97 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 30 May 2024 18:03:24 +0200 Subject: [PATCH 04/28] cicd: Update action version to test for compatibility with test and release process #TASK-6264 --- .github/workflows/test-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 413f274921..7725528041 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -11,11 +11,11 @@ jobs: name: Test and push Sonar analysis runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: '0' - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' From 9ab2fe6364edb28d0ff625134727b46ab041ace9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 4 Jun 2024 13:05:41 +0200 Subject: [PATCH 05/28] lib: use the same OpenCGA normalization parameters in the "build" for clinical variants, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java --- .../lib/builders/clinical/variant/ClinicalIndexer.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java index bbe33017fd..2b34f86a50 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinicalIndexer.java @@ -74,13 +74,12 @@ public abstract class ClinicalIndexer { protected VariantNormalizer normalizer; public ClinicalIndexer(Path genomeSequenceFilePath) throws IOException { - // Forcing decomposition here in all cases - assuming the way CellBase stores clinical variants from here - // onwards will be decomposed and Adaptors will deal with phased/no-phased queries + // Use the same OpenCGA normalization parameters VariantNormalizer.VariantNormalizerConfig variantNormalizerConfig = (new VariantNormalizer.VariantNormalizerConfig()) .setReuseVariants(true) - .setNormalizeAlleles(false) - .setDecomposeMNVs(true); + .setNormalizeAlleles(true) + .setDecomposeMNVs(false); if (genomeSequenceFilePath != null) { logger.info("Enabling left aligning by using sequence at {}", genomeSequenceFilePath.toString()); From 499407d378d471db2d07e9f5b936b7dda3ea02e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 4 Jun 2024 16:28:37 +0200 Subject: [PATCH 06/28] lib: update configuration file to download last versions of ClinVar and Gwas, files to download; and update Cosmic version, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-core/src/main/resources/configuration.yml modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/CosmicIndexer.java modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ClinicalDownloadManager.java --- cellbase-core/src/main/resources/configuration.yml | 8 ++++++-- .../builders/clinical/variant/CosmicIndexer.java | 2 +- .../lib/download/ClinicalDownloadManager.java | 13 +++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cellbase-core/src/main/resources/configuration.yml b/cellbase-core/src/main/resources/configuration.yml index bbeee761bb..e8aee3ca4d 100644 --- a/cellbase-core/src/main/resources/configuration.yml +++ b/cellbase-core/src/main/resources/configuration.yml @@ -106,11 +106,15 @@ download: clinvar: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-02.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz +# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2022-11.xml.gz + host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/RCV_xml_old_format/ClinVarFullRelease_2024-05.xml.gz + version: 2024-05 clinvarVariation: # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2021-07.xml.gz # host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-02.xml.gz - host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz +# host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/clinvar_variation/ClinVarVariationRelease_2022-11.xml.gz + host: https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/VCV_xml_old_format/ClinVarVariationRelease_2024-05.xml.gz + version: 2024-05 clinvarSummary: host: http://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz clinvarVariationAllele: diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/CosmicIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/CosmicIndexer.java index f8d2f16d15..a26d18c60c 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/CosmicIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/CosmicIndexer.java @@ -41,7 +41,7 @@ public class CosmicIndexer extends ClinicalIndexer { private Pattern mutationGRCh37GenomePositionPattern; private Pattern snvPattern; - private static final String COSMIC_VERSION = "v95"; + private static final String COSMIC_VERSION = "v99"; private static final int GENE_NAMES_COLUMN = 0; private static final int HGNC_COLUMN = 3; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ClinicalDownloadManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ClinicalDownloadManager.java index eb1f28db2d..bb9e0c36e4 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ClinicalDownloadManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/download/ClinicalDownloadManager.java @@ -81,8 +81,10 @@ public List downloadClinical() throws IOException, InterruptedExce url = configuration.getDownload().getClinvarVariationAllele().getHost(); downloadFiles.add(downloadFile(url, clinicalFolder.resolve(EtlCommons.CLINVAR_VARIATION_ALLELE_FILE).toString())); clinvarUrls.add(url); - saveVersionData(EtlCommons.CLINICAL_VARIANTS_DATA, CLINVAR_NAME, getClinVarVersion(), getTimeStamp(), clinvarUrls, - clinicalFolder.resolve("clinvarVersion.json")); + saveVersionData(EtlCommons.CLINICAL_VARIANTS_DATA, CLINVAR_NAME, configuration.getDownload().getClinvar() + .getVersion(), getTimeStamp(), clinvarUrls, clinicalFolder.resolve("clinvarVersion.json")); + + logger.info("\t\tDone"); // Gwas catalog logger.info("\t\tDownloading GWAS catalog file ..."); @@ -91,6 +93,7 @@ public List downloadClinical() throws IOException, InterruptedExce downloadFiles.add(downloadFile(url, clinicalFolder.resolve(EtlCommons.GWAS_FILE).toString())); saveVersionData(EtlCommons.CLINICAL_VARIANTS_DATA, GWAS_NAME, gwasCatalog.getVersion(), getTimeStamp(), Collections.singletonList(url), clinicalFolder.resolve("gwasVersion.json")); + logger.info("\t\tDone"); // List hgvsList = getDocmHgvsList(); // if (!hgvsList.isEmpty()) { @@ -236,10 +239,4 @@ private List getDocmHgvsList() throws IOException { return hgvsList; } - - private String getClinVarVersion() { - // ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2015-12.xml.gz - return configuration.getDownload().getClinvar().getHost().split("_")[1].split("\\.")[0]; - } - } From 45987b92961f095f0227b5840bf50bedb9da014a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 4 Jun 2024 22:14:09 +0200 Subject: [PATCH 07/28] core: update the configuration file to download the most recent version of the GWAS Catalog, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-core/src/main/resources/configuration.yml --- cellbase-core/src/main/resources/configuration.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cellbase-core/src/main/resources/configuration.yml b/cellbase-core/src/main/resources/configuration.yml index e8aee3ca4d..84378507b4 100644 --- a/cellbase-core/src/main/resources/configuration.yml +++ b/cellbase-core/src/main/resources/configuration.yml @@ -139,8 +139,10 @@ download: genomicSuperDups: host: http://hgdownload.cse.ucsc.edu/goldenPath gwasCatalog: - host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv - version: "1.0.2 associations_e106_r2022-05-17" + #host: http://resources.opencb.org/opencb/cellbase/data/gwas/gwas_catalog_v1.0.2-associations_e106_r2022-05-17.tsv + host: "https://ftp.ebi.ac.uk/pub/databases/gwas/releases/2024/05/20/gwas-catalog-associations_ontology-annotated.tsv" + #version: "1.0.2 associations_e106_r2022-05-17" + version: "2024-05-20" hpo: host: https://ci.monarchinitiative.org/view/hpo/job/hpo.annotations/lastSuccessfulBuild/artifact/rare-diseases/util/annotation/phenotype_to_genes.txt disgenet: From 9b4e446cd125ea23ba16d9912818e9da9a702435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 4 Jun 2024 22:27:33 +0200 Subject: [PATCH 08/28] lib: update ClinVar filename, version and date to the most recent version, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java --- .../src/main/java/org/opencb/cellbase/lib/EtlCommons.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java index edf41c1e11..d09291bc3e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/EtlCommons.java @@ -56,9 +56,9 @@ public class EtlCommons { public static final String PHARMGKB_VERSION_FILENAME = "pharmgkbVersion.json"; public static final String CLINICAL_VARIANTS_FOLDER = "clinicalVariant"; - public static final String CLINVAR_VERSION = "2022.11"; - public static final String CLINVAR_DATE = "2022-11"; - public static final String CLINVAR_XML_FILE = "ClinVarFullRelease_2022-11.xml.gz"; + public static final String CLINVAR_VERSION = "2024-05"; + public static final String CLINVAR_DATE = "2024-05"; + public static final String CLINVAR_XML_FILE = "ClinVarFullRelease_2024-05.xml.gz"; public static final String CLINVAR_EFO_FILE = "ClinVar_Traits_EFO_Names.csv"; public static final String CLINVAR_SUMMARY_FILE = "variant_summary.txt.gz"; public static final String CLINVAR_VARIATION_ALLELE_FILE = "variation_allele.txt.gz"; From 9b91471bfd787e1735e0ae84e55430f62de70670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Wed, 12 Jun 2024 13:45:25 +0200 Subject: [PATCH 09/28] lib: catch exceptions when parsing ClinVar file, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java --- .../clinical/variant/ClinVarIndexer.java | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java index a31bd8d5e6..746012880a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java @@ -210,7 +210,7 @@ private void printSummary() { } private boolean updateRocksDB(SequenceLocation sequenceLocation, String variationId, String[] lineFields, - String mateVariantString, Map traitsToEfoTermsMap) + String mateVariantString, Map traitsToEfoTermsMap) throws RocksDBException, IOException { // More than one variant being returned from the normalisation process would mean it's and MNV which has been // decomposed @@ -266,13 +266,34 @@ private boolean updateRocksDB(AlleleLocationData alleleLocationData, PublicSetTy } // parse RCVs - String accession = publicSet.getReferenceClinVarAssertion().getClinVarAccession().getAcc(); - String clinicalSignficanceDescription = publicSet.getReferenceClinVarAssertion() - .getClinicalSignificance() - .getDescription(); - String reviewStatusName = publicSet.getReferenceClinVarAssertion().getClinicalSignificance() - .getReviewStatus().name(); - List getObservedIn = publicSet.getReferenceClinVarAssertion().getObservedIn(); + String accession = null; + try { + accession = publicSet.getReferenceClinVarAssertion().getClinVarAccession().getAcc(); + } catch (Exception e) { + logger.warn("Error getting accession\n" + StringUtils.join(e.getStackTrace(), "\n")); + } + String clinicalSignficanceDescription = null; + try { + clinicalSignficanceDescription = publicSet.getReferenceClinVarAssertion() + .getClinicalSignificance() + .getDescription(); + } catch (Exception e) { + logger.warn("Error getting clinical Significance description\n" + StringUtils.join(e.getStackTrace(), "\n")); + } + String reviewStatusName = null; + try { + reviewStatusName = publicSet.getReferenceClinVarAssertion().getClinicalSignificance() + .getReviewStatus().name(); + } catch (Exception e) { + logger.warn("Error getting review status name\n" + StringUtils.join(e.getStackTrace(), "\n")); + } + List getObservedIn = null; + try { + getObservedIn = publicSet.getReferenceClinVarAssertion().getObservedIn(); + } catch (Exception e) { + logger.warn("Error getting observed in\n" + StringUtils.join(e.getStackTrace(), "\n")); + } + addNewEntries(variantAnnotation, publicSet, alleleLocationData.getAlleleId(), mateVariantString, clinicalHaplotypeString, traitsToEfoTermsMap, accession, clinicalSignficanceDescription, reviewStatusName, getObservedIn); @@ -388,7 +409,7 @@ private void addNewEntries(VariantAnnotation variantAnnotation, PublicSetType pu Map traitsToEfoTermsMap, String accession, String clinicalSignficanceDescription, String reviewStatusName, List getObservedIn) - throws JsonProcessingException { + throws JsonProcessingException { List additionalProperties = new ArrayList<>(3); EvidenceSource evidenceSource = new EvidenceSource(EtlCommons.CLINVAR_DATA, CLINVAR_VERSION, CLINVAR_DATE); @@ -544,7 +565,7 @@ private ModeOfInheritance getModeOfInheritance(String modeOfInheritance) { private List getGenomicFeature(PublicSetType publicSet, String alleleId) { if (publicSet.getReferenceClinVarAssertion().getMeasureSet() != null) { return getGenomicFeature(publicSet.getReferenceClinVarAssertion().getMeasureSet()); - // No measureSet means there must be genotypeSet + // No measureSet means there must be genotypeSet } else if (publicSet.getReferenceClinVarAssertion().getGenotypeSet() != null) { for (MeasureSetType measureSet : publicSet.getReferenceClinVarAssertion().getGenotypeSet().getMeasureSet()) { if (measureSet.getMeasure() != null) { @@ -596,7 +617,7 @@ private List getHeritableTrait(PublicSetType publicSet, Map 0) { logger.warn("ClinVar record found " + publicSet.getReferenceClinVarAssertion().getClinVarAccession().getAcc() + " with no preferred trait provided. Arbitrarily selecting first one: {}", trait.getName() .get(0).getElementValue().getValue()); return trait.getName().get(0).getElementValue().getValue(); - // No trait name provided at all + // No trait name provided at all } else { throw new IllegalArgumentException("ClinVar record found " + publicSet.getReferenceClinVarAssertion().getClinVarAccession().getAcc() From f11f507e4c7885ebd0dbaca5e32ba9a851b22bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 18 Jun 2024 23:09:05 +0200 Subject: [PATCH 10/28] lib: add NCBI ID support for dbSNP file (and .tbi) when parsing GWAS file; and fix some sonnar issues, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/GwasIndexer.java --- .../clinical/variant/GwasIndexer.java | 151 +++++++++++------- 1 file changed, 93 insertions(+), 58 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/GwasIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/GwasIndexer.java index 2b4f2e4d8b..0fe3b0f115 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/GwasIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/GwasIndexer.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.text.NumberFormat; import java.util.*; +import java.util.stream.Collectors; public class GwasIndexer extends ClinicalIndexer { @@ -46,6 +47,8 @@ public class GwasIndexer extends ClinicalIndexer { private int gwasLinesNotFoundInDbsnp; private int invalidVariantRecords; + private int lineCounter = 0; + public GwasIndexer(Path gwasFile, Path dbSnpTabixFile, Path genomeSequenceFilePath, String assembly, RocksDB rdb) throws IOException { super(genomeSequenceFilePath); @@ -56,36 +59,31 @@ public GwasIndexer(Path gwasFile, Path dbSnpTabixFile, Path genomeSequenceFilePa } public void index() throws RocksDBException, IOException { - logger.info("Parsing GWAS catalog file ..."); - - BufferedReader inputReader = null; - TabixReader dbsnpTabixReader = null; - - try { - logger.info("Opening GWAS catalog file " + gwasFile + " ..."); - inputReader = new BufferedReader(new FileReader(gwasFile.toFile())); + try (BufferedReader inputReader = new BufferedReader(new FileReader(gwasFile.toFile())); + TabixReader dbsnpTabixReader = new TabixReader(dbSnpTabixFile.toString())) { logger.info("Ignoring GWAS catalog file header line ..."); - String line = inputReader.readLine(); + inputReader.readLine(); + ++lineCounter; + Map chromosomeMap = buildChromosomeMap(dbsnpTabixReader); Map gwasMap = new HashMap<>(); - logger.info("Opening dbSNP tabix file " + dbSnpTabixFile + " ..."); - dbsnpTabixReader = new TabixReader(dbSnpTabixFile.toString()); long processedGwasLines = 0; - logger.info("Parsing GWAS catalog file ..."); + logger.info("Parsing GWAS catalog file {} ...", gwasFile); + String line; while ((line = inputReader.readLine()) != null) { + ++lineCounter; if (!line.isEmpty()) { processedGwasLines++; if (processedGwasLines % 10000 == 0) { logger.info("{} lines parsed", processedGwasLines); } - processGwasCatalogLine(line.split("\t"), dbsnpTabixReader, gwasMap); + processGwasCatalogLine(line.split("\t"), dbsnpTabixReader, gwasMap, chromosomeMap); } } - dbsnpTabixReader.close(); logger.info("Updating clinical variant annotation..."); long counter = 0; @@ -118,16 +116,9 @@ public void index() throws RocksDBException, IOException { rdb.put(entry.getKey().getBytes(), jsonObjectWriter.writeValueAsBytes(variantAnnotation)); } this.printSummary(processedGwasLines, gwasMap); - } catch (RocksDBException | IOException e) { + } catch (RocksDBException | IOException e) { logger.error("Error reading/writing from/to the RocksDB index while indexing GWAS catalog file"); throw e; - } finally { - if (inputReader != null) { - inputReader.close(); - } - if (dbsnpTabixReader != null) { - dbsnpTabixReader.close(); - } } } @@ -184,13 +175,14 @@ significant digit (for example, a published p-value of 4.8 x 10-7 is rounded to 37 GENOTYPING_TECHNOLOGY* +: Genotyping technology/ies used in this study, with additional array information (ex. Immunochip or Exome array) in brackets. */ - private void processGwasCatalogLine(String[] values, TabixReader dbsnpTabixReader, Map gwasMap) { + private void processGwasCatalogLine(String[] values, TabixReader dbsnpTabixReader, Map gwasMap, + Map chromosomeMap) throws IOException { Integer start = parseStart(values); if (start != null) { String chromosome = parseChromosome(values[11]); if (StringUtils.isNotEmpty(chromosome)) { String snpId = "rs" + values[23].trim(); - String[] refAndAlt = getRefAndAltFromDbsnp(chromosome, start, snpId, dbsnpTabixReader); + String[] refAndAlt = getRefAndAltFromDbsnp(chromosome, start, snpId, dbsnpTabixReader, chromosomeMap); if (refAndAlt != null) { // Create variant Variant variant; @@ -270,21 +262,27 @@ private void processGwasCatalogLine(String[] values, TabixReader dbsnpTabixReade // Scores management GwasAssociationStudyTraitScores scores = new GwasAssociationStudyTraitScores(); - try { - scores.setPValue(Double.parseDouble(values[27])); - } catch (NumberFormatException e) { -// logger.warn(e.getMessage() + ". Parsing pValue: " + values[27]); + if (StringUtils.isNotEmpty(values[27])) { + try { + scores.setPValue(Double.parseDouble(values[27])); + } catch (NumberFormatException e) { + logger.warn(e.getMessage() + ". Parsing pValue: " + values[27]); + } } - try { - scores.setPValueMlog(Double.parseDouble(values[28])); - } catch (NumberFormatException e) { -// logger.warn(e.getMessage() + ". Parsing pValue mlog: " + values[28]); + if (StringUtils.isNotEmpty(values[28])) { + try { + scores.setPValueMlog(Double.parseDouble(values[28])); + } catch (NumberFormatException e) { + logger.warn(e.getMessage() + ". Parsing pValue mlog: " + values[28]); + } } scores.setPValueText(values[29]); - try { - scores.setOrBeta(Double.parseDouble(values[30])); - } catch (NumberFormatException e) { -// logger.warn(e.getMessage() + ". Parsing Odd or beta: " + values[30]); + if (StringUtils.isNotEmpty(values[30])) { + try { + scores.setOrBeta(Double.parseDouble(values[30])); + } catch (NumberFormatException e) { + logger.warn(e.getMessage() + ". Parsing Odd or beta: " + values[30]); + } } scores.setPercentCI(values[31]); @@ -301,15 +299,15 @@ private void processGwasCatalogLine(String[] values, TabixReader dbsnpTabixReade gwasMap.put(key, gwas); } } else { -// logger.warn("Variant not found in dbSNP " + snpId + ". Line: " + StringUtils.join(values, "\t\t\t")); + logger.warn("dbSNP {} not found. Line: {}", snpId, lineCounter); gwasLinesNotFoundInDbsnp++; } } else { -// logger.warn("Invalid chromosome " + chromosome + ". Line: " + StringUtils.join(values, "\t\t\t")); + logger.warn("Invalid chromosome {}. Line: {}", chromosome, lineCounter); invalidChromosome++; } } else { -// logger.warn("Invalid position " + start + ". Line: " + StringUtils.join(values, "\t\t\t")); + logger.warn("Invalid position {}. Line: {}", start, lineCounter); invalidStartRecords++; } } @@ -342,6 +340,39 @@ private String parseChromosome(String chromosome) { return transformedChromosome; } + private Map buildChromosomeMap(TabixReader dbsnpTabixReader) { + List chroms = dbsnpTabixReader.getChromosomes().stream().filter(name -> name.startsWith("NC_")) + .collect(Collectors.toList()); + + Map chromMap = new HashMap<>(); + for (int i = 1; i < 22; i++) { + chromMap.put(Integer.toString(i), Integer.toString(i)); + } + chromMap.put("X", "X"); + chromMap.put("Y", "Y"); + chromMap.put("MT", "MT"); + + for (String chrom : chroms) { + String[] split = chrom.split("[_\\.]"); + int value = Integer.parseInt(split[1]); + switch (value) { + case 23: + chromMap.put("X", chrom); + break; + case 24: + chromMap.put("Y", chrom); + break; + case 12920: + chromMap.put("MT", chrom); + break; + default: + chromMap.put(Integer.toString(value), chrom); + break; + } + } + return chromMap; + } + private Float parseFloat(String value) { Float riskAlleleFrequency = null; if (NumberUtils.isNumber(value)) { @@ -350,29 +381,33 @@ private Float parseFloat(String value) { return riskAlleleFrequency; } - private String[] getRefAndAltFromDbsnp(String chromosome, Integer start, String snpId, TabixReader dbsnpTabixReader) { + private String[] getRefAndAltFromDbsnp(String chromosome, Integer start, String snpId, TabixReader dbsnpTabixReader, + Map chromosomeMap) throws IOException { + boolean found = false; + Set foundSnpIds = new HashSet<>(); String[] refAndAlt = null; - TabixReader.Iterator dbsnpIterator = dbsnpTabixReader.query(chromosome + ":" + start + "-" + start); - try { - String dbSnpRecord = dbsnpIterator.next(); - boolean found = false; - while (dbSnpRecord != null && !found) { - String[] dbsnpFields = dbSnpRecord.split("\t"); - - if (snpId.equalsIgnoreCase(dbsnpFields[2])) { - refAndAlt = new String[2]; - refAndAlt[REF] = dbsnpFields[3]; - refAndAlt[ALT] = dbsnpFields[4]; - found = true; - } - - dbSnpRecord = dbsnpIterator.next(); + String query = chromosomeMap.get(chromosome) + ":" + start + "-" + start; + TabixReader.Iterator dbsnpIterator = dbsnpTabixReader.query(query); + String dbSnpRecord = null; + dbSnpRecord = dbsnpIterator.next(); + while (dbSnpRecord != null && !found) { + String[] dbsnpFields = dbSnpRecord.split("\t"); + + if (snpId.equalsIgnoreCase(dbsnpFields[2])) { + refAndAlt = new String[2]; + refAndAlt[REF] = dbsnpFields[3]; + refAndAlt[ALT] = dbsnpFields[4]; + found = true; + } else { + foundSnpIds.add(dbsnpFields[2]); } - } catch (IOException e) { - logger.warn("Error reading position '" + chromosome + ":" + start + "' in dbSNP: " + e.getMessage()); - } + dbSnpRecord = dbsnpIterator.next(); + } + if (!found) { + logger.warn("dbSNP {} not found from query {}. Found: {}", snpId, query, foundSnpIds); + } return refAndAlt; } From 547c8fe8121a9f5502310b22882e1c08d8128298 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 14:33:25 +0200 Subject: [PATCH 11/28] cicd: Create branch and pull-reques-approved.yml #TASK-6399 --- .github/workflows/pull-request-approved.yml | 34 +++++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index eb410c9cba..8a60928fc8 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -1,15 +1,37 @@ name: Pull request approve workflow +run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}' on: pull_request_review: types: [ submitted ] jobs: - build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + calculate-xetabase-branch: + name: Calculate Xetabase branch + runs-on: ubuntu-22.04 + outputs: + xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }} + steps: + - name: Clone java-common-libs + uses: actions/checkout@v4 + with: + fetch-depth: '10' + - id: get_xetabase_branch + name: "Get current branch for Xetabase from target branch" + run: | + chmod +x ./.github/workflows/scripts/xetabase-branch.sh + ls ./.github/workflows/scripts/ + ls ./.github/workflows/ + bash --version + xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} + echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT test: - name: "Test analysis" - uses: ./.github/workflows/test-analysis.yml - needs: build - secrets: inherit + name: "Run all tests before merging" + needs: calculate-xetabase-branch + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + with: + branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} + task: ${{ github.event.pull_request.head.ref }} + secrets: inherit \ No newline at end of file From d00ef5bc200e8732f5420f7ff1c8184542cc8323 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 25 Jun 2024 14:48:16 +0200 Subject: [PATCH 12/28] cicd: added script xetabase_branch #TASK-6399 --- .github/workflows/scripts/xetabase-branch.sh | 42 ++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/scripts/xetabase-branch.sh diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/xetabase-branch.sh new file mode 100644 index 0000000000..af17f7f126 --- /dev/null +++ b/.github/workflows/scripts/xetabase-branch.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Function to calculate the corresponding branch of Xetabase project +get_xetabase_branch() { + # Input parameter (branch name) + input_branch="$1" + + # Check if the branch name is "develop" in that case return the same branch name + if [[ "$input_branch" == "develop" ]]; then + echo "develop" + return 0 + fi + + # Check if the branch name starts with "release-" and follows the patterns "release-a.b.x" or "release-a.b.c.x" + if [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.([0-9]+)\.x$ ]]; then + # Extract the MAJOR part of the branch name + MAJOR=${BASH_REMATCH[1]} + # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR + XETABASE_MAJOR=$((MAJOR - 3)) + # Check if the XETABASE_MAJOR is negative + if (( XETABASE_MAJOR < 0 )); then + echo "Error: 'MAJOR' digit after subtraction results in a negative number." + return 1 + fi + # Construct and echo the new branch name + echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}" + return 0 + fi + + # If the branch name does not match any of the expected patterns + echo "Error: The branch name is not correct." + return 1 +} + +# Check if the script receives exactly one argument +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Call the function with the input branch name +get_xetabase_branch "$1" From f820260f07a4d69456dbf9f5d8b7dfa343234c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20T=C3=A1rraga=20Gim=C3=A9nez?= Date: Tue, 25 Jun 2024 15:57:11 +0200 Subject: [PATCH 13/28] lib: improve log messages for ClinVar indexer, #TASK-6347 On branch TASK-6347 Changes to be committed: modified: cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java --- .../lib/builders/clinical/variant/ClinVarIndexer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java index 746012880a..8b88f821f6 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java @@ -270,7 +270,7 @@ private boolean updateRocksDB(AlleleLocationData alleleLocationData, PublicSetTy try { accession = publicSet.getReferenceClinVarAssertion().getClinVarAccession().getAcc(); } catch (Exception e) { - logger.warn("Error getting accession\n" + StringUtils.join(e.getStackTrace(), "\n")); + logger.warn("Error getting accession. Ignore error and leave it as null.", e); } String clinicalSignficanceDescription = null; try { @@ -278,20 +278,20 @@ private boolean updateRocksDB(AlleleLocationData alleleLocationData, PublicSetTy .getClinicalSignificance() .getDescription(); } catch (Exception e) { - logger.warn("Error getting clinical Significance description\n" + StringUtils.join(e.getStackTrace(), "\n")); + logger.warn("Error getting clinical significance description. Ignore error and leave it as null.", e); } String reviewStatusName = null; try { reviewStatusName = publicSet.getReferenceClinVarAssertion().getClinicalSignificance() .getReviewStatus().name(); } catch (Exception e) { - logger.warn("Error getting review status name\n" + StringUtils.join(e.getStackTrace(), "\n")); + logger.warn("Error getting review status name. Ignore error and leave it as null.", e); } List getObservedIn = null; try { getObservedIn = publicSet.getReferenceClinVarAssertion().getObservedIn(); } catch (Exception e) { - logger.warn("Error getting observed in\n" + StringUtils.join(e.getStackTrace(), "\n")); + logger.warn("Error getting observed in. Ignore error and leave it as null.", e); } addNewEntries(variantAnnotation, publicSet, alleleLocationData.getAlleleId(), mateVariantString, From 60abe588b24292bae0e7fc9faf7a820e7fd3bf48 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 1 Jul 2024 17:37:43 +0200 Subject: [PATCH 14/28] cicd: added script get-xetabase-branch #TASK-6399 --- .github/workflows/pull-request-approved.yml | 7 ++----- .../{xetabase-branch.sh => get-xetabase-branch.sh} | 10 +++++++++- 2 files changed, 11 insertions(+), 6 deletions(-) rename .github/workflows/scripts/{xetabase-branch.sh => get-xetabase-branch.sh} (80%) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 8a60928fc8..a0d481faed 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -19,11 +19,8 @@ jobs: - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | - chmod +x ./.github/workflows/scripts/xetabase-branch.sh - ls ./.github/workflows/scripts/ - ls ./.github/workflows/ - bash --version - xetabase_branch=$(./.github/workflows/scripts/xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh similarity index 80% rename from .github/workflows/scripts/xetabase-branch.sh rename to .github/workflows/scripts/get-xetabase-branch.sh index af17f7f126..f6f8de5d67 100644 --- a/.github/workflows/scripts/xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -5,6 +5,14 @@ get_xetabase_branch() { # Input parameter (branch name) input_branch="$1" + # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it + if [[ $input_branch == TASK* ]]; then + if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then + echo "$GIT_BRANCH"; + exit 0; + fi + fi + # Check if the branch name is "develop" in that case return the same branch name if [[ "$input_branch" == "develop" ]]; then echo "develop" @@ -16,7 +24,7 @@ get_xetabase_branch() { # Extract the MAJOR part of the branch name MAJOR=${BASH_REMATCH[1]} # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR - XETABASE_MAJOR=$((MAJOR - 3)) + XETABASE_MAJOR=$((MAJOR - 4)) # Check if the XETABASE_MAJOR is negative if (( XETABASE_MAJOR < 0 )); then echo "Error: 'MAJOR' digit after subtraction results in a negative number." From 73057b56e5d1360dbb4b04de6ce32db8b7f64b96 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 5 Jul 2024 09:24:58 +0200 Subject: [PATCH 15/28] Prepare release 5.8.4 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 0767c9b05f..482f21352b 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index e4fd5d5b7d..6a16f543fa 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 0b5f87db6c..a0c238ce8a 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 760c8e8ce0..d6a4aacde0 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 8b1b6a44e5..e9ebc0de6c 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 ../pom.xml diff --git a/pom.xml b/pom.xml index bfcc6602cd..a88d83275f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4-SNAPSHOT + 5.8.4 pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 4.12.1-SNAPSHOT - 2.12.3-SNAPSHOT + 4.12.0 + 2.12.2 0.1.0 2.11.4 1.9.13 From 386a5109df681b82dde4ae05d6e9f7c9f0233ea4 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 17 Jul 2024 10:55:42 +0200 Subject: [PATCH 16/28] Prepare new development branch release-6.2.x --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index aed90e9897..441dc47bff 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 7424c21bbb..a843ef0685 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 7c74e13d92..db167df9bb 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 9b4dd7fbe0..28ccca9267 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index fe4509c6fc..304cca8aa2 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 302ae6c6fc..1c619a886b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.3.0-SNAPSHOT + 6.2.1-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 5.3.0-SNAPSHOT - 3.3.0-SNAPSHOT + 5.2.1-SNAPSHOT + 3.2.1-SNAPSHOT 0.1.0 9.4.51.v20230217 From b52c83d5cef729b0ef1594bc8239cccfa4d405f8 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 30 Jul 2024 09:36:57 +0200 Subject: [PATCH 17/28] Prepare Port patch #TASK-6515 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 482f21352b..441dc47bff 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 6a16f543fa..a843ef0685 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index a0c238ce8a..db167df9bb 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index d6a4aacde0..5f7510deb1 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index e9ebc0de6c..304cca8aa2 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index a88d83275f..fff82f83a8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.4 + 6.2.1-SNAPSHOT pom CellBase project From 0f2a6d25a92df05b318908bcbdc9e9a75311d9ff Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 5 Aug 2024 12:54:47 +0200 Subject: [PATCH 18/28] Prepare Port Patch Cellbase 5.8.3 -> 6.2.1 #TASK-6647 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index b561d3cc62..441dc47bff 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 96f8c1d062..a843ef0685 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index cd396d0e1e..db167df9bb 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 15351ef02b..5f7510deb1 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 101a284617..304cca8aa2 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index e41d354314..fff82f83a8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 5.8.3 + 6.2.1-SNAPSHOT pom CellBase project From 8bde7d1989050c42d0df89a98ecce3862648f8af Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Wed, 4 Sep 2024 18:32:19 +0200 Subject: [PATCH 19/28] cicd: Upload reference to develop branch in pull-request-approve to test-xetabase-workflow #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index a0d481faed..2370972080 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -27,7 +27,7 @@ jobs: test: name: "Run all tests before merging" needs: calculate-xetabase-branch - uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@TASK-6399 + uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop with: branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }} task: ${{ github.event.pull_request.head.ref }} From 2a0983a380d07637b95f5eeebf10322138eb4e5b Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 9 Sep 2024 16:42:38 +0200 Subject: [PATCH 20/28] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 7 ++++++- .github/workflows/scripts/get-xetabase-branch.sh | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 2370972080..e4535a190a 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -20,9 +20,14 @@ jobs: name: "Get current branch for Xetabase from target branch" run: | chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" + echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" + echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT + env: + ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }} test: name: "Run all tests before merging" diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh index f6f8de5d67..16ae8c4711 100644 --- a/.github/workflows/scripts/get-xetabase-branch.sh +++ b/.github/workflows/scripts/get-xetabase-branch.sh @@ -7,9 +7,9 @@ get_xetabase_branch() { # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it if [[ $input_branch == TASK* ]]; then - if [ "$(git ls-remote https://github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then - echo "$GIT_BRANCH"; - exit 0; + if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then + echo $input_branch; + return 0; fi fi From df4f83d769fe3ec7f2a48d74db2e9653fbce0d24 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 10 Sep 2024 09:10:37 +0200 Subject: [PATCH 21/28] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index e4535a190a..c2fe27a07f 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -16,6 +16,8 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: '10' + ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" + persist-credentials: false - id: get_xetabase_branch name: "Get current branch for Xetabase from target branch" run: | From 7762ee8812fe1d0b7910ac87935e0b16bca793b2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 10 Sep 2024 09:26:50 +0200 Subject: [PATCH 22/28] cicd: Fix xetabase branch calculation #TASK-6807 --- .github/workflows/pull-request-approved.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index c2fe27a07f..d339f65b96 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -24,7 +24,6 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT From 16a1eaabf90aa4fad9e75714a1194535e2cc6b54 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 12:41:17 +0200 Subject: [PATCH 23/28] Prepare release 6.2.1 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 441dc47bff..df777079c3 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index a843ef0685..2e9b1262ac 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index db167df9bb..183634a928 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 28ccca9267..bc7b005e88 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 304cca8aa2..96319c7cbf 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 7aab94cee9..a62001d6c8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 5.2.1-SNAPSHOT - 3.2.1-SNAPSHOT + 5.2.1 + 3.2.1 0.1.0 9.4.51.v20230217 From ae526174e034bffb5a8bc7fcfcec12cbd8c0d5dd Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 12:52:24 +0200 Subject: [PATCH 24/28] Prepare release 6.2.1 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index df777079c3..441dc47bff 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 2e9b1262ac..a843ef0685 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 183634a928..db167df9bb 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index bc7b005e88..28ccca9267 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 96319c7cbf..304cca8aa2 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index a62001d6c8..61421787a1 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 6.2.1-SNAPSHOT pom CellBase project From b86ce69848e848e3bb58d1e3c1152a93344fefd2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 12:55:42 +0200 Subject: [PATCH 25/28] Prepare release 6.2.0 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 4b05d287c1..cfd8a7de30 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index f4ecc84585..47a5c7ca9a 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 1a35c185cc..5eca656e8d 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index c1ed6f6ba6..0ddb09059b 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 2bee90c836..aaa2015e0b 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 ../pom.xml diff --git a/pom.xml b/pom.xml index f44762ec0b..0b39ddc2df 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.2-SNAPSHOT + 6.2.0 pom CellBase project From 67f14a01413341caa73512f2dd770e1e3be10dc8 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 13:39:51 +0200 Subject: [PATCH 26/28] Prepare release 6.2.1 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index 441dc47bff..df777079c3 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index a843ef0685..2e9b1262ac 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index db167df9bb..183634a928 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 28ccca9267..bc7b005e88 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 304cca8aa2..96319c7cbf 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 61421787a1..a62001d6c8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1-SNAPSHOT + 6.2.1 pom CellBase project From b7ce5b1561d3d0aa651e4e1478e9b776fa151cda Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 13 Sep 2024 13:48:56 +0200 Subject: [PATCH 27/28] Prepare release 6.2.1 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index cfd8a7de30..df777079c3 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 47a5c7ca9a..2e9b1262ac 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 5eca656e8d..183634a928 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 0ddb09059b..0ccb570c21 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index aaa2015e0b..96319c7cbf 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index 0b39ddc2df..6c047d1a2b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.0 + 6.2.1 pom CellBase project From d3521cdc835b7a65e4d2577a7856ac71f95c06d2 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 3 Oct 2024 13:22:19 +0200 Subject: [PATCH 28/28] Port Patch 6.2.1 -> 7.0.0 XB 2.2.1 -> 3.0.0 #TASK-6780 --- cellbase-app/pom.xml | 2 +- cellbase-client/pom.xml | 2 +- cellbase-core/pom.xml | 2 +- cellbase-lib/pom.xml | 2 +- cellbase-server/pom.xml | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cellbase-app/pom.xml b/cellbase-app/pom.xml index df777079c3..2877826403 100644 --- a/cellbase-app/pom.xml +++ b/cellbase-app/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-client/pom.xml b/cellbase-client/pom.xml index 2e9b1262ac..9fc6f72f07 100644 --- a/cellbase-client/pom.xml +++ b/cellbase-client/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 183634a928..f4659e2759 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-lib/pom.xml b/cellbase-lib/pom.xml index 0ccb570c21..514d844894 100644 --- a/cellbase-lib/pom.xml +++ b/cellbase-lib/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT ../pom.xml diff --git a/cellbase-server/pom.xml b/cellbase-server/pom.xml index 96319c7cbf..d89aa0e106 100644 --- a/cellbase-server/pom.xml +++ b/cellbase-server/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 6c047d1a2b..8f7ee392ad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.opencb.cellbase cellbase - 6.2.1 + 7.0.0-SNAPSHOT pom CellBase project @@ -23,8 +23,8 @@ ${project.version} - 5.2.2-SNAPSHOT - 3.2.2-SNAPSHOT + 6.0.0-SNAPSHOT + 4.0.0-SNAPSHOT 0.1.0 2.11.4