Merge pull request #629 from deeptools/develop

Develop

azure-pipelines.yml

@@ -13,6 +13,8 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
 
   steps:
   - bash: |
@@ -54,6 +56,8 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
 
   steps:
   - bash: |

docs/conf.py

@@ -30,7 +30,7 @@
                 'mpl_toolkits', 'mpl_toolkits.axisartist', 'mpl_toolkits.mplot3d', 'mpl_toolkits.axes_grid1',
                 'Bio', 'Bio.Seq', 'Bio.Alphabet', 'pyBigWig', 'tables', 'pytables', 'future', 'past', 'past.builtins',
                 'future.utils', 'cooler', 'logging', 'unidecode', 'hic2cool', 'hicmatrix', 'hicmatrix.HiCMatrix',
-                'hicmatrix.lib', 'krbalancing', 'fit_nbinom', 'pybedtools', 'graphviz', 'hyperopt']
+                'hicmatrix.lib', 'krbalancing', 'fit_nbinom', 'pybedtools', 'graphviz', 'hyperopt', 'sklearn', 'sklearn.cluster']
 
 for mod_name in MOCK_MODULES:
     sys.modules[mod_name] = Mock()

docs/content/News.rst

@@ -1,12 +1,28 @@
 News and Developments
 =====================
 
+
+Release 3.6
+-----------
+**10 November 2020**
+
+- hicAggregateContacts, thanks @LeilyR:
+    - hicAggragateContact has been updated to be able to handle inter chromosomal contacts as well as inter chromosomal contacts
+    - Added scikit-learn to dependencies
+- hicBuildMatrix: Fixing another bug concerning scaffolds without any restriction enzyme cut site
+- Updated dependencies
+- Adding default values to the documentation. Thanks @lldelisle
+- hicTransform: Fixing a bug in case one of the intermediate matrices is empty
+- Official Python 3.8 support:
+   - Manually setting 'fork' as the start method for multiprocessing because the default on macOS was set to 'spawn'
+
+
 Release 3.5.3
 -------------
 **14 October 2020**
 
 - Bug fix release: 
-   - Reads from scaffolds without any restriction enzym cut site are considered as 'same fragment'. An appearance of such a read will not lead to a crash anymore
+   - Reads from scaffolds without any restriction enzyme cut site are considered as 'same fragment'. An appearance of such a read will not lead to a crash anymore
 - Minor documentation improvements
 
 

docs/content/citation.rst

@@ -0,0 +1,14 @@
+
+Citation
+========
+
+Please cite HiCExplorer as follows:
+
+Joachim Wolff, Leily Rabbani, Ralf Gilsbach, Gautier Richard, Thomas Manke, Rolf Backofen, Björn A Grüning.
+**Galaxy HiCExplorer 3: a web server for reproducible Hi-C, capture Hi-C and single-cell Hi-C data analysis, quality control and visualization, Nucleic Acids Research**, Nucleic Acids Research, Volume 48, Issue W1, 02 July 2020, Pages W177–W184, https://doi.org/10.1093/nar/gkaa220
+
+Joachim Wolff, Vivek Bhardwaj, Stephan Nothjunge, Gautier Richard, Gina Renschler, Ralf Gilsbach, Thomas Manke, Rolf Backofen, Fidel Ramírez, Björn A Grüning. 
+**Galaxy HiCExplorer: a web server for reproducible Hi-C data analysis, quality control and visualization**, Nucleic Acids Research, Volume 46, Issue W1, 2 July 2018, Pages W11–W16, doi: https://doi.org/10.1093/nar/gky504
+
+Fidel Ramirez, Vivek Bhardwaj, Jose Villaveces, Laura Arrigoni, Bjoern A Gruening,Kin Chung Lam, Bianca Habermann, Asifa Akhtar, Thomas Manke.
+**"High-resolution TADs reveal DNA sequences underlying genome organization in flies". Nature Communications**, Volume 9, Article number: 189 (2018), doi: https://doi.org/10.1038/s41467-017-02525-w

docs/content/tools/hicAggregateContacts.rst

@@ -28,7 +28,7 @@ Below, you can find an example of an aggregate Hi-C matrix obtained from *Drosop
     $ hicAggregateContacts --matrix Dmel.h5 --BED ChIP-seq-peaks.bed \
     --outFileName Dmel_aggregate_Contacts --vMin 0.8 --vMax 2.2 \
     --range 300000:1000000 --numberOfBins 30 --chromosomes X \
-    --avgType mean --transform obs/exp
+    --operationType mean --transform obs/exp --mode intra-chr
 
 .. image:: ../../images/hicAggregateContacts.png
 

docs/index.rst

@@ -84,6 +84,7 @@ Contents:
    content/list-of-tools
    content/example_usage
    content/News
+   content/citation
 
 
 Citation

hicexplorer/__init__.py

@@ -15,3 +15,6 @@
 
 warnings.simplefilter(action="ignore", category=RuntimeWarning)
 warnings.simplefilter(action="ignore", category=PendingDeprecationWarning)
+
+import multiprocessing as mp
+mp.set_start_method('fork')

hicexplorer/chicAggregateStatistic.py

@@ -60,31 +60,35 @@ def parse_arguments(args=None):
     parserOpt = parser.add_argument_group('Optional arguments')
 
     parserOpt.add_argument('--outFileNameSuffix', '-suffix',
-                           help='File name suffix to save the result.',
+                           help='File name suffix to save the result'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='_aggregate_target.txt')
 
     parserOpt.add_argument('--interactionFileFolder', '-iff',
-                           help='Folder where the interaction files are stored. Applies only for batch mode.',
+                           help='Folder where the interaction files are stored. Applies only for batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--targetFileFolder', '-tff',
                            help='Folder where the target files are stored. Applies only for batch mode.',
                            required=False)
     parserOpt.add_argument('--outputFolder', '-o',
-                           help='Output folder containing the files.',
+                           help='Output folder containing the files'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='aggregatedFiles')
     parserOpt.add_argument('--writeFileNamesToFile', '-w',
-                           help='',
+                           help='(Default: %(default)s).',
                            default='aggregatedFilesBatch.txt')
     parserOpt.add_argument('--batchMode', '-bm',
                            help='turns on batch mode. The files provided by --interactionFile and/or --targetFile contain a list of the files to be processed.',
                            required=False,
                            action='store_true')
 
     parserOpt.add_argument('--threads', '-t',
-                           help='Number of threads (uses the python multiprocessing module). ',
+                           help='Number of threads (uses the python multiprocessing module)ist'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=4,
                            type=int

hicexplorer/chicDifferentialTest.py

@@ -53,35 +53,39 @@ def parse_arguments(args=None):
     parserRequired.add_argument('--alpha', '-a',
                                 help='define a significance level (alpha) for accepting samples',
                                 type=float,
-                                default=0.05,
                                 required=True)
 
     parserOpt = parser.add_argument_group('Optional arguments')
 
     parserOpt.add_argument('--interactionFileFolder', '-iff',
-                           help='Folder where the interaction files are stored. Applies only for batch mode.',
+                           help='Folder where the interaction files are stored. Applies only for batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--outputFolder', '-o',
-                           help='Output folder of the files.',
+                           help='Output folder of the files'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='differentialResults')
     parserOpt.add_argument('--statisticTest',
-                           help='Type of test used: fisher\'s exact test or chi2 contingency',
+                           help='Type of test used: fisher\'s exact test or chi2 contingency'
+                           ' (Default: %(default)s).',
                            choices=['fisher', 'chi2'],
                            default='fisher')
     parserOpt.add_argument('--batchMode', '-bm',
                            help='turn on batch mode. The given file for --interactionFile and or --targetFile contain a list of the to be processed files.',
                            required=False,
                            action='store_true')
     parserOpt.add_argument('--threads', '-t',
-                           help='Number of threads (uses the python multiprocessing module)',
+                           help='Number of threads (uses the python multiprocessing module)'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=4,
                            type=int
                            )
     parserOpt.add_argument('--rejectedFileNamesToFile', '-r',
-                           help='Writes the names of the rejected H0 (therefore containing the differential interactions) to file. Can be used for batch processing mode of chicPlotViewpoint.',
+                           help='Writes the names of the rejected H0 (therefore containing the differential interactions) to file. Can be used for batch processing mode of chicPlotViewpoint.'
+                           ' (Default: %(default)s).',
                            default='rejected_H0.txt')
     parserOpt.add_argument("--help", "-h", action="help",
                            help="show this help message and exit")

hicexplorer/chicPlotViewpoint.py

@@ -49,7 +49,6 @@ def parse_arguments(args=None):
                                 'Format is --region upstream downstream, e.g.: --region 500000 500000 plots 500kb up- and 500kb downstream. This value should not exceed the range used in the other chic-tools.',
                                 required=True,
                                 type=int,
-                                default=[500000, 500000],
                                 nargs=2)
 
     parserOpt = parser.add_argument_group('Optional arguments')
@@ -58,19 +57,22 @@ def parse_arguments(args=None):
                            help='path to the background file which should be used for plotting',
                            required=False)
     parserOpt.add_argument('--interactionFileFolder', '-iff',
-                           help='Folder where the interaction files are stored. Applies only for batch mode.',
+                           help='Folder where the interaction files are stored. Applies only for batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--differentialTestResult', '-dif',
                            help='Path to the H0 rejected files to highlight the regions in the plot.',
                            required=False,
                            nargs='+')
     parserOpt.add_argument('--significantInteractionFileFolder', '-siff',
-                           help='Folder where the files with detected significant interactions are stored. Applies only for batch mode.',
+                           help='Folder where the files with detected significant interactions are stored. Applies only for batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--differentialTestResultsFolder', '-diff',
-                           help='Folder where the H0 rejected files are stored. Applies only for batch mode.',
+                           help='Folder where the H0 rejected files are stored. Applies only for batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--significantInteractions', '-si',
@@ -82,36 +84,43 @@ def parse_arguments(args=None):
                            required=False,
                            action='store_true')
     parserOpt.add_argument('--outputFolder', '-of',
-                           help='Output folder of the files.',
+                           help='Output folder of the files'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='.')
     parserOpt.add_argument('--outputFormat', '-format',
-                           help='Output format of the plot.',
+                           help='Output format of the plot'
+                           ' (Default: %(default)s).',
                            required=False,
                            default='png')
     parserOpt.add_argument('--dpi',
                            help='Optional parameter: Resolution for the image, if'
-                           'output is a raster graphics image (e.g png, jpg)',
+                           'output is a raster graphics image (e.g png, jpg)'
+                           ' (Default: %(default)s).',
                            type=int,
                            default=300,
                            required=False)
     parserOpt.add_argument('--binResolution', '-r',
-                           help='Resolution of the bin in genomic units. Values are set as number of bases, e.g. 1000 for a 1kb, 5000 for a 5kb or 10000 for a 10kb resolution.',
+                           help='Resolution of the bin in genomic units. Values are set as number of bases, e.g. 1000 for a 1kb, 5000 for a 5kb or 10000 for a 10kb resolution'
+                           ' (Default: %(default)s).',
                            type=int,
                            default=1000,
                            required=False)
 
     parserOpt.add_argument('--colorMapPvalue',
                            help='Color map to use for the p-value. Available '
                            'values can be seen here: '
-                           'http://matplotlib.org/examples/color/colormaps_reference.html',
+                           'http://matplotlib.org/examples/color/colormaps_reference.html'
+                           ' (Default: %(default)s).',
                            default='RdYlBu')
     parserOpt.add_argument('--maxPValue', '-map',
-                           help='Maximal value for p-value. Values above this threshold are set to this value.',
+                           help='Maximal value for p-value. Values above this threshold are set to this value'
+                           ' (Default: %(default)s).',
                            type=float,
                            default=0.1)
     parserOpt.add_argument('--minPValue', '-mp',
-                           help='Minimal value for p-value. Values below this threshold are set to this value.',
+                           help='Minimal value for p-value. Values below this threshold are set to this value'
+                           ' (Default: %(default)s).',
                            type=float,
                            default=0.0)
 
@@ -139,18 +148,20 @@ def parse_arguments(args=None):
                            required=False,
                            action='store_true')
     parserOpt.add_argument('--plotSampleNumber', '-psn',
-                           help='Number of samples per plot. Applies only in batch mode.',
+                           help='Number of samples per plot. Applies only in batch mode'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=2,
                            type=int)
     parserOpt.add_argument('--colorList', '-cl',
-                           help='Colorlist for the viewpoint lines.',
+                           help='Colorlist for the viewpoint lines (Default g b c m y k).',
                            required=False,
                            default=['g', 'b', 'c', 'm', 'y', 'k'],
                            type=str,
                            nargs='+')
     parserOpt.add_argument('--threads', '-t',
-                           help='Number of threads (uses the python multiprocessing module).',
+                           help='Number of threads (uses the python multiprocessing module)'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=4,
                            type=int

hicexplorer/chicQualityControl.py

@@ -53,29 +53,35 @@ def parse_arguments(args=None):
     parserOpt = parser.add_argument_group('Optional arguments')
 
     parserOpt.add_argument('--outFileName', '-o',
-                           help='The output file name of the passed reference points. Used as prefix for the plots as well.',
+                           help='The output file name of the passed reference points. Used as prefix for the plots as well'
+                           ' (Default: %(default)s).',
                            default='new_referencepoints.bed')
     parserOpt.add_argument('--outFileNameHistogram', '-oh',
-                           help='The output file for the histogram plot.',
+                           help='The output file for the histogram plot'
+                           ' (Default: %(default)s).',
                            default='histogram.png')
     parserOpt.add_argument('--outFileNameSparsity', '-os',
-                           help='The output file for the sparsity distribution plot.',
+                           help='The output file for the sparsity distribution plot'
+                           ' (Default: %(default)s).',
                            default='sparsity.png')
     parserOpt.add_argument('--threads', '-t',
-                           help='Number of threads.',
+                           help='Number of threads'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=4,
                            type=int
                            )
     parserOpt.add_argument('--fixateRange', '-fs',
-                           help='Fixate score of background model starting at distance x. E.g. all values greater than 500kb are set to the value of the 500kb bin.',
+                           help='Fixate score of background model starting at distance x. E.g. all values greater than 500kb are set to the value of the 500kb bin'
+                           ' (Default: %(default)s).',
                            required=False,
                            default=500000,
                            type=int
                            )
     parserOpt.add_argument('--dpi',
                            help='Optional parameter: Resolution for the image if the'
-                           'output is a raster graphics image (e.g png, jpg)',
+                           'output is a raster graphics image (e.g png, jpg)'
+                           ' (Default: %(default)s).',
                            type=int,
                            default=300,
                            required=False)