Skip to content
Snippets Groups Projects
Commit 6a231363 authored by Alex Kanitz's avatar Alex Kanitz
Browse files

clean up tests

- set up integration test for Snakefile in dedicated folder; current test case was left untouched for the time being, despite requiring large input files
- set up DAG chart creation test in dedicated folder; script creates an SVG representation of the workflow DAG at `images/workflow_dag.svg`
- both tests have been added to the GitLab CI/CD configuration; the latter test ensures that always the latest version of the
- all tests are now located inside subdirectories of `tests/`; test scripts and configuration files for test runs etc. have been moved to the appropriate test directories
- for the time being, required input files for each test are placed within the individual test directories; a layout for common test files should be introduced later and paths and bind paths in tests adapted
- make script `scripts/labkey_api.py` executable
parent 3d9a272c
No related branches found
No related tags found
No related merge requests found
Showing
with 129 additions and 35 deletions
......@@ -5,10 +5,11 @@ before_script:
test:
script:
# add code quality tests here
# add unit tests here
# add script tests here
- bash tests/test_scripts_labkey_to_snakemake/test.sh
- cd snakemake # fix this in future version; all tests
- snakemake -n # should be called from home directory
- bash run_test.sh #
- cd .. #
# add additional tests here
# add integration tests here
- bash tests/test_create_dag_chart/test.sh
- bash tests/test_integration_workflow/test.local.sh
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.42.3 (20191010.1750)
-->
<!-- Title: snakemake_dag Pages: 1 -->
<svg width="699pt" height="188pt"
viewBox="0.00 0.00 699.00 188.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 184)">
<title>snakemake_dag</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-184 695,-184 695,4 -4,4"/>
<!-- 0 -->
<g id="node1" class="node">
<title>0</title>
<path fill="none" stroke="#d88d56" stroke-width="2" d="M357.5,-36C357.5,-36 327.5,-36 327.5,-36 321.5,-36 315.5,-30 315.5,-24 315.5,-24 315.5,-12 315.5,-12 315.5,-6 321.5,0 327.5,0 327.5,0 357.5,0 357.5,0 363.5,0 369.5,-6 369.5,-12 369.5,-12 369.5,-24 369.5,-24 369.5,-30 363.5,-36 357.5,-36"/>
<text text-anchor="middle" x="342.5" y="-15.5" font-family="sans" font-size="10.00">finish</text>
</g>
<!-- 1 -->
<g id="node2" class="node">
<title>1</title>
<path fill="none" stroke="#56d873" stroke-width="2" d="M155,-108C155,-108 12,-108 12,-108 6,-108 0,-102 0,-96 0,-96 0,-84 0,-84 0,-78 6,-72 12,-72 12,-72 155,-72 155,-72 161,-72 167,-78 167,-84 167,-84 167,-96 167,-96 167,-102 161,-108 155,-108"/>
<text text-anchor="middle" x="83.5" y="-93" font-family="sans" font-size="10.00">pe_fastqc</text>
<text text-anchor="middle" x="83.5" y="-82" font-family="sans" font-size="10.00">sample: HNRNPC_control_rep1</text>
</g>
<!-- 1&#45;&gt;0 -->
<g id="edge1" class="edge">
<title>1&#45;&gt;0</title>
<path fill="none" stroke="grey" stroke-width="2" d="M146.53,-71.97C196.33,-58.51 264.12,-40.18 305.52,-29"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="306.45,-32.37 315.19,-26.38 304.63,-25.61 306.45,-32.37"/>
</g>
<!-- 2 -->
<g id="node3" class="node">
<title>2</title>
<path fill="none" stroke="#56d873" stroke-width="2" d="M320,-108C320,-108 197,-108 197,-108 191,-108 185,-102 185,-96 185,-96 185,-84 185,-84 185,-78 191,-72 197,-72 197,-72 320,-72 320,-72 326,-72 332,-78 332,-84 332,-84 332,-96 332,-96 332,-102 326,-108 320,-108"/>
<text text-anchor="middle" x="258.5" y="-93" font-family="sans" font-size="10.00">pe_fastqc</text>
<text text-anchor="middle" x="258.5" y="-82" font-family="sans" font-size="10.00">sample: HNRNPC_KD_rep1</text>
</g>
<!-- 2&#45;&gt;0 -->
<g id="edge2" class="edge">
<title>2&#45;&gt;0</title>
<path fill="none" stroke="grey" stroke-width="2" d="M279.26,-71.7C289.74,-62.97 302.61,-52.24 314,-42.75"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="316.53,-45.19 321.97,-36.1 312.05,-39.82 316.53,-45.19"/>
</g>
<!-- 3 -->
<g id="node4" class="node">
<title>3</title>
<path fill="none" stroke="#afd856" stroke-width="2" d="M490.5,-108C490.5,-108 362.5,-108 362.5,-108 356.5,-108 350.5,-102 350.5,-96 350.5,-96 350.5,-84 350.5,-84 350.5,-78 356.5,-72 362.5,-72 362.5,-72 490.5,-72 490.5,-72 496.5,-72 502.5,-78 502.5,-84 502.5,-84 502.5,-96 502.5,-96 502.5,-102 496.5,-108 490.5,-108"/>
<text text-anchor="middle" x="426.5" y="-87.5" font-family="sans" font-size="10.00">pe_remove_polya_cutadapt</text>
</g>
<!-- 3&#45;&gt;0 -->
<g id="edge3" class="edge">
<title>3&#45;&gt;0</title>
<path fill="none" stroke="grey" stroke-width="2" d="M405.74,-71.7C395.26,-62.97 382.39,-52.24 371,-42.75"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="372.95,-39.82 363.03,-36.1 368.47,-45.19 372.95,-39.82"/>
</g>
<!-- 4 -->
<g id="node5" class="node">
<title>4</title>
<path fill="none" stroke="#afd856" stroke-width="2" d="M667.5,-108C667.5,-108 539.5,-108 539.5,-108 533.5,-108 527.5,-102 527.5,-96 527.5,-96 527.5,-84 527.5,-84 527.5,-78 533.5,-72 539.5,-72 539.5,-72 667.5,-72 667.5,-72 673.5,-72 679.5,-78 679.5,-84 679.5,-84 679.5,-96 679.5,-96 679.5,-102 673.5,-108 667.5,-108"/>
<text text-anchor="middle" x="603.5" y="-87.5" font-family="sans" font-size="10.00">pe_remove_polya_cutadapt</text>
</g>
<!-- 4&#45;&gt;0 -->
<g id="edge4" class="edge">
<title>4&#45;&gt;0</title>
<path fill="none" stroke="grey" stroke-width="2" d="M539.99,-71.97C489.66,-58.47 421.11,-40.08 379.42,-28.9"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="380.25,-25.5 369.69,-26.29 378.44,-32.26 380.25,-25.5"/>
</g>
<!-- 5 -->
<g id="node6" class="node">
<title>5</title>
<path fill="none" stroke="#5692d8" stroke-width="2" d="M494,-180C494,-180 351,-180 351,-180 345,-180 339,-174 339,-168 339,-168 339,-156 339,-156 339,-150 345,-144 351,-144 351,-144 494,-144 494,-144 500,-144 506,-150 506,-156 506,-156 506,-168 506,-168 506,-174 500,-180 494,-180"/>
<text text-anchor="middle" x="422.5" y="-165" font-family="sans" font-size="10.00">pe_remove_adapters_cutadapt</text>
<text text-anchor="middle" x="422.5" y="-154" font-family="sans" font-size="10.00">sample: HNRNPC_control_rep1</text>
</g>
<!-- 5&#45;&gt;3 -->
<g id="edge5" class="edge">
<title>5&#45;&gt;3</title>
<path fill="none" stroke="grey" stroke-width="2" d="M423.49,-143.7C423.93,-135.98 424.46,-126.71 424.95,-118.11"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="428.45,-118.29 425.52,-108.1 421.46,-117.89 428.45,-118.29"/>
</g>
<!-- 6 -->
<g id="node7" class="node">
<title>6</title>
<path fill="none" stroke="#5692d8" stroke-width="2" d="M679,-180C679,-180 536,-180 536,-180 530,-180 524,-174 524,-168 524,-168 524,-156 524,-156 524,-150 530,-144 536,-144 536,-144 679,-144 679,-144 685,-144 691,-150 691,-156 691,-156 691,-168 691,-168 691,-174 685,-180 679,-180"/>
<text text-anchor="middle" x="607.5" y="-165" font-family="sans" font-size="10.00">pe_remove_adapters_cutadapt</text>
<text text-anchor="middle" x="607.5" y="-154" font-family="sans" font-size="10.00">sample: HNRNPC_KD_rep1</text>
</g>
<!-- 6&#45;&gt;4 -->
<g id="edge6" class="edge">
<title>6&#45;&gt;4</title>
<path fill="none" stroke="grey" stroke-width="2" d="M606.51,-143.7C606.07,-135.98 605.54,-126.71 605.05,-118.11"/>
<polygon fill="grey" stroke="grey" stroke-width="2" points="608.54,-117.89 604.48,-108.1 601.55,-118.29 608.54,-117.89"/>
</g>
</g>
</svg>
File mode changed from 100644 to 100755
snakemake --dag -np | dot -Tpng > dag.png
# set -e
mkdir -p logs/cluster_log
mkdir -p logs/local_log
snakemake \
--cluster-config cluster.json \
--cluster "sbatch --cpus-per-task={cluster.threads} --mem={cluster.mem} --qos={cluster.queue} --time={cluster.time} --job-name={cluster.name} -o {cluster.out} -p scicore" \
--cores 256 \
-p \
--rerun-incomplete \
--use-singularity \
--singularity-args "--bind ${PWD}"
# set -e
mkdir -p logs/cluster_log
mkdir -p logs/local_log
snakemake \
--cores 4 \
-p \
--rerun-incomplete \
--use-singularity \
--singularity-args "--bind ${PWD}/../tests/input_files"
sample fq1 fq2 fq1_3p fq1_5p fq2_3p fq2_5p fq1_polya fq2_polya organism index_size multimappers soft_clip pass_mode gtf_filtered libtype kallisto_directionality mean sd genome gtf tr_fasta_filtered kmer seqmode
HNRNPC_control_rep1 ../tests/input_files/GSM1502498_1.fastq.gz ../tests/input_files/GSM1502498_2.fastq.gz AGATCGGAAGAGCACA XXXXXXXX AGATCGGAAGAGCGT XXXXXXXX AAAAAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTTT homo_sapiens 100 10 Local Basic ../tests/input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf A kallisto_directionality 250 100 ../tests/input_files/Homo_sapiens.GRCh38.dna_sm.primary_assembly.chrom_22.fa ../tests/input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf ../tests/input_files/transcripts_chrom_22.fa 31 fr
HNRNPC_KD_rep1 ../tests/input_files/GSM1502500_1.fastq.gz ../tests/input_files/GSM1502500_2.fastq.gz AGATCGGAAGAGCACA XXXXXXXX AGATCGGAAGAGCGT XXXXXXXX AAAAAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTTT homo_sapiens 100 10 Local Basic ../tests/input_files/Homo_sapiens.GRCh38.99.chr.filterd.chrom_22.gtf A kallisto_directionality 250 100 ../tests/input_files/Homo_sapiens.GRCh38.dna_sm.primary_assembly.chrom_22.fa ../tests/input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf ../tests/input_files/transcripts_chrom_22.fa 31 fr
FileNotFoundError in line 13 of /scicore/home/zavolan/kanitz/SANDBOX/rnaseqpipeline/snakemake/Snakefile:
[Errno 2] No such file or directory: 'input_files/samples.tsv'
File "/scicore/home/zavolan/kanitz/SANDBOX/rnaseqpipeline/snakemake/Snakefile", line 13, in <module>
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/parsers.py", line 676, in parser_f
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/parsers.py", line 448, in _read
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/parsers.py", line 880, in __init__
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/parsers.py", line 1126, in _make_engine
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/parsers.py", line 2269, in __init__
File "/scicore/home/zavolan/kanitz/REMOVE_FROM_HOME/soft/miniconda3/envs/rnaseq_pipeline/lib/python3.6/site-packages/pandas/io/common.py", line 431, in get_handle
Building DAG of jobs...
MissingInputException in line 2 of /scicore/home/zavolan/kanitz/SANDBOX/rnaseqpipeline/snakemake/paired_end.snakefile:
Missing input files for rule pe_fastqc:
input_files/GSM1502498_1.fastq.gz
input_files/GSM1502498_2.fastq.gz
Building DAG of jobs...
MissingInputException in line 2 of /scicore/home/zavolan/kanitz/SANDBOX/rnaseqpipeline/snakemake/paired_end.snakefile:
Missing input files for rule pe_fastqc:
input_files/GSM1502500_1.fastq.gz
input_files/GSM1502500_2.fastq.gz
Building DAG of jobs...
Building DAG of jobs...
Building DAG of jobs...
Building DAG of jobs...
......@@ -13,7 +13,7 @@
##############################################################################
### Output and log directories
##############################################################################
database_path: "/scicore/home/zavolan/GROUP/Rna_Seq_pipeline/Blabla"
database_path: "/some/path"
STAR_idx_folder: "STAR_indices"
output_dir: "results"
star_indexes: "results"
......@@ -25,5 +25,5 @@
##############################################################################
### Sample info
##############################################################################
samples: "../tests/samples.tsv"
samples: "input_files/samples.tsv"
...
sample fq1 fq2 fq1_3p fq1_5p fq2_3p fq2_5p fq1_polya fq2_polya organism index_size multimappers soft_clip pass_mode gtf_filtered libtype kallisto_directionality mean sd genome gtf tr_fasta_filtered kmer seqmode
HNRNPC_control_rep1 input_files/GSM1502498_1.fastq.gz input_files/GSM1502498_2.fastq.gz AGATCGGAAGAGCACA XXXXXXXX AGATCGGAAGAGCGT XXXXXXXX AAAAAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTTT homo_sapiens 100 10 Local Basic input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf A kallisto_directionality 250 100 input_files/Homo_sapiens.GRCh38.dna_sm.primary_assembly.chrom_22.fa input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf input_files/transcripts_chrom_22.fa 31 fr
HNRNPC_KD_rep1 input_files/GSM1502500_1.fastq.gz input_files/GSM1502500_2.fastq.gz AGATCGGAAGAGCACA XXXXXXXX AGATCGGAAGAGCGT XXXXXXXX AAAAAAAAAAAAAAAAAAAA TTTTTTTTTTTTTTTTTTT homo_sapiens 100 10 Local Basic input_files/Homo_sapiens.GRCh38.99.chr.filterd.chrom_22.gtf A kallisto_directionality 250 100 input_files/Homo_sapiens.GRCh38.dna_sm.primary_assembly.chrom_22.fa input_files/Homo_sapiens.GRCh38.99.chr.chrom_22.gtf input_files/transcripts_chrom_22.fa 31 fr
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment