test: fix Slurm submission

RUNS/JOB/map/cluster.json

@@ -47,7 +47,7 @@
     "threads":"{resources.threads}"
   },
 
-   "sort_genome_oligomap":
+  "sort_genome_oligomap":
   {
     "time": "{resources.time}:00:00",
     "threads":"{resources.threads}"

RUNS/JOB/map/run_workflow_slurm.sh

@@ -18,10 +18,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 cd $script_dir
 
 # Have to match directories indicated in config.yaml
-mkdir -p logs/cluster
-mkdir -p logs/local
-mkdir -p results
-
+mkdir -p logs/cluster/{sample_1,sample_2}
+mkdir -p logs/local/{sample_1,sample_2}
+mkdir -p results/{sample_1,sample_2}
 
 # Run workflow
 snakemake \
@@ -40,6 +39,7 @@ snakemake \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../../../" \
     --jobscript="../../../jobscript.sh" \
+    --jobs=20 \
     --cores=256 \
     --printshellcmds \
     --rerun-incomplete \

RUNS/JOB/prepare/run_workflow_slurm.sh

@@ -17,9 +17,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 cd $script_dir
 
 # Have to match directories indicated in config.yaml
-mkdir -p logs/cluster
-mkdir -p logs/local
-mkdir -p results
+mkdir -p logs/cluster/org/pre
+mkdir -p logs/local/org/pre
+mkdir -p results/org/pre
 
 # Run workflow
 snakemake \
@@ -38,6 +38,7 @@ snakemake \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../../../" \
     --jobscript="../../../jobscript.sh" \
+    --jobs=20 \
     --cores=256 \
     --printshellcmds \
     --rerun-incomplete \

RUNS/JOB/quantify/cluster.json

@@ -7,9 +7,9 @@
     "mem": "4G"
   },
 
-    "sortaligment":
+  "sort_alignment":
   {
     "mem":"{resources.mem}G",
     "threads":"{resources.threads}"
   }
-}
\ No newline at end of file
+}

RUNS/JOB/quantify/run_workflow_slurm.sh

@@ -38,6 +38,7 @@ snakemake \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../../../" \
     --jobscript="../../../jobscript.sh" \
+    --jobs=20 \
     --cores=256 \
     --printshellcmds \
     --rerun-incomplete \

test/cluster.json

+{
+  "__default__" :
+  {
+    "queue": "30min",
+    "time": "00:05:00",
+    "threads": "1",
+    "mem": "4G"
+  },
+
+  "cutadapt":
+  {
+    "threads":"{resources.threads}"
+  },
+
+  "mapping_genome_segemehl":
+  {
+    "mem":"{resources.mem}G"
+  },
+
+  "mapping_transcriptome_segemehl":
+  {
+    "mem":"{resources.mem}G"
+  },
+
+  "mapping_genome_oligomap":
+  {
+    "mem":"{resources.mem}G"
+  },
+
+  "mapping_transcriptome_oligomap":
+  {
+    "mem":"{resources.mem}G"
+  },
+
+  "sort_transcriptome_oligomap":
+  {
+    "threads":"{resources.threads}"
+  },
+
+  "sort_genome_oligomap":
+  {
+    "threads":"{resources.threads}"
+  },
+
+  "remove_inferiors":
+  {
+    "threads":"{resources.threads}",
+    "mem":"{resources.mem}G"
+  },
+
+  "generate_segemehl_index_transcriptome":
+  {
+    "threads":"{resources.threads}",
+    "mem":"{resources.mem}G"
+  },
+
+  "generate_segemehl_index_genome":
+  {
+    "threads":"{resources.threads}",
+    "mem":"{resources.mem}G"
+  },
+
+  "sort_alignment":
+  {
+    "mem":"{resources.mem}G",
+    "threads":"{resources.threads}"
+  }
+}

test/test_workflow_slurm.sh

@@ -15,15 +15,17 @@ set -x  # facilitates debugging by printing out executed commands
 user_dir=$PWD
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 cd $script_dir
-mkdir -p logs/cluster/{homo_sapiens/chrY,results/test_lib}
-mkdir -p logs/local/{homo_sapiens/chrY,results/test_lib}
-mkdir -p results/{homo_sapiens/chrY,results/test_lib}
+
+# Have to match directories indicated in config.yaml files
+mkdir -p logs/cluster/{homo_sapiens/chrY,test_lib}
+mkdir -p logs/local/{homo_sapiens/chrY,test_lib}
+mkdir -p results/{homo_sapiens/chrY,test_lib}
 
 # Run test: prepare workflow
 snakemake \
     --snakefile="../workflow/prepare/Snakefile" \
     --configfile="config_prepare.yaml" \
-    --cluster-config="../RUNS/JOB/prepare/cluster.json" \
+    --cluster-config="cluster.json" \
     --cluster "sbatch \
         --cpus-per-task={cluster.threads} \
         --mem={cluster.mem} \
@@ -34,6 +36,7 @@ snakemake \
         -p scicore \
         --open-mode=append" \
     --jobscript="../jobscript.sh" \
+    --jobs=20 \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../" \
     --cores=256 \
@@ -45,7 +48,7 @@ snakemake \
 snakemake \
     --snakefile="../workflow/map/Snakefile" \
     --configfile="config_map.yaml" \
-    --cluster-config="../RUNS/JOB/map/cluster.json" \
+    --cluster-config="cluster.json" \
     --cluster "sbatch \
         --cpus-per-task={cluster.threads} \
         --mem={cluster.mem} \
@@ -56,6 +59,7 @@ snakemake \
         -p scicore \
         --open-mode=append" \
     --jobscript="../jobscript.sh" \
+    --jobs=20 \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../" \
     --cores=256 \
@@ -67,7 +71,7 @@ snakemake \
 snakemake \
     --snakefile="../workflow/quantify/Snakefile" \
     --configfile="config_quantify.yaml" \
-    --cluster-config="../RUNS/JOB/quantify/cluster.json" \
+    --cluster-config="cluster.json" \
     --cluster "sbatch \
         --cpus-per-task={cluster.threads} \
         --mem={cluster.mem} \
@@ -78,6 +82,7 @@ snakemake \
         -p scicore \
         --open-mode=append" \
     --jobscript="../jobscript.sh" \
+    --jobs=20 \
     --use-singularity \
     --singularity-args="--no-home --bind ${PWD}/../" \
     --cores=256 \