From 880406dfe03f2e08bbf0cf9a2d6d6791d319307d Mon Sep 17 00:00:00 2001
From: Alex Kanitz <alexander.kanitz@alumni.ethz.ch>
Date: Fri, 8 Jul 2022 19:14:51 +0200
Subject: [PATCH] test: fix Slurm submission
---
RUNS/JOB/map/cluster.json | 2 +-
RUNS/JOB/map/run_workflow_slurm.sh | 8 +--
RUNS/JOB/prepare/run_workflow_slurm.sh | 7 +--
RUNS/JOB/quantify/cluster.json | 4 +-
RUNS/JOB/quantify/run_workflow_slurm.sh | 1 +
test/cluster.json | 68 +++++++++++++++++++++++++
test/test_workflow_slurm.sh | 17 ++++---
7 files changed, 91 insertions(+), 16 deletions(-)
create mode 100644 test/cluster.json
diff --git a/RUNS/JOB/map/cluster.json b/RUNS/JOB/map/cluster.json
index f771d03..8056f93 100644
--- a/RUNS/JOB/map/cluster.json
+++ b/RUNS/JOB/map/cluster.json
@@ -47,7 +47,7 @@
"threads":"{resources.threads}"
},
- "sort_genome_oligomap":
+ "sort_genome_oligomap":
{
"time": "{resources.time}:00:00",
"threads":"{resources.threads}"
diff --git a/RUNS/JOB/map/run_workflow_slurm.sh b/RUNS/JOB/map/run_workflow_slurm.sh
index 7f21a9e..8f23407 100755
--- a/RUNS/JOB/map/run_workflow_slurm.sh
+++ b/RUNS/JOB/map/run_workflow_slurm.sh
@@ -18,10 +18,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
cd $script_dir
# Have to match directories indicated in config.yaml
-mkdir -p logs/cluster
-mkdir -p logs/local
-mkdir -p results
-
+mkdir -p logs/cluster/{sample_1,sample_2}
+mkdir -p logs/local/{sample_1,sample_2}
+mkdir -p results/{sample_1,sample_2}
# Run workflow
snakemake \
@@ -40,6 +39,7 @@ snakemake \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../../../" \
--jobscript="../../../jobscript.sh" \
+ --jobs=20 \
--cores=256 \
--printshellcmds \
--rerun-incomplete \
diff --git a/RUNS/JOB/prepare/run_workflow_slurm.sh b/RUNS/JOB/prepare/run_workflow_slurm.sh
index f9f1f5a..e9138f6 100755
--- a/RUNS/JOB/prepare/run_workflow_slurm.sh
+++ b/RUNS/JOB/prepare/run_workflow_slurm.sh
@@ -17,9 +17,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
cd $script_dir
# Have to match directories indicated in config.yaml
-mkdir -p logs/cluster
-mkdir -p logs/local
-mkdir -p results
+mkdir -p logs/cluster/org/pre
+mkdir -p logs/local/org/pre
+mkdir -p results/org/pre
# Run workflow
snakemake \
@@ -38,6 +38,7 @@ snakemake \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../../../" \
--jobscript="../../../jobscript.sh" \
+ --jobs=20 \
--cores=256 \
--printshellcmds \
--rerun-incomplete \
diff --git a/RUNS/JOB/quantify/cluster.json b/RUNS/JOB/quantify/cluster.json
index ef1ea5b..29cdfb9 100644
--- a/RUNS/JOB/quantify/cluster.json
+++ b/RUNS/JOB/quantify/cluster.json
@@ -7,9 +7,9 @@
"mem": "4G"
},
- "sortaligment":
+ "sort_alignment":
{
"mem":"{resources.mem}G",
"threads":"{resources.threads}"
}
-}
\ No newline at end of file
+}
diff --git a/RUNS/JOB/quantify/run_workflow_slurm.sh b/RUNS/JOB/quantify/run_workflow_slurm.sh
index 15fd9d3..54b62b2 100755
--- a/RUNS/JOB/quantify/run_workflow_slurm.sh
+++ b/RUNS/JOB/quantify/run_workflow_slurm.sh
@@ -38,6 +38,7 @@ snakemake \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../../../" \
--jobscript="../../../jobscript.sh" \
+ --jobs=20 \
--cores=256 \
--printshellcmds \
--rerun-incomplete \
diff --git a/test/cluster.json b/test/cluster.json
new file mode 100644
index 0000000..0fbb1ef
--- /dev/null
+++ b/test/cluster.json
@@ -0,0 +1,68 @@
+{
+ "__default__" :
+ {
+ "queue": "30min",
+ "time": "00:05:00",
+ "threads": "1",
+ "mem": "4G"
+ },
+
+ "cutadapt":
+ {
+ "threads":"{resources.threads}"
+ },
+
+ "mapping_genome_segemehl":
+ {
+ "mem":"{resources.mem}G"
+ },
+
+ "mapping_transcriptome_segemehl":
+ {
+ "mem":"{resources.mem}G"
+ },
+
+ "mapping_genome_oligomap":
+ {
+ "mem":"{resources.mem}G"
+ },
+
+ "mapping_transcriptome_oligomap":
+ {
+ "mem":"{resources.mem}G"
+ },
+
+ "sort_transcriptome_oligomap":
+ {
+ "threads":"{resources.threads}"
+ },
+
+ "sort_genome_oligomap":
+ {
+ "threads":"{resources.threads}"
+ },
+
+ "remove_inferiors":
+ {
+ "threads":"{resources.threads}",
+ "mem":"{resources.mem}G"
+ },
+
+ "generate_segemehl_index_transcriptome":
+ {
+ "threads":"{resources.threads}",
+ "mem":"{resources.mem}G"
+ },
+
+ "generate_segemehl_index_genome":
+ {
+ "threads":"{resources.threads}",
+ "mem":"{resources.mem}G"
+ },
+
+ "sort_alignment":
+ {
+ "mem":"{resources.mem}G",
+ "threads":"{resources.threads}"
+ }
+}
diff --git a/test/test_workflow_slurm.sh b/test/test_workflow_slurm.sh
index 5ae9432..0c50de3 100755
--- a/test/test_workflow_slurm.sh
+++ b/test/test_workflow_slurm.sh
@@ -15,15 +15,17 @@ set -x # facilitates debugging by printing out executed commands
user_dir=$PWD
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
cd $script_dir
-mkdir -p logs/cluster/{homo_sapiens/chrY,results/test_lib}
-mkdir -p logs/local/{homo_sapiens/chrY,results/test_lib}
-mkdir -p results/{homo_sapiens/chrY,results/test_lib}
+
+# Have to match directories indicated in config.yaml files
+mkdir -p logs/cluster/{homo_sapiens/chrY,test_lib}
+mkdir -p logs/local/{homo_sapiens/chrY,test_lib}
+mkdir -p results/{homo_sapiens/chrY,test_lib}
# Run test: prepare workflow
snakemake \
--snakefile="../workflow/prepare/Snakefile" \
--configfile="config_prepare.yaml" \
- --cluster-config="../RUNS/JOB/prepare/cluster.json" \
+ --cluster-config="cluster.json" \
--cluster "sbatch \
--cpus-per-task={cluster.threads} \
--mem={cluster.mem} \
@@ -34,6 +36,7 @@ snakemake \
-p scicore \
--open-mode=append" \
--jobscript="../jobscript.sh" \
+ --jobs=20 \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../" \
--cores=256 \
@@ -45,7 +48,7 @@ snakemake \
snakemake \
--snakefile="../workflow/map/Snakefile" \
--configfile="config_map.yaml" \
- --cluster-config="../RUNS/JOB/map/cluster.json" \
+ --cluster-config="cluster.json" \
--cluster "sbatch \
--cpus-per-task={cluster.threads} \
--mem={cluster.mem} \
@@ -56,6 +59,7 @@ snakemake \
-p scicore \
--open-mode=append" \
--jobscript="../jobscript.sh" \
+ --jobs=20 \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../" \
--cores=256 \
@@ -67,7 +71,7 @@ snakemake \
snakemake \
--snakefile="../workflow/quantify/Snakefile" \
--configfile="config_quantify.yaml" \
- --cluster-config="../RUNS/JOB/quantify/cluster.json" \
+ --cluster-config="cluster.json" \
--cluster "sbatch \
--cpus-per-task={cluster.threads} \
--mem={cluster.mem} \
@@ -78,6 +82,7 @@ snakemake \
-p scicore \
--open-mode=append" \
--jobscript="../jobscript.sh" \
+ --jobs=20 \
--use-singularity \
--singularity-args="--no-home --bind ${PWD}/../" \
--cores=256 \
--
GitLab