From 5ee3624961310f03f0c6629472ef449e86a53309 Mon Sep 17 00:00:00 2001
From: Maciej Bak <maciej.bak@unibas.ch>
Date: Thu, 19 Aug 2021 16:44:00 +0200
Subject: [PATCH] refactor: remove cluster config path hardcoding
---
profiles/slurm-conda/config.yaml | 1 -
profiles/slurm-config.json | 2 +-
profiles/slurm-singularity/config.yaml | 1 -
resources/config_schema.json | 2 +-
tests/input_files/config.yaml | 1 +
workflow/Snakefile | 30 +++++++++++++++++++++++++
workflow/rules/paired_end.snakefile.smk | 5 +++++
workflow/rules/single_end.snakefile.smk | 5 +++++
8 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/profiles/slurm-conda/config.yaml b/profiles/slurm-conda/config.yaml
index dcf0976..2fb0969 100644
--- a/profiles/slurm-conda/config.yaml
+++ b/profiles/slurm-conda/config.yaml
@@ -1,7 +1,6 @@
jobscript: "../slurm-jobscript.sh"
cluster: "../slurm-submit.py"
cluster-status: "../slurm-status.py"
-config: "cluster_logs='logs/cluster'"
snakefile: "../../workflow/Snakefile"
cores: 256
jobs: 256
diff --git a/profiles/slurm-config.json b/profiles/slurm-config.json
index 2de6092..c64e5af 100644
--- a/profiles/slurm-config.json
+++ b/profiles/slurm-config.json
@@ -6,7 +6,7 @@
"cpus-per-task": "1",
"mem": "4G",
"job-name": "{rule}.{wildcards}",
- "output": "logs/cluster/{rule}.{wildcards}-%j-%N.out"
+ "output": "{params.cluster_log_path}/{rule}.{wildcards}-%j-%N.out"
},
"create_index_star":
{
diff --git a/profiles/slurm-singularity/config.yaml b/profiles/slurm-singularity/config.yaml
index 960bee6..4fc0a6b 100644
--- a/profiles/slurm-singularity/config.yaml
+++ b/profiles/slurm-singularity/config.yaml
@@ -1,7 +1,6 @@
jobscript: "../slurm-jobscript.sh"
cluster: "../slurm-submit.py"
cluster-status: "../slurm-status.py"
-config: "cluster_logs='logs/cluster'"
snakefile: "../../workflow/Snakefile"
cores: 256
jobs: 256
diff --git a/resources/config_schema.json b/resources/config_schema.json
index 299c43a..d5b6ab9 100644
--- a/resources/config_schema.json
+++ b/resources/config_schema.json
@@ -17,7 +17,7 @@
"type": "string",
"description": "Path to log directory."
},
- "cluster_logs": {
+ "cluster_log_dir": {
"type": "string",
"description": "Path to cluster log directory."
},
diff --git a/tests/input_files/config.yaml b/tests/input_files/config.yaml
index 1fc480c..06a9704 100644
--- a/tests/input_files/config.yaml
+++ b/tests/input_files/config.yaml
@@ -3,6 +3,7 @@
samples: "../input_files/samples.tsv"
output_dir: "results"
log_dir: "logs"
+ cluster_log_dir: "logs/cluster"
kallisto_indexes: "results/kallisto_indexes"
salmon_indexes: "results/salmon_indexes"
star_indexes: "results/star_indexes"
diff --git a/workflow/Snakefile b/workflow/Snakefile
index fb6b5a1..6a53b1f 100644
--- a/workflow/Snakefile
+++ b/workflow/Snakefile
@@ -164,6 +164,9 @@ rule start:
"{sample}",
"start",
"{sample}.{mate}.fastq.gz")
+
+ params:
+ cluster_log_path = config["cluster_log_dir"]
log:
stderr = os.path.join(
@@ -208,6 +211,7 @@ rule fastqc:
"{mate}"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -278,6 +282,7 @@ rule create_index_star:
"chrName.txt")
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config['star_indexes'],
"{organism}",
@@ -357,6 +362,7 @@ rule extract_transcriptome:
"transcriptome.fa"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -413,6 +419,9 @@ rule concatenate_transcriptome_and_genome:
"transcriptome",
"{organism}",
"genome_transcriptome.fa"))
+
+ params:
+ cluster_log_path = config["cluster_log_dir"]
singularity:
"docker://ubuntu:focal-20210416"
@@ -456,6 +465,7 @@ rule create_index_salmon:
"salmon.idx"))
params:
+ cluster_log_path = config["cluster_log_dir"],
kmerLen = "{kmer}",
additional_params = parse_rule_config(
rule_config,
@@ -514,6 +524,7 @@ rule create_index_kallisto:
"kallisto.idx")
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config['kallisto_indexes'],
"{organism}"),
@@ -564,6 +575,7 @@ rule extract_transcripts_as_bed12:
"full_transcripts_protein_coding.bed"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -618,6 +630,7 @@ rule index_genomic_alignment_samtools:
"{sample}.{seqmode}.Aligned.sortedByCoord.out.bam.bai")
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -696,6 +709,7 @@ rule calculate_TIN_scores:
"TIN_score.tsv"))
params:
+ cluster_log_path = config["cluster_log_dir"],
sample = "{sample}",
additional_params = parse_rule_config(
rule_config,
@@ -760,6 +774,7 @@ rule salmon_quantmerge_genes:
"genes_{salmon_merge_on}.tsv")
params:
+ cluster_log_path = config["cluster_log_dir"],
salmon_in = expand(
os.path.join(
config["output_dir"],
@@ -846,6 +861,7 @@ rule salmon_quantmerge_transcripts:
"transcripts_{salmon_merge_on}.tsv")
params:
+ cluster_log_path = config["cluster_log_dir"],
salmon_in = expand(
os.path.join(
config["output_dir"],
@@ -935,6 +951,7 @@ rule kallisto_merge_genes:
"genes_counts.tsv")
params:
+ cluster_log_path = config["cluster_log_dir"],
dir_out = os.path.join(
config["output_dir"],
"summary_kallisto"),
@@ -1020,6 +1037,7 @@ rule kallisto_merge_transcripts:
"transcripts_counts.tsv")
params:
+ cluster_log_path = config["cluster_log_dir"],
dir_out = os.path.join(
config["output_dir"],
"summary_kallisto"),
@@ -1086,6 +1104,7 @@ rule pca_salmon:
"pca_salmon_{molecule}"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -1135,6 +1154,7 @@ rule pca_kallisto:
"pca_kallisto_{molecule}"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -1230,6 +1250,7 @@ rule star_rpm:
shadow: "full"
params:
+ cluster_log_path = config["cluster_log_dir"],
out_dir = lambda wildcards, output:
os.path.dirname(output.str1),
prefix = lambda wildcards, output:
@@ -1327,6 +1348,9 @@ rule rename_star_rpm_for_alfa:
"ALFA",
"{unique}",
"{sample}.{unique}.minus.bg"))
+
+ params:
+ cluster_log_path = config["cluster_log_dir"]
log:
stderr = os.path.join(
@@ -1380,6 +1404,7 @@ rule generate_alfa_index:
"sorted_genes.unstranded.ALFA_index")
params:
+ cluster_log_path = config["cluster_log_dir"],
genome_index = "sorted_genes",
out_dir = lambda wildcards, output:
os.path.dirname(output.index_stranded),
@@ -1475,6 +1500,7 @@ rule alfa_qc:
"{sample}.ALFA_feature_counts.tsv")
params:
+ cluster_log_path = config["cluster_log_dir"],
out_dir = lambda wildcards, output:
os.path.dirname(output.biotypes),
genome_index = lambda wildcards, input:
@@ -1542,6 +1568,7 @@ rule prepare_multiqc_config:
"multiqc_config.yaml")
params:
+ cluster_log_path = config["cluster_log_dir"],
logo_path = config['report_logo'],
multiqc_intro_text = config['report_description'],
url = config['report_url'],
@@ -1661,6 +1688,7 @@ rule multiqc_report:
"multiqc_summary"))
params:
+ cluster_log_path = config["cluster_log_dir"],
results_dir = os.path.join(
config["output_dir"]),
log_dir = config["log_dir"],
@@ -1720,6 +1748,7 @@ rule sort_bed_4_big:
"{sample}_{unique}_{strand}.sorted.bg"))
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
@@ -1785,6 +1814,7 @@ rule prepare_bigWig:
"{sample}_{unique}_{strand}.bw")
params:
+ cluster_log_path = config["cluster_log_dir"],
additional_params = parse_rule_config(
rule_config,
current_rule=current_rule,
diff --git a/workflow/rules/paired_end.snakefile.smk b/workflow/rules/paired_end.snakefile.smk
index 76a88f1..d0a9bc2 100644
--- a/workflow/rules/paired_end.snakefile.smk
+++ b/workflow/rules/paired_end.snakefile.smk
@@ -31,6 +31,7 @@ rule pe_remove_adapters_cutadapt:
"{sample}.pe.remove_adapters_mate2.fastq.gz"))
params:
+ cluster_log_path = config["cluster_log_dir"],
adapter_3_mate1 = lambda wildcards:
get_sample('fq1_3p', search_id='index', search_value=wildcards.sample),
adapter_5_mate1 = lambda wildcards:
@@ -118,6 +119,7 @@ rule pe_remove_polya_cutadapt:
"{sample}.pe.remove_polya_mate2.fastq.gz"))
params:
+ cluster_log_path = config["cluster_log_dir"],
polya_3_mate1 = lambda wildcards:
get_sample(
'fq1_polya_3p',
@@ -234,6 +236,7 @@ rule pe_map_genome_star:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
sample_id = "{sample}",
index = lambda wildcards:
os.path.abspath(os.path.join(
@@ -385,6 +388,7 @@ rule pe_quantification_salmon:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config["output_dir"],
"samples",
@@ -485,6 +489,7 @@ rule pe_genome_quantification_kallisto:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config["output_dir"],
"samples",
diff --git a/workflow/rules/single_end.snakefile.smk b/workflow/rules/single_end.snakefile.smk
index cd7bb1d..7dfb2c2 100644
--- a/workflow/rules/single_end.snakefile.smk
+++ b/workflow/rules/single_end.snakefile.smk
@@ -19,6 +19,7 @@ rule remove_adapters_cutadapt:
"{sample}.se.remove_adapters_mate1.fastq.gz"))
params:
+ cluster_log_path = config["cluster_log_dir"],
adapters_3 = lambda wildcards:
get_sample(
'fq1_3p',
@@ -93,6 +94,7 @@ rule remove_polya_cutadapt:
"{sample}.se.remove_polya_mate1.fastq.gz"))
params:
+ cluster_log_path = config["cluster_log_dir"],
polya_3 = lambda wildcards:
get_sample(
'fq1_polya_3p',
@@ -184,6 +186,7 @@ rule map_genome_star:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
sample_id = "{sample}",
index = lambda wildcards:
os.path.abspath(os.path.join(
@@ -324,6 +327,7 @@ rule quantification_salmon:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config["output_dir"],
"samples",
@@ -428,6 +432,7 @@ rule genome_quantification_kallisto:
shadow: "minimal"
params:
+ cluster_log_path = config["cluster_log_dir"],
output_dir = os.path.join(
config["output_dir"],
"samples",
--
GitLab