Commit 3ffcd3f6 authored by Zdralovic Edim's avatar Zdralovic Edim

Merged

parents 974b8145 cd0f1347
......@@ -36,7 +36,8 @@ RUN apt-get -y install build-essential checkinstall \
&& tar xvf Python-3.6.0.tar.xz \
&& cd Python-3.6.0/ \
&& ./configure \
&& make altinstall
&& make altinstall \
&& apt-get install sqlite3
COPY Flask/ /krinicwl/
COPY requirements.txt /krinicwl/
......@@ -45,6 +46,4 @@ RUN pip3.6 install -r requirements.txt
WORKDIR /krinicwl/cwlREST
EXPOSE 5000
CMD ["python3.6", "cwlRESTService.py"]
\ No newline at end of file
CMD ["python3.6", "cwlRESTService.py"]
......@@ -27,7 +27,7 @@ import glob
app = Flask(__name__)
cors = CORS(app, resources={r"*": {"origins": "*"}})
UPLOAD_FOLDER = '../cwlREST/uploaded_files/'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif', 'yml', 'html'])
ALLOWED_EXTENSIONS = set(['txt', 'yml', 'html', 'yaml', 'cwl'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
wesParser = wes_parser.WESParser()
wesDAO = wes_dao.WES_DAO()
......@@ -39,15 +39,15 @@ class CWLRestAPI(FlaskView):
"""
This class represents the API for the Workflows.
Available routes:
/workflows
A POST on '/workflows/' expects two arguments in JSON: the token of the
A POST on '/workflows/' expects two arguments in JSON: the token of the
authenticated user and his role.
After successfully starting a workflow the ID for this workflow is yielded
which can be used to retrieve the status of this workflow.
A GET on this directive will yield all workflows for the user with their current status in JSON format.
/workflows/'workflowID'
A GET on this directive will yield all availaible workflow informations matching the given ID in JSON format.
......@@ -62,7 +62,12 @@ class CWLRestAPI(FlaskView):
route_base = '/' #flask_classful specific, class variable
def __init__(self):
self.baseDir = './workflows/' #for developing only
self.__wfManager = WorkflowManager()
#self.testCWL = './data/ex2.cwl' #for developing only
#self.testYML = './data/ex2.yml' #for developing only
#self.mirzaCWL = '/home/cuhoni66/MIRZAG/mirzag_seed.cwl'
#self.mirzaYML = '/home/cuhoni66/MIRZAG/job.yml'
self.__WorkflowIDCounter = 0
self.__wfManager = WorkflowManager()
############################################################
#Section: Routed Methods
############################################################
......@@ -96,7 +101,7 @@ class CWLRestAPI(FlaskView):
def postWorkflow(self):
"""
The data from the POST is decoded and a WorkflowWrapper object instantiated.
The WorkflowManager recieves this object and starts the thread, which will
The WorkflowManager recieves this object and starts the thread, which will
execute the toil-cwl-runner with the corresponding parameters.
"""
......@@ -127,8 +132,8 @@ class CWLRestAPI(FlaskView):
"""
The status of the workflow matching the given ID (toil Workflow ID) with all job infos is returned in JSON format.
"""
wf = self.__wfManager.restartWorkflow(workflowID)
return json.dumps(wf)
#wf = self.__wfManager.restartWorkflow(workflowID)
return json.dumps(self.__wfManager.restartWorkflow(workflowID))
@route('/workflows/<workflowID>', methods=['DELETE'])
def killWorkflow(self, workflowID):
......@@ -146,15 +151,15 @@ class CWLRestAPI(FlaskView):
wf = self.__wfManager.getWorkflowStatusByID(workflowID)
return json.dumps(wf)
#@route('/workflows/<userToken>', methods=['GET'])
#def getWorkflowsFromUser(self, userToken):
# """
# A list of all workflows for one user are returned in JSON format.
# """
# wfList = self.__wfManager.getAllWorkflows(str(userToken))
# return json.dumps(wfList)
@route('/workflows/user/<userToken>', methods=['GET'])
def getWorkflowsFromUser(self, userToken):
"""
A list of all workflows for one user are returned in JSON format.
"""
wfList = self.__wfManager.getAllWorkflows(str(userToken))
return json.dumps(wfList)
#TODO: service-info, what fields can we fill?
#TODO: service-info, what fields can we fill?
@route('/service-info', methods=['GET'])
def serviceInfo(self):
"""
......@@ -172,25 +177,25 @@ class CWLRestAPI(FlaskView):
filename = ""
# check if the post request has the file part
if 'file' not in request.files:
return {"error": "No file part found"}
return jsonify({"NoFilePartError": "No file part found."})
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
return {"error": "No selected file found"}
return jsonify({"NoFileError": "No selected file found."})
if not self.allowed_file(file.filename):
return {"error": "Given filetype not supported"}
return jsonify({"TypeError": "Given filetype not supported."})
if file:
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
return jsonify({"success": "File successfully uploaded", "filename": filename})
return jsonify({"Success": "File successfully uploaded.", "filename": filename})
else:
return jsonify({"error": "File could not be uploaded"})
return jsonify({"RequestError": "Wrong request method used."})
@route('/workflows/gitURL', methods=['POST'])
def postGit(self):
......@@ -214,10 +219,14 @@ class WorkflowManager:
This method recieves the incoming workflow from the API and starts it in a thread.
"""
self.workflowList.append(wf)
#wf.start()
#while not wf.workflowID:
# pass
return {"yaml":wf.yamlStream}
def startWorkflow(self, wfName, yamlStream):
def startWorkflow(self, wfName):
"""
This method recieves the incoming workflow from the API and starts it in a thread.
"""
......@@ -227,7 +236,7 @@ class WorkflowManager:
if i.workflowName == wfName:
wf = i
break
wf.yamlStream = yamlStream
wf.start()
while not wf.workflowID:
......@@ -241,6 +250,7 @@ class WorkflowManager:
If the pipeline was stopped by the user and the workflow has been edited, the workflow
has to start from scratch to actually apply these changes.
"""
#TODO: Restarting workflow, thread has to be started again (initializing new object, copying doesn't work)
wfToRestart = None
for i in self.workflowList:
......@@ -248,9 +258,8 @@ class WorkflowManager:
wfToRestart = i
break
wfToRestart.restart = True
wfToRestart.stopped = False
wfToRestart.callToilRunner()
wfToRestart.restart = False
wfToRestart.restartToilRunner()
return {"workflow_id":wf.workflowID}
def getWorkflowInfoByID(self, id):
"""
......@@ -467,6 +476,14 @@ class WorkflowWrapper(threading.Thread):
self.callToilRunner()
if not self.stopped:
wesParser.parse_log(self.logFilePath, wesDAO, self.workflowDBID)
def restartToilRunner(self):
self.stopped = False
self.toilProcess = threading.Thread(target=callToilRunner)
self.toilProcess.start()
self.toilProcess.join()
#self.callToilRunner()
self.restart = False
def killToilRunner(self):
self.stopped = True
......@@ -538,5 +555,5 @@ class WorkflowWrapper(threading.Thread):
############################################################
if __name__ == '__main__':
CWLRestAPI.register(app, route_base="/")
app.run()
app.run(host='0.0.0.0')
#call this file with "python3.6 cwl.py"
......@@ -31,7 +31,7 @@ class CWLParser(ABCParser):
if userid is None:
userid = wes_dao.add_User(authtoken)
workflowid = wes_dao.add_Workflow(userid,'',workflow_name,workflow_descriptor, workflow_type, workflow_type_version,'','QUEUEING')
workflowid = wes_dao.add_Workflow(userid,'',workflow_name,workflow_descriptor, workflow_type, workflow_type_version,'','QUEUED')
for key, value in yaml_json.items():
match = inputs.get(key)
......@@ -67,4 +67,4 @@ class CWLParser(ABCParser):
if __name__ == '__main__':
wesdao = WES_DAO()
parser = CWLParser()
parser.parse_workflow("iwos", "test1","data/ex2.cwl",wesdao, "data/ex2.yml")
\ No newline at end of file
parser.parse_workflow("iwos", "test1","data/ex2.cwl",wesdao, "data/ex2.yml")
FROM ubuntu:16.04
LABEL ubuntu.version="16.04" cutadapt.version="1.16" \
maintainer="christina.herrmann@unibas.ch"
RUN apt-get update && apt-get -yy install python-pip \
&& pip install cutadapt==1.16
WORKDIR /data/
### Optional Inputs ###
# Options that influence how the adapters are found:
adapter:
- TGGAATTCTCGGGTGCCAAGG
#- AAAAAA$
#- GGGGGG$
#front:
# - CCCCCC
#anywhere:
# - GGGGGGG
#errorRate: 0.1
#noIndels: true
#times: 2
#overlap: 4
#matchReadWildcards: true
# PairedEnd options:
#adapterPaired:
# - AAA$
# - GGG$
#frontPaired:
# - CCCCCC
#anywherePaired:
# - GGGGGGG
# Options that influence what gets output to where:
#quiet: true
#infoFile: info.txt
#restFile: rest.txt
#wildcardFile: wildcard.txt
#tooShortOutput: tooshort.txt
#tooLongOutput: toolong.txt
#untrimmedOutput: untrimmed.txt
# Options for filtering of processed reads:
#discardUntrimmed: true
#discardTrimmed: true
#noTrim: true
#maskAdapter: true
minimumLength: 4
#maximumLength: 70
#maxN: 1
# Additional modifications to the reads:
#bwa: true
#stripF3: true
#maq: true
#trimN: true
#noZeroCap: true
#doubleEncode: true
#colorspace: true
#cut: -20
#qualityCutoff: [5]
#qualityBase: 5
#prefix: PRE___
#suffix: ___SUF
#stripSuffix: GCA
#lengthTag: "123"
#untrimmedPairedOutput: untrimmedpaired.txt
#cutPaired: 5
#pairedOutput: cuttedFasta2
# Misc:
#format: .fasta does not work
### Necessary Inputs ###
inputFile:
class: File
# Path has to be relative to location of this yaml file
path: ../../tests/testfiles_Aseq_cl/valid5p_fq2fa_ungzip.fa
#path: ../gzipOutTest.fq
#class: File
#path: ../fasta_test_cut_adapt_2.fa
targetFileName: cutadapt_valid5p_fq2fa_ungzip.fa
### Optional Inputs ###
# Options that influence how the adapters are found:
adapter:
- TGGAATTCTCGGGTGCCAAGG
#- AAAAAA$
#- GGGGGG$
#front:
# - CCCCCC
# - ^CCCCCC
#anywhere:
# - GGGGGGG
#errorRate: 0.1
#noIndels: true
#times: 2
#overlap: 4
#matchReadWildcards: true
# PairedEnd options:
#adapterPaired:
# - AAA$
# - GGG$
#frontPaired:
# - CCCCCC
#anywherePaired:
# - GGGGGGG
# Options that influence what gets output to where:
#quiet: true
#infoFile: info.txt
#restFile: rest.txt
#wildcardFile: wildcard.txt
#tooShortOutput: tooshort.txt
#tooLongOutput: toolong.txt
#untrimmedOutput: untrimmed.txt
# Options for filtering of processed reads:
#discardUntrimmed: true
#discardTrimmed: true
#noTrim: true
#maskAdapter: true
minimumLength: 4
#maximumLength: 70
#maxN: 1
# Additional modifications to the reads:
#bwa: true
#stripF3: true
#maq: true
#trimN: true
#noZeroCap: true
#doubleEncode: true
#colorspace: true
#cut: -20
#qualityCutoff: [5]
#qualityBase: 5
#prefix: PRE___
#suffix: ___SUF
#stripSuffix: GCA
#lengthTag: "123"
#untrimmedPairedOutput: untrimmedpaired.txt
#cutPaired: 5
#pairedOutput: cuttedFasta2
# Misc:
#format: .fasta does not work
### Necessary Inputs ###
inputFile:
class: File
# Path has to be relative to location of this yaml file
path: ../../testfiles_Aseq_cl/valid5p_fq2fa_ungzip.fa
#path: ../gzipOutTest.fq
#class: File
#path: ../fasta_test_cut_adapt_2.fa
targetFileName: cutadapt_valid5p_fq2fa_ungzip.fa
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: CommandLineTool
label: "FASTQ to FASTA"
doc: |
executes the fastq_to_fasta command of FASTX implemented for version 0.0.14:
optional args:
[rename, keepUnknown, verbose, compress]
fixed args:
[inputFile, targetFileName]
baseCommand: fastq_to_fasta
requirements:
# - $import: runtimeSettingsMedium.yml
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: cjh4zavolab/fastx:0.0.14
- class: SoftwareRequirement
packages:
fastx-toolkit:
specs: ["http://identifiers.org/RRID:SCR_005534"]
version: ["0.0.14"]
inputs:
### Optional Inputs ###
rename: #tested
type: boolean?
inputBinding:
position: 1
prefix: -r
doc: |
Rename sequence identifiers to numbers.
keepUnknown: #tested
type: boolean?
inputBinding:
position: 2
prefix: -n
doc: |
keep sequences with unknown (N) nucleotides.
Default is to discard such sequences.
verbose: #tested
type: boolean?
inputBinding:
position: 3
prefix: -v
doc: |
Verbose - report number of sequences.
If [-o] is specified, report will be printed to STDOUT.
If [-o] is not specified (and output goes to STDOUT),
report will be printed to STDERR.
compress: # TODO: runs through but cant be decompressed
type: boolean?
inputBinding:
position: 4
prefix: -z
doc: |
Compress output with GZIP. (take care of the targetFileName to have .gz as suffix)
quality:
type: boolean?
inputBinding:
position: 5
prefix: -Q33
doc: |
Make the most qualities readable
### Necessary Inputs ###
inputFile: #tested
type: File
inputBinding:
position: 5
prefix: -i
doc: |
FASTAQ input file.
targetFileName: #tested #drawback: outPath must be relative, prefix is location of cwl tool
type: string
doc: |
Name of the outputfile. Given as String.
outputs:
q2aResult:
type: stdout
format: edam:format_1929
stdout: $(inputs.inputFile.nameroot).fa
doc: |
usage: fastq_to_fasta [-h] [-r] [-n] [-v] [-z] [-i INFILE] [-o OUTFILE]
Part of FASTX Toolkit 0.0.14 by A. Gordon (assafgordon@gmail.com)
[-h] = This helpful help screen.
[-r] = Rename sequence identifiers to numbers.
[-n] = keep sequences with unknown (N) nucleotides.
Default is to discard such sequences.
[-v] = Verbose - report number of sequences.
If [-o] is specified, report will be printed to STDOUT.
If [-o] is not specified (and output goes to STDOUT),
report will be printed to STDERR.
[-z] = Compress output with GZIP.
[-i INFILE] = FASTA/Q input file. default is STDIN.
[-o OUTFILE] = FASTA output file. default is STDOUT.
############################
s:dateModified: "2018-02-28"
s:author:
- class: s:Person
s:name: Students
- class: s:Person
s:name: Christina J. Herrmann
$namespaces:
s: https://schema.org/
edam: http://edamontology.org/
$schemas:
- https://schema.org/docs/schema_org_rdfa.html
- http://edamontology.org/EDAM_1.18.owl
#!/bin/bash
#SBATCH -Q
#SBATCH -J norm_job_0
#SBATCH --mem=1024
#SBATCH --cpus-per-task=1
module load FASTX-Toolkit/0.0.14-goolf-1.7.20
fastq_to_fasta -r -n -i ../repetitions.fq > repet.fa
\ No newline at end of file
inputFile:
class: File
# Path has to be relative to location of this yaml file
path: ../../tests/testfiles_Aseq_cl/ungzip_fastq.fq
rename: true
keepUnknown: false
#verbose: true
#compress: true
quality: true
targetFileName: fastq2fasta_ungzip.fa
FROM ubuntu:16.04
ENV libgtextutils_version 0.7
ENV fastx_version 0.0.14
RUN apt-get update \
&& apt-get install --yes gcc g++ pkg-config wget \
&& wget https://github.com/agordon/libgtextutils/releases/download/${libgtextutils_version}/libgtextutils-${libgtextutils_version}.tar.gz \
&& tar -xzf libgtextutils-${libgtextutils_version}.tar.gz \
&& cd libgtextutils-${libgtextutils_version} \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& wget https://github.com/agordon/fastx_toolkit/releases/download/${fastx_version}/fastx_toolkit-${fastx_version}.tar.bz2 \
&& tar -xjf fastx_toolkit-${fastx_version}.tar.bz2 \
&& cd fastx_toolkit-${fastx_version} \
&& ./configure \
&& make \
&& make install \
&& cd ..
LABEL maintainer "christina.herrmann@unibas.ch"
FROM ubuntu:16.04
ENV libgtextutils_version 0.7
ENV fastx_version 0.0.14
RUN apt-get update \
&& apt-get install --yes gcc g++ pkg-config wget \
&& wget https://github.com/agordon/libgtextutils/releases/download/${libgtextutils_version}/libgtextutils-${libgtextutils_version}.tar.gz \
&& tar -xzf libgtextutils-${libgtextutils_version}.tar.gz \
&& cd libgtextutils-${libgtextutils_version} \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& wget https://github.com/agordon/fastx_toolkit/releases/download/${fastx_version}/fastx_toolkit-${fastx_version}.tar.bz2 \
&& tar -xjf fastx_toolkit-${fastx_version}.tar.bz2 \
&& cd fastx_toolkit-${fastx_version} \
&& ./configure \
&& make \
&& make install \
&& cd ..
LABEL maintainer "christina.herrmann@unibas.ch"
#!/usr/bin/env cwl-runner
cwlVersion: v1.0
class: CommandLineTool
label: "FASTQ/A Reverse Complement"
doc: |
executes the fastx_reverse_complement command of FASTX implemented for version 0.0.13:
optional args:
[compress]
fixed args:
[inputFile, targetFileName]
Producing the Reverse-complement of each sequence in a FASTQ/FASTA file.
usage: fastx_reverse_complement [-h] [-r] [-z] [-v] [-i INFILE] [-o OUTFILE]
Part of FASTX Toolkit 0.0.13 by A. Gordon (assafgordon@gmail.com)
[-h] = This helpful help screen.
[-z] = Compress output with GZIP.
[-i INFILE] = FASTA/Q input file. default is STDIN.
[-o OUTFILE] = FASTA/Q output file. default is STDOUT.
baseCommand: fastx_reverse_complement
stdout: $(inputs.targetFileName)
requirements:
# - $import: runtimeSettingsMedium.yml
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: cjh4zavolab/fastx:0.0.14
#dockerFile: >
# $import: fastx-Dockerfile
- class: SoftwareRequirement
packages:
fastx-toolkit:
specs: ["http://identifiers.org/RRID:SCR_005534"]
version: ["0.0.13"]
inputs:
### Optional Inputs ###
compress: # TODO: runs through but cant be decompressed ???
type: boolean?
inputBinding:
position: 1
prefix: -z
doc: |
Compress output with GZIP. (take care of the targetFileName to have .gz as suffix)
### Necessary Inputs ###
inputFile:
type: File
#format: [edam:format_1929, edam:format_1930]
inputBinding:
position: 5
prefix: -i
doc: |
FASTAQ input file.
targetFileName:
type: string
doc: |
Name of the outputfile. Given as String.
outputs:
reverse_complement_out:
type: stdout
#format: edam:format_1929
############################
s:dateCreated: "2017-11-30"
s:author:
- class: s:Person
s:name: Christina J. Herrmann
s:email: mailto:christina.herrmann@unibas.ch
$namespaces:
s: https://schema.org/
edam: http://edamontology.org/
$schemas:
- https://schema.org/docs/schema_org_rdfa.html # Might need to pip install html5lib
- http://edamontology.org/EDAM_1.18.owl
inputFile:
class: File
format: http://edamontology.org/format_1929
# Path has to be relative to location of this yaml file
path: ../../tests/testfiles_Aseq_cl/cutadapt_valid5p_fq2fa_ungzip.fa
targetFileName: "revComp_cutadapt_valid5p_fq2fa_ungzip.fa"
compress: false
cwlVersion: v1.0
class: CommandLineTool
baseCommand: gzip
doc: |
executes the gzip command implemented for version 1.3.12:
optional args:
[decompress, ascii, force, list, name, no-name, quiet, recursive, suffix, test, verbose, (best | fast | speed)]
fixed args:
[stdout, zipFile, targetFileName]
This commandline tool can zip or unzip Files into or from .gz files.
requirements:
# - $import: runtimeSettingsMedium.yml
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: cjh4zavolab/gzip:1 #the system temporary directory which
#must be located at /tmp.
stdout: $(inputs.targetFileName)
#passing arguments via boolean: allows to specify the possible arguments