Commit 7dbf69ce authored by Zdralovic Edim's avatar Zdralovic Edim

Changed directories to shared storage

parent ef4ad09b
......@@ -2,13 +2,14 @@ FROM python:2.7
ENV LANG C.UTF-8
LABEL MAINTAINER "edim.zdralovic@unibas.ch"
#Installing Singularity(provided by Nina)
ENV SINGU_VERSION 2.4.2
#Installing Singularity
ENV SINGU_VERSION 2.5.0
WORKDIR /home
RUN apt-get -y update
RUN apt-get install --yes python python-pip build-essential wget \
squashfs-tools git-all nodejs\
&& pip install html5lib \
&& pip install libarchive \
&& wget https://github.com/singularityware/singularity/releases/download/$SINGU_VERSION/singularity-$SINGU_VERSION.tar.gz \
&& tar xvf singularity-$SINGU_VERSION.tar.gz \
&& cd singularity-$SINGU_VERSION \
......@@ -18,16 +19,23 @@ RUN apt-get install --yes python python-pip build-essential wget \
#Installing Toil
WORKDIR /home
RUN git clone https://github.com/common-workflow-language/cwltool.git \
&& cd cwltool \
&& pip2 install . \
&& CWL_VERSION=$(cwltool --version | cut -d' ' -f 2 | cut -d'.' -f 3) \
&& cd .. \
&& git clone https://github.com/BD2KGenomics/toil.git \
&& cd toil \
#RUN git clone https://github.com/common-workflow-language/cwltool.git \
# && cd cwltool \
# && pip2 install . \
# && CWL_VERSION=$(cwltool --version | cut -d' ' -f 2 | cut -d'.' -f 3) \
# && cd .. \
# && git clone https://github.com/BD2KGenomics/toil.git \
# && cd toil \
# Replace toil's default cwltool version with current one
&& sed -ri "s/cwltool==1\.0\.[0-9]{14}/cwltool==1\.0\.$CWL_VERSION/" setup.py \
&& pip install .[cwl]
# && sed -ri "s/cwltool==1\.0\.[0-9]{14}/cwltool==1\.0\.$CWL_VERSION/" setup.py \
# && pip install .[cwl]
RUN git clone --branch 3.15.0 https://github.com/BD2KGenomics/toil.git \
&& cd toil \
&& pip2 install .[cwl] \
&& cd .. \
&& git clone --branch 1.0.20180306140409 https://github.com/common-workflow-language/cwltool.git \
&& cd cwltool \
&& pip2 install .
#Installing Python 3
RUN apt-get -y install build-essential checkinstall \
......
......@@ -58,6 +58,7 @@ class CWLRestAPI(FlaskView):
route_base = '/' #flask_classful specific, class variable
def __init__(self):
self.baseDir = './workflows/' #for developing only
#self.sharedStorage = '/home/cuhoni66/toilCWLTest/'
self.__wfManager = WorkflowManager()
############################################################
#Section: Routed Methods
......@@ -67,8 +68,8 @@ class CWLRestAPI(FlaskView):
<h2>The service is available.</h2>
"""
@route('/workflows/gitURL', methods=['POST'])
def postGit(self):
# @route('/workflows/setPath', methods=['POST'])
# def postGit(self):
"""
A POST on this route expects three arguments:
1) Usertoken
......@@ -78,10 +79,10 @@ class CWLRestAPI(FlaskView):
the outmost directive. The YAML file is parsed, converted in JSON and returned to the frontend.
A WorkflowWrapper object is instantiated. This only sets up a pipeline but does not start it.
"""
byteData = request.data
data = json.loads(byteData.decode("utf-8"))
wf = WorkflowWrapper(data["usertoken"], self.baseDir, data["workflowName"], data["giturl"])
return json.dumps(self.__wfManager.addWorkflow(wf))
# byteData = request.data
# data = json.loads(byteData.decode("utf-8"))
# wf = WorkflowWrapper(data["usertoken"], self.sharedStorage, data["workflowName"])
# return json.dumps(self.__wfManager.addWorkflow(wf))
@route('/workflows', methods=['POST'])
def postWorkflow(self):
......@@ -91,7 +92,11 @@ class CWLRestAPI(FlaskView):
"""
byteData = request.data
data = json.loads(byteData.decode("utf-8"))
return json.dumps(self.__wfManager.startWorkflow(data["workflowName"], data["yaml"]))
wf = WorkflowWrapper(userToken = data["usertoken"], sharedStorage = data["sharedStoragePath"], workflowName = data["workflowName"], baseDir = self.baseDir)
self.__wfManager.addWorkflow(wf)
return json.dumps(self.__wfManager.startWorkflow(wf.workflowName))
@route('/workflows', methods=['GET'])
def getAllWorkflows(self):
......@@ -201,9 +206,8 @@ class WorkflowManager:
This method recieves the incoming workflow from the API and saves the WorkflowWrapper object.
"""
self.workflowList.append(wf)
return {"yaml":wf.yamlStream}
def startWorkflow(self, wfName, yaml):
def startWorkflow(self, wfName):
"""
This method recieves the incoming workflow from the API and starts it in a thread.
"""
......@@ -213,7 +217,7 @@ class WorkflowManager:
if i.workflowName == wfName:
wf = i
break
wf.yamlStream = yaml
wf.start()
while not wf.workflowID:
......@@ -371,15 +375,16 @@ class WorkflowWrapper(threading.Thread):
############################################################
#Section: Variables
############################################################
def __init__(self, userToken="", baseDir="", workflowName="", gitRepoUrl="", restart=False): #Constructor
def __init__(self, userToken="", baseDir="", workflowName="", gitRepoUrl="", restart=False, sharedStorage=""): #Constructor
self.restart = restart
self.sharedStorage = sharedStorage
self.workflowID = ""
self.yamlStream = ""
self.cwlStream = ""
self.baseDir = baseDir
self.logFilePath = ""
self.ymlFilePath = ""
self.cwlFilePath = ""
self.ymlFilePath = glob.glob(self.sharedStorage+"/*.yml")[0]
self.cwlFilePath = glob.glob(self.sharedStorage+"/*.cwl")[0]
self.toilProcess = None
self.workflowName = workflowName
self.gitRepoUrl = gitRepoUrl
......@@ -387,21 +392,23 @@ class WorkflowWrapper(threading.Thread):
self.workDir = sys.intern(baseDir+userToken+"/work/")
self.jobStore = sys.intern(baseDir+userToken+"/jobStore/")
self.logDir = sys.intern(baseDir+userToken+"/logs/")
self.dataDir = sys.intern(baseDir+userToken+"/data/")
#self.dataDir = sys.intern(baseDir+userToken+"/data/")
self.userToken = userToken
self.dataFilesDir = ""
#self.dataFilesDir = ""
self.workflowDBID = None
self.stopped = False
self.jobStoreForPipeline = self.jobStore+str(self.workflowName)
self.dataFilesDir = self.dataDir+str(self.workflowName)
#self.dataFilesDir = self.dataDir+str(self.workflowName)
if not self.restart:
self.createDirs()
self.saveFilesToDataFileDir()
self.workflowDBID = wesParser.parse_workflow(self.userToken, workflowName, self.cwlStream, wesDAO, self.yamlStream)
#self.saveFilesToDataFileDir()
self.workflowDBID = wesParser.parse_workflow(self.userToken, workflowName, self.cwlFilePath, wesDAO, self.ymlFilePath)
self.toilRunnerArgsList = ["toil-cwl-runner", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
#self.toilRunnerArgsList = ["toil-cwl-runner", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
self.toilRunnerArgsList = ["toil-cwl-runner", "--debugWorker", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
self.toilRunnerArgsListRestart = ["toil-cwl-runner", "--restart", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
threading.Thread.__init__(self)
############################################################
......@@ -417,8 +424,8 @@ class WorkflowWrapper(threading.Thread):
os.makedirs(self.workDir)
os.makedirs(self.logDir)
os.makedirs(self.jobStore)
os.makedirs(self.dataDir)
os.makedirs(self.dataFilesDir)
#os.makedirs(self.dataDir)
#os.makedirs(self.dataFilesDir)
def saveFilesToDataFileDir(self):
"""
......
......@@ -12,15 +12,15 @@ class CWLParser(ABCParser):
Parses a CWL-based workflow and extracts all necessary information to follow the ga4gh-specification
"""
#if the workflow is given in file format use:
#cwl_json_file = open(workflow_file,'r').read()
#cwl_json = yaml.load(cwl_json_file)
cwl_json_file = open(workflow_file,'r').read()
cwl_json = yaml.load(cwl_json_file)
#if the yaml is given in file format use:
#yaml_json_file = open(yaml_file,'r').read()
#yaml_json = yaml.load(yaml_json_file)
yaml_json_file = open(yaml_file,'r').read()
yaml_json = yaml.load(yaml_json_file)
cwl_json = workflow_file
yaml_json = yaml_file
#cwl_json = workflow_file
#yaml_json = yaml_file
workflow_descriptor = cwl_json['doc']
workflow_type = 'cwl'
workflow_type_version = cwl_json['cwlVersion']
......
POST with everything in JSON:
curl -d \
'{"usertoken":"user1","workflowName":"myHornyWorkflow" ,"yaml":{"path2localRepo": "/home/christina/Work/CWL_projects/CWL_test_pAp", "fastqgz": {"class": "File","path": "../tests/testfiles_Aseq_cl/A_seq_test.fq.gz"}, "outFileName": null, "adapter5p": null}, "cwl":{"class":"Workflow","cwlVersion":"v1.0","label":"Aseq pre-processing for the polyA pipeline","doc":"This workflow does pre-processing of Aseq data for the polyA pipeline.\n(*Citation needed*)\nSequencing data are converted to fasta, checked for valid 5'' and 3'' ends and reverse complemented.\nSteps are:\n- gnu_zip fastq\n- fastq_to_fasta\n- rs-filter-by-5p-adapter.keep5pAdapter.pl\n- cutadapt\n- fastx_reverse_complement\n- gzip\n","requirements":[{"class":"StepInputExpressionRequirement"},{"class":"ScatterFeatureRequirement"},{"class":"InlineJavascriptRequirement"},{"class":"SubworkflowFeatureRequirement"},{"class":"MultipleInputFeatureRequirement"}],"hints":{"SoftwareRequirement":{"packages":{"fastx-toolkit":{"specs":["http://identifiers.org/RRID:SCR_005534"],"version":["0.0.14"]},"cutadapt":{"specs":["http://identifiers.org/RRID:SCR_011841"],"version":["1.15"]}}}},"inputs":{"fastqgz":{"type":"File","label":"Reads in .fq.gz format"},"outFileName":"string?","adapter5p":"string?"},"outputs":{"result1":{"type":"File","outputSource":"unzip_fastq/zipResult"},"result2":{"type":"File","outputSource":"fastq_to_fasta/q2aResult"},"result3":{"type":"File","outputSource":"select_valid_5p/valid5pResult"},"result4":{"type":"File","outputSource":"cutadapt/cutadapt_fasta_out"},"result5":{"type":"File","outputSource":"reverse_complement/reverse_complement_out"},"result":{"type":"File","outputSource":"zip_fasta/zipResult"}},"steps":{"unzip_fastq":{"run":"./commandLineTools/gnu_gzip/gnu_gzip.cwl","in":{"zipFile":"fastqgz","decompress":{"default":true},"targetFileName":{"default":"ungzip_fastq.fq"}},"out":["zipResult"]},"fastq_to_fasta":{"run":"./commandLineTools/fastq_to_fasta/fastq_to_fasta.cwl","in":{"inputFile":{"source":"unzip_fastq/zipResult"},"rename":{"default":true},"keepUnknown":{"default":false},"targetFileName":{"default":"fastq2fasta_ungzip.fa"}},"out":["q2aResult"]},"select_valid_5p":{"run":"./commandLineTools/select_valid_5p/select_valid_5p.cwl","in":{"inputFile":"fastq_to_fasta/q2aResult","adapter":{"source":"adapter5p","default":"....TTT"},"targetFileName":{"default":"valid5p_fq2fa_ungzip.fa"},"zippedInput":{"default":false}},"out":["valid5pResult"]},"cutadapt":{"run":"./commandLineTools/cutadapt/cutadapt.cwl","in":{"inputFile":"select_valid_5p/valid5pResult","adapter":{"default":["TGGAATTCTCGGGTGCCAAGG"]},"minimumLength":{"default":4},"targetFileName":{"default":"cutadapt_valid5p_fq2fa_ungzip.fa"}},"out":["cutadapt_fasta_out"]},"reverse_complement":{"run":"./commandLineTools/fastx_reverse_complement/fastx_reverse_complement.cwl","in":{"inputFile":"cutadapt/cutadapt_fasta_out","compress":{"default":false},"targetFileName":{"default":"revComp_cutadapt_valid5p_fq2fa_ungzip.fa"}},"out":["reverse_complement_out"]},"zip_fasta":{"run":"./commandLineTools/gnu_gzip/gnu_gzip.cwl","in":{"zipFile":"reverse_complement/reverse_complement_out","decompress":{"default":false},"no-name":{"default":true},"targetFileName":{"default":"revComp_cutadapt_valid5p_fq2fa_ungzip.fa.gz"}},"out":["zipResult"]},"zip_fasta2":{"run":"./commandLineTools/gnu_gzip/gnu_gzip.cwl","in":{"zipFile":"reverse_complement/reverse_complement_out","decompress":{"default":false},"no-name":{"default":true},"targetFileName":{"default":"revComp_cutadapt_valid5p_fq2fa_ungzip.fa.gz"}},"out":["zipResult"]}}}}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows
curl -d '{"usertoken":"user1"}' -H "Content-Type: application/json" -X GET http://localhost:5000/workflows
curl -X GET http://localhost:5000/workflows/
DELETE workflow
curl -X DELETE http://localhost:5000/workflows/e78e64e5-7c4c-4c7a-9af0-61543d37dbeb
......@@ -19,7 +11,7 @@ https://damyanon.net/post/flask-series-testing/
build and run docker container:
sudo docker build -t cwlflask .
sudo docker run -it cwlflask /bin/sh --> gets into the docker container, enables usage of terminal
docker run -p 8080:5000 cwlflask
sudo docker run -p 5000:5000 cwlflask
curl -d \
'{"giturl":"https://github.com/shinra-tensei/toilCWLTest.git","usertoken":"user1","workflowName":"myHornyWorkflow"}' \
......@@ -29,6 +21,10 @@ curl -d \
'{"workflowName":"myHornyWorkflow", "yaml": {"path2localRepo": "/home/christina/Work/CWL_projects/CWL_test_pAp", "fastqgz": {"class": "File", "path": "./tests/testfiles_Aseq_cl/A_seq_test.fq.gz"}, "outFileName": "q2a_renamed_valid5p.fa", "adapter5p": "....TTT"}}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows
curl -d \
'{"workflowName":"myHornyWorkflow", "usertoken": "hornyBoi", "sharedStoragePath": "/home/cuhoni66/toilCWLTest/"}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows
toil-cwl-runner --debugWorker --outdir=/home/cuhoni66/Schreibtisch/dot/data/outdir/ --workDir=/home/cuhoni66/Schreibtisch/dot/data/workdir/ --jobStore=/home/cuhoni66/Schreibtisch/dot/data/jobStore/ --singularity ex3.cwl ex2.yml
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment