Commit 62ff9eda authored by Zdralovic Edim's avatar Zdralovic Edim

Refactored the restarting of workflows

parent 3ffcd3f6
......@@ -62,10 +62,6 @@ class CWLRestAPI(FlaskView):
route_base = '/' #flask_classful specific, class variable
def __init__(self):
self.baseDir = './workflows/' #for developing only
#self.testCWL = './data/ex2.cwl' #for developing only
#self.testYML = './data/ex2.yml' #for developing only
#self.mirzaCWL = '/home/cuhoni66/MIRZAG/mirzag_seed.cwl'
#self.mirzaYML = '/home/cuhoni66/MIRZAG/job.yml'
self.__WorkflowIDCounter = 0
self.__wfManager = WorkflowManager()
############################################################
......@@ -226,7 +222,7 @@ class WorkflowManager:
return {"yaml":wf.yamlStream}
def startWorkflow(self, wfName):
def startWorkflow(self, wfName, yaml):
"""
This method recieves the incoming workflow from the API and starts it in a thread.
"""
......@@ -236,7 +232,7 @@ class WorkflowManager:
if i.workflowName == wfName:
wf = i
break
wf.yamlStream = yaml
wf.start()
while not wf.workflowID:
......@@ -258,8 +254,51 @@ class WorkflowManager:
wfToRestart = i
break
wfToRestart.restart = True
wfToRestart.restartToilRunner()
return {"workflow_id":wf.workflowID}
newWorkflow = WorkflowWrapper(restart=True)
print("old")
print(wfToRestart.workDir)
print(wfToRestart.jobStoreForPipeline)
print(wfToRestart.outDir)
print(wfToRestart.toilRunnerArgsList)
print(wfToRestart.toilRunnerArgsListRestart)
newWorkflow.workflowID = wfToRestart.workflowID
newWorkflow.yamlStream = wfToRestart.yamlStream
newWorkflow.cwlStream = wfToRestart.cwlStream
newWorkflow.baseDir = wfToRestart.baseDir
newWorkflow.logFilePath = wfToRestart.logFilePath
newWorkflow.ymlFilePath = wfToRestart.ymlFilePath
newWorkflow.cwlFilePath = wfToRestart.cwlFilePath
newWorkflow.workflowName = wfToRestart.workflowName
newWorkflow.gitRepoUrl = wfToRestart.gitRepoUrl
newWorkflow.outDir = wfToRestart.outDir
newWorkflow.workDir = wfToRestart.workDir
newWorkflow.logDir = wfToRestart.logDir
newWorkflow.dataDir = wfToRestart.dataDir
newWorkflow.userToken = wfToRestart.userToken
newWorkflow.workflowDBID = wfToRestart.workflowDBID
newWorkflow.jobs = wfToRestart.jobs
newWorkflow.stopped = True
newWorkflow.jobStore = wfToRestart.jobStore
newWorkflow.jobStoreForPipeline = wfToRestart.jobStoreForPipeline
newWorkflow.dataFilesDir = wfToRestart.dataFilesDir
newWorkflow.toilRunnerArgsList = wfToRestart.toilRunnerArgsList
newWorkflow.toilRunnerArgsListRestart = wfToRestart.toilRunnerArgsListRestart
print("new")
print(newWorkflow.workDir)
print(newWorkflow.jobStoreForPipeline)
print(newWorkflow.outDir)
print(newWorkflow.toilRunnerArgsList)
print(newWorkflow.toilRunnerArgsListRestart)
self.workflowList.remove(wfToRestart)
self.workflowList.append(newWorkflow)
newWorkflow.start()
return {"workflow_id":newWorkflow.workflowID}
def getWorkflowInfoByID(self, id):
"""
......@@ -389,8 +428,8 @@ class WorkflowWrapper(threading.Thread):
############################################################
#Section: Variables
############################################################
def __init__(self, userToken="", baseDir="", workflowName="", gitRepoUrl=""): #Constructor
self.restart = False
def __init__(self, userToken="", baseDir="", workflowName="", gitRepoUrl="", restart=False): #Constructor
self.restart = restart
self.workflowID = ""
self.yamlStream = ""
self.cwlStream = ""
......@@ -403,32 +442,27 @@ class WorkflowWrapper(threading.Thread):
self.workflowName = workflowName
self.gitRepoUrl = gitRepoUrl
self.outDir = self.baseDir+userToken+"/out/"
self.workDir = baseDir+userToken+"/work/"
self.jobStore = baseDir+userToken+"/jobStore/"
self.jobStoreForPipeline = ""
self.logDir = baseDir+userToken+"/logs/"
self.dataDir = baseDir+userToken+"/data/"
self.outDir = sys.intern(self.baseDir+userToken+"/out/")
self.workDir = sys.intern(baseDir+userToken+"/work/")
self.jobStore = sys.intern(baseDir+userToken+"/jobStore/")
self.logDir = sys.intern(baseDir+userToken+"/logs/")
self.dataDir = sys.intern(baseDir+userToken+"/data/")
self.userToken = userToken
self.dataFilesDir = ""
self.workflowDBID = None
self.jobs = []
self.stopped = False
self.jobStoreForPipeline = self.jobStore+str(self.workflowName)
self.dataFilesDir = self.dataDir+str(self.workflowName)
self.createDirs()
self.saveFilesToDataFileDir()
self.workflowDBID = wesParser.parse_workflow(self.userToken, workflowName, self.cwlStream, wesDAO, self.yamlStream)
self.jobs = self.getJobListFromFile()
if not self.restart:
self.createDirs()
self.saveFilesToDataFileDir()
self.workflowDBID = wesParser.parse_workflow(self.userToken, workflowName, self.cwlStream, wesDAO, self.yamlStream)
self.jobs = self.getJobListFromFile()
#Problem encountered when using the file paths for the cwl and yml
#test and commandLineTools have to be in the same directory
self.toilRunnerArgsList = ["toil-cwl-runner", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
self.toilRunnerArgsListRestart = ["toil-cwl-runner", "--restart", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
self.toilRunnerArgsList = ["toil-cwl-runner", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
self.toilRunnerArgsListRestart = ["toil-cwl-runner", "--restart", "--debugWorker", "--clean=onSuccess", "--outdir="+self.outDir,"--workDir="+self.workDir, "--jobStore="+self.jobStoreForPipeline, "--singularity", self.cwlFilePath, self.ymlFilePath,]
threading.Thread.__init__(self)
############################################################
#Section: Public Methods
......@@ -453,21 +487,6 @@ class WorkflowWrapper(threading.Thread):
self.cwlFilePath = glob.glob(self.dataFilesDir+"/*.cwl")[0]
self.yamlStream = yaml.safe_load(open(self.ymlFilePath, "r"))
self.cwlStream = yaml.safe_load(open(self.cwlFilePath, "r"))
#glob.glob("/home/adam/*.txt")
#copying needed tools and tests, for developing only
# shutil.copytree("./data/commandLineTools", self.dataFilesDir+"/files/commandLineTools")
# shutil.copytree("./tests", self.dataFilesDir+"/tests")
# self.ymlFilePath = self.dataFilesDir+"/files/params.yml"
# yamlFile = open(self.ymlFilePath, "w")
# yamlFile.write(yaml.safe_dump(self.yamlStream, default_flow_style=False))
# yamlFile.close()
# self.cwlFilePath = self.dataFilesDir+"/files/wf.json"
# cwlFile = open(self.cwlFilePath, "w")
# cwlFile.write(json.dumps(self.cwlStream))
# cwlFile.close()
def run(self):
"""
......@@ -476,14 +495,6 @@ class WorkflowWrapper(threading.Thread):
self.callToilRunner()
if not self.stopped:
wesParser.parse_log(self.logFilePath, wesDAO, self.workflowDBID)
def restartToilRunner(self):
self.stopped = False
self.toilProcess = threading.Thread(target=callToilRunner)
self.toilProcess.start()
self.toilProcess.join()
#self.callToilRunner()
self.restart = False
def killToilRunner(self):
self.stopped = True
......@@ -499,9 +510,10 @@ class WorkflowWrapper(threading.Thread):
self.logFilePath = self.logDir+"/log_"+str(self.workflowName)+".txt"
logfile = open(self.logFilePath, "w")
if self.restart is False:
if not self.restart:
self.toilProcess = subprocess.Popen(self.toilRunnerArgsList, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
else:
self.toilProcess = None
self.toilProcess = subprocess.Popen(self.toilRunnerArgsListRestart, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
for l in self.toilProcess.stdout:
......
......@@ -8,10 +8,10 @@ curl -d '{"usertoken":"user1"}' -H "Content-Type: application/json" -X GET http:
curl -X GET http://localhost:5000/workflows/
DELETE workflow
curl -X DELETE http://localhost:5000/workflows/b9c7ef70-b363-42ed-be4a-17d6537090b4
curl -X DELETE http://localhost:5000/workflows/
RESTART workflow
curl -X POST http://localhost:5000/workflows/b9c7ef70-b363-42ed-be4a-17d6537090b4
curl -X POST http://localhost:5000/workflows/
testing flask:
https://damyanon.net/post/flask-series-testing/
......@@ -19,12 +19,21 @@ https://damyanon.net/post/flask-series-testing/
sudo docker run -it cwlflask /bin/sh --> gets into the docker container, enables usage of terminal
docker run -p 8080:5000 cwlflask
curl -d \
'{"giturl":"https://github.com/shinra-tensei/toilCWLTest.git","usertoken":"user1","workflowName":"myHornyWorkflow"}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows/gitURL
curl -d \
'{"workflowName":"myHornyWorkflow"}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows
\ No newline at end of file
'{"workflowName":"myHornyWorkflow", "yaml": {"path2localRepo": "/home/christina/Work/CWL_projects/CWL_test_pAp", "fastqgz": {"class": "File", "path": "./tests/testfiles_Aseq_cl/A_seq_test.fq.gz"}, "outFileName": "q2a_renamed_valid5p.fa", "adapter5p": "....TTT"}}' \
-H "Content-Type: application/json" -X POST http://localhost:5000/workflows
toil-cwl-runner --debugWorker --outdir=/home/cuhoni66/Schreibtisch/dot/data/outdir/ --workDir=/home/cuhoni66/Schreibtisch/dot/data/workdir/ --jobStore=/home/cuhoni66/Schreibtisch/dot/data/jobStore/ --singularity ex3.cwl ex2.yml
toil-cwl-runner --debugWorker --outdir=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/out/ --workDir=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/work/ --jobStore=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/jobStore/myHornyWorkflow/ --singularity ex3.cwl ex2.yml
toil-cwl-runner --restart --debugWorker --outdir=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/out/ --workDir=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/work/ --jobStore=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/jobStore/myHornyWorkflow/ --singularity ex3.cwl ex2.yml
toil-cwl-runner --restart --debugWorker --outdir=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/out/ --workDir=./workflows/user1/work/ --jobStore=/home/cuhoni66/kriniGit/krini-cwl/Flask/cwlREST/workflows/user1/jobStore/myHornyWorkflow/ --singularity ex3.cwl ex2.yml
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment