From 8723a6ed6ce938136e27cf546d799a09575c8896 Mon Sep 17 00:00:00 2001
From: Gabriel Studer <gabriel.studer@unibas.ch>
Date: Thu, 28 Nov 2019 13:49:54 +0100
Subject: [PATCH] silence resource warnings for non-closed files etc
This only happens when running the unit tests due to increased log levels.
These warnings could manually be triggered for normal scripts with:
python -Wd world_domination.py.
---
modules/base/pymod/table.py | 38 ++++++++++++++++++++++++--
modules/base/tests/test_table.py | 1 +
modules/bindings/pymod/blast.py | 7 +++--
modules/bindings/pymod/cadscore.py | 5 ++--
modules/bindings/pymod/clustalw.py | 4 +--
modules/bindings/pymod/kclust.py | 6 ++--
modules/bindings/tests/test_blast.py | 3 +-
modules/bindings/tests/test_hhblits.py | 32 ++++++++++++----------
8 files changed, 69 insertions(+), 27 deletions(-)
diff --git a/modules/base/pymod/table.py b/modules/base/pymod/table.py
index bb7714f03..8697d63d0 100644
--- a/modules/base/pymod/table.py
+++ b/modules/base/pymod/table.py
@@ -880,8 +880,10 @@ Statistics for column %(col)s
def _LoadOST(stream_or_filename):
fieldname_pattern=re.compile(r'(?P<name>[^[]+)(\[(?P<type>\w+)\])?')
values_pattern=re.compile("([^\" ]+|\"[^\"]*\")+")
+ file_opened=False
if not hasattr(stream_or_filename, 'read'):
stream=open(stream_or_filename, 'r')
+ file_opened=True
else:
stream=stream_or_filename
header=False
@@ -904,10 +906,18 @@ Statistics for column %(col)s
else:
fieldtypes.append('string')
fieldnames.append(match.group('name'))
- tab=Table(fieldnames, fieldtypes)
+ try:
+ tab=Table(fieldnames, fieldtypes)
+ except Exception as e:
+ # potentially fails if we read in crap... clean up and pass on error
+ if file_opened:
+ stream.close()
+ raise e
header=True
continue
tab.AddRow([x.strip('"') for x in values_pattern.findall(line)])
+ if file_opened:
+ stream.close()
if num_lines==0:
raise IOError("Cannot read table from empty stream")
return tab
@@ -921,8 +931,10 @@ Statistics for column %(col)s
@staticmethod
def _LoadCSV(stream_or_filename, sep):
+ file_opened=False
if not hasattr(stream_or_filename, 'read'):
stream=open(stream_or_filename, 'r')
+ file_opened=True
else:
stream=stream_or_filename
reader=csv.reader(stream, delimiter=sep)
@@ -935,6 +947,8 @@ Statistics for column %(col)s
first=False
else:
tab.AddRow(row)
+ if file_opened:
+ stream.close()
if first:
raise IOError('trying to load table from empty CSV stream/file')
@@ -943,11 +957,16 @@ Statistics for column %(col)s
@staticmethod
def _LoadPickle(stream_or_filename):
+ file_opened=False
if not hasattr(stream_or_filename, 'read'):
stream=open(stream_or_filename, 'rb')
+ file_opened=True
else:
stream=stream_or_filename
- return pickle.load(stream)
+ tab = pickle.load(stream)
+ if file_opened:
+ stream.close()
+ return tab
@staticmethod
def _GuessFormat(filename):
@@ -2210,9 +2229,13 @@ Statistics for column %(col)s
raise ValueError('unknown format "%s"' % format)
def _SavePickle(self, stream):
+ file_opened=False
if not hasattr(stream, 'write'):
stream=open(stream, 'wb')
+ file_opened=True
pickle.dump(self, stream, pickle.HIGHEST_PROTOCOL)
+ if file_opened:
+ stream.close()
def _SaveHTML(self, stream_or_filename):
def _escape(s):
@@ -2288,8 +2311,10 @@ Statistics for column %(col)s
stream.close()
def _SaveCSV(self, stream, sep):
+ file_opened=False
if not hasattr(stream, 'write'):
stream=open(stream, 'w')
+ file_opened=True
writer=csv.writer(stream, delimiter=sep)
writer.writerow(['%s' % n for n in self.col_names])
@@ -2299,13 +2324,18 @@ Statistics for column %(col)s
if c==None:
row[i]='NA'
writer.writerow(row)
+ if file_opened:
+ stream.close()
+
def _SaveOST(self, stream):
+ file_opened=False
if hasattr(stream, 'write'):
writer=csv.writer(stream, delimiter=' ')
else:
stream=open(stream, 'w')
writer=csv.writer(stream, delimiter=' ')
+ file_opened=True
if self.comment:
stream.write(''.join(['# %s\n' % l for l in self.comment.split('\n')]))
writer.writerow(['%s[%s]' % t for t in zip(self.col_names, self.col_types)])
@@ -2315,7 +2345,9 @@ Statistics for column %(col)s
if c==None:
row[i]='NA'
writer.writerow(row)
-
+ if file_opened:
+ stream.close()
+
def GetNumpyMatrix(self, *args):
'''
diff --git a/modules/base/tests/test_table.py b/modules/base/tests/test_table.py
index 640e7c366..5d56f67ee 100644
--- a/modules/base/tests/test_table.py
+++ b/modules/base/tests/test_table.py
@@ -907,6 +907,7 @@ class TestTable(unittest.TestCase):
self.assertRaises(IOError, Table.Load, os.path.join('testfiles','emptytable.tab'))
in_stream = open(os.path.join('testfiles','emptytable.csv'), 'r')
self.assertRaises(IOError, Table.Load, in_stream)
+ in_stream.close()
def testSaveLoadTableOSTWithSpaces(self):
tab = self.CreateTestTable()
diff --git a/modules/bindings/pymod/blast.py b/modules/bindings/pymod/blast.py
index 80409f492..89893323a 100644
--- a/modules/bindings/pymod/blast.py
+++ b/modules/bindings/pymod/blast.py
@@ -166,9 +166,10 @@ def CreateDB(infasta, dbout, mkdb_cmd=None):
else:
raise IOError('mkdb command must either be the path to formatdb or makeblastdb!')
- cmd=' '.join(args)
- ost.LogInfo('creating blast DB (%s)' % cmd)
- os.system(cmd)
+ ost.LogInfo('creating blast DB (%s)' % ' '.join(args))
+ blast_pipe=subprocess.Popen(args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ blast_pipe.communicate()
def BlastVersion(blast_location=None):
"""
diff --git a/modules/bindings/pymod/cadscore.py b/modules/bindings/pymod/cadscore.py
index 362325c87..193bfc87f 100644
--- a/modules/bindings/pymod/cadscore.py
+++ b/modules/bindings/pymod/cadscore.py
@@ -186,7 +186,7 @@ def _RunCAD(tmp_dir, mode, cad_bin_path, old_regime):
"cadtemp"))
ps1=subprocess.Popen(command1, shell=True, stdout=subprocess.PIPE)
- ps1.wait()
+ ps1.communicate()
ps2=subprocess.Popen(command2, shell=True, stdout=subprocess.PIPE)
stdout,_ = ps2.communicate()
lines=stdout.decode().splitlines()
@@ -232,7 +232,8 @@ def _RunCAD(tmp_dir, mode, cad_bin_path, old_regime):
except:
raise RuntimeError("CAD calculation failed")
try:
- localAA = _ParseVoronotaLocal(open(local_score_filename).readlines())
+ with open(local_score_filename) as f:
+ localAA = _ParseVoronotaLocal(f.readlines())
except:
raise RuntimeError("CAD calculation failed")
diff --git a/modules/bindings/pymod/clustalw.py b/modules/bindings/pymod/clustalw.py
index 7ed4e8259..074199283 100644
--- a/modules/bindings/pymod/clustalw.py
+++ b/modules/bindings/pymod/clustalw.py
@@ -129,8 +129,8 @@ def ClustalW(seq1, seq2=None, clustalw=None, keep_files=False, nopgap=False,
if clustalw_option_string!=False:
command=command+" "+clustalw_option_string #see useful flags: http://toolkit.tuebingen.mpg.de/clustalw/help_params
- ps=subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
- ps.wait()
+ with subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) as ps:
+ ps.wait()
aln=io.LoadAlignment(out)
diff --git a/modules/bindings/pymod/kclust.py b/modules/bindings/pymod/kclust.py
index edcbc4797..076141f6b 100644
--- a/modules/bindings/pymod/kclust.py
+++ b/modules/bindings/pymod/kclust.py
@@ -56,8 +56,10 @@ def _CleanupFiles(tmp_dir_name):
def _ParseOutput(tmp_dir_name):
- header_data=open(os.path.join(tmp_dir_name,'headers.dmp'),'r').readlines()
- cluster_data=open(os.path.join(tmp_dir_name,'clusters.dmp'),'r').readlines()
+ with open(os.path.join(tmp_dir_name,'headers.dmp'),'r') as f:
+ header_data=f.readlines()
+ with open(os.path.join(tmp_dir_name,'clusters.dmp'),'r') as f:
+ cluster_data=f.readlines()
sequences=io.LoadSequenceList(os.path.join(tmp_dir_name,'fastadb.fasta'))
clusters=dict()
diff --git a/modules/bindings/tests/test_blast.py b/modules/bindings/tests/test_blast.py
index fbadfccb1..9be501b00 100644
--- a/modules/bindings/tests/test_blast.py
+++ b/modules/bindings/tests/test_blast.py
@@ -52,7 +52,8 @@ class TestBlastBindings(unittest.TestCase):
def testBlastParseOutput(self):
- raw_out=open('testfiles/raw_blastout.txt','r').read()
+ with open('testfiles/raw_blastout.txt','r') as f:
+ raw_out=f.read()
parsed_out=blast.ParseBlastOutput(raw_out)
diff --git a/modules/bindings/tests/test_hhblits.py b/modules/bindings/tests/test_hhblits.py
index eaff9a3f7..1bc6b3652 100644
--- a/modules/bindings/tests/test_hhblits.py
+++ b/modules/bindings/tests/test_hhblits.py
@@ -129,10 +129,10 @@ class TestHHblitsBindings(unittest.TestCase):
os.remove(self.tmpfile)
hhfile = self.hh.A3MToProfile("testfiles/testali.a3m",
hhm_file=self.tmpfile)
- tfh = open(hhfile)
- efh = open("testfiles/test.hmm")
- elst = efh.readlines()
- tlst = tfh.readlines()
+ with open(hhfile) as tfh:
+ tlst = tfh.readlines()
+ with open("testfiles/test.hmm") as efh:
+ elst = efh.readlines()
self.assertEqual(len(elst), len(tlst))
for i in range(0, len(elst)):
if not elst[i].startswith(('FILE', 'COM', 'DATE')):
@@ -147,10 +147,10 @@ class TestHHblitsBindings(unittest.TestCase):
'TSKYR')
self.hh = hhblits.HHblits(query_seq, self.hhroot)
hhfile = self.hh.A3MToProfile("testfiles/testali.a3m")
- tfh = open(hhfile)
- efh = open("testfiles/test.hmm")
- elst = efh.readlines()
- tlst = tfh.readlines()
+ with open(hhfile) as tfh:
+ tlst = tfh.readlines()
+ with open("testfiles/test.hmm") as efh:
+ elst = efh.readlines()
self.assertEqual(len(elst), len(tlst))
for i in range(0, len(elst)):
if not elst[i].startswith(('FILE', 'COM', 'DATE')):
@@ -244,10 +244,12 @@ class TestHHblitsBindings(unittest.TestCase):
self.hh = hhblits.HHblits(query_seq, self.hhroot)
search_file = self.hh.Search("testfiles/testali.a3m",
'testfiles/hhblitsdb/hhblitsdb')
- tfh = open(search_file)
- efh = open("testfiles/test.hhr")
- elst = efh.readlines()
- tlst = tfh.readlines()
+
+ with open(search_file) as tfh:
+ tlst = tfh.readlines()
+ with open("testfiles/test.hhr") as efh:
+ elst = efh.readlines()
+
self.assertEqual(len(elst), len(tlst))
for i in range(0, len(elst)):
if not elst[i].startswith(('Date', 'Command')):
@@ -286,7 +288,8 @@ class TestHHblitsBindings(unittest.TestCase):
def testParseHHMNotWorking(self):
# get info from an HHM file
with self.assertRaises(IOError) as ioe:
- hhblits.ParseHHM(open('testfiles/testali.a3m'))
+ with open('testfiles/testali.a3m') as f:
+ hhblits.ParseHHM(f)
self.assertEqual(ioe.exception.args[0],
'Profile file "testfiles/testali.a3m" is missing '+
'the "Consensus" section')
@@ -317,7 +320,8 @@ class TestHHblitsBindings(unittest.TestCase):
self.assertAlmostEqual(hit.ss_score, 34.1)
def testParseHHblitsOutput(self):
- header, hits = hhblits.ParseHHblitsOutput(open("testfiles/test.hhr"))
+ with open("testfiles/test.hhr") as f:
+ header, hits = hhblits.ParseHHblitsOutput(f)
self.assertEqual(header.query, 'Test')
self.assertEqual(header.match_columns, 141)
self.assertEqual(header.n_eff, 9.4)
--
GitLab