From 6c4e2f3eb9287d5ec2606d293805d5cc6dcd5375 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 15:49:09 +0800 Subject: [PATCH] Changes import_file_dict interface to Importer internal. Users of Importer should now addImportFile and addImportDirectory, set paramaters, then run either runImport - all files or runUpdated - modified files. Comments out postgres regression test until fixed Adds initial code for importing regression test files. --- pyfpdb/GuiAutoImport.py | 25 +++------------- pyfpdb/GuiBulkImport.py | 12 ++++---- pyfpdb/RegressionTest.py | 61 ++++++++++++++++++++++---------------- pyfpdb/fpdb_import.py | 64 +++++++++++++++++++++++++++++++++------- 4 files changed, 98 insertions(+), 64 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 2c4246dc..9cc10906 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -50,16 +50,7 @@ class GuiAutoImport (threading.Thread): def do_import(self): """Callback for timer to do an import iteration.""" - for file in os.listdir(self.path): - if os.path.isdir(file): - print "AutoImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile = os.path.join(self.path, file) - stat_info = os.stat(self.inputFile) - if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]: - self.importer.import_file_dict() - self.import_files[self.inputFile] = stat_info.st_mtime - + self.importer.runUpdated() print "GuiAutoImport.import_dir done" return True @@ -96,17 +87,9 @@ class GuiAutoImport (threading.Thread): # self.pipe_to_hud = os.popen(command, 'w') self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter()) -# Iniitally populate the self.import_files dict, which keeps mtimes for the files watched - - self.import_files = {} - for file in os.listdir(self.path): - if os.path.isdir(file): - pass # skip subdirs for now - else: - inputFile = os.path.join(self.path, file) - stat_info = os.stat(inputFile) - self.import_files[inputFile] = stat_info.st_mtime - +# Add directory to importer object and set the initial mtime reference. + self.importer.addImportDirectory(self.path) + self.importer.setWatchTime() self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 4f3d7ef6..c2347528 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -27,12 +27,8 @@ class GuiBulkImport (threading.Thread): def import_dir(self): """imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it""" self.path=self.inputFile - for file in os.listdir(self.path): - if os.path.isdir(file): - print "BulkImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile=self.path+os.sep+file - self.importer.import_file_dict() + self.importer.addImportDirectory(self.path) + self.importer.runImport() print "GuiBulkImport.import_dir done" def load_clicked(self, widget, data=None): @@ -67,7 +63,9 @@ class GuiBulkImport (threading.Thread): if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.import_file_dict() + self.importer.addImportFile() + self.importer.runImport() + self.importer.clearFileList() def get_vbox(self): """returns the vbox of this thread""" diff --git a/pyfpdb/RegressionTest.py b/pyfpdb/RegressionTest.py index 8a8facc5..b4dd1b7d 100644 --- a/pyfpdb/RegressionTest.py +++ b/pyfpdb/RegressionTest.py @@ -25,51 +25,62 @@ import os import sys import fpdb_db +import fpdb_import import FpdbSQLQueries import unittest class TestSequenceFunctions(unittest.TestCase): - def setUp(self): - """Configure MySQL settings/database and establish connection""" - self.mysql_settings={ 'db-host':"localhost", 'db-backend':2, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.mysql_db = fpdb_db.fpdb_db() - self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], - self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], - self.mysql_settings['db-password']) - self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + def setUp(self): + """Configure MySQL settings/database and establish connection""" + self.mysql_settings={ 'db-host':"localhost", + 'db-backend':2, + 'db-databaseName':"fpdbtest", + 'db-user':"fpdb", + 'db-password':"fpdb"} + self.mysql_db = fpdb_db.fpdb_db() + self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], + self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], + self.mysql_settings['db-password']) + self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + self.mysqlimporter = fpdb_import.Importer(self, self.mysql_settings) - """Configure Postgres settings/database and establish connection""" - self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.pg_db = fpdb_db.fpdb_db() - self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], - self.pg_settings['db-databaseName'], self.pg_settings['db-user'], - self.pg_settings['db-password']) - self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') +# """Configure Postgres settings/database and establish connection""" +# self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} +# self.pg_db = fpdb_db.fpdb_db() +# self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], +# self.pg_settings['db-databaseName'], self.pg_settings['db-user'], +# self.pg_settings['db-password']) +# self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') def testDatabaseConnection(self): """Test all supported DBs""" self.result = self.mysql_db.cursor.execute(self.mysqldict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - - print self.pgdict.query['list_tables'] - - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testMySQLRecreateTables(self): """Test droping then recreating fpdb table schema""" self.mysql_db.recreate_tables() self.result = self.mysql_db.cursor.execute("SHOW TABLES") self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - def testPostgresSQLRecreateTables(self): - """Test droping then recreating fpdb table schema""" - self.pg_db.recreate_tables() - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testImportHandHistoryFiles(self): + """Test import of single HH file""" + self.mysqlimporter.addImportFile("regression-test-files/hand-histories/ps-lhe-ring-3hands.txt") + self.mysqlimporter.runImport() + self.mysqlimporter.addImportDirectory("regression-test-files/hand-histories") + self.mysqlimporter.runImport() + +# def testPostgresSQLRecreateTables(self): +# """Test droping then recreating fpdb table schema""" +# self.pg_db.recreate_tables() +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) if __name__ == '__main__': unittest.main() diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 9c2955eb..c844c00d 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -46,6 +46,9 @@ class Importer: self.caller=caller self.db = None self.cursor = None + self.filelist = [] + self.queued = [] + self.updated = 0 #Time last import was run, used as mtime reference self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file @@ -74,12 +77,10 @@ class Importer: pass self.cursor = self.db.cursor() + #Set functions def setCallHud(self, value): self.callHud = value - def addImportFile(self, filename): - self.caller.inputFile = filename - def setMinPrint(self, value): self.settings['minPrint'] = int(value) @@ -92,21 +93,62 @@ class Importer: def setFailOnError(self, value): self.settings['failOnError'] = value - def import_file_dict(self): + def setWatchTime(self): + self.updated = time() + + def clearFileList(self): + self.filelist = [] + + #Add an individual file to filelist + def addImportFile(self, filename): + #todo: test it is a valid file + self.filelist = self.filelist + [filename] + print "Filelist in addImportFile: ", self.filelist + #Remove duplicates + set(filelist) + + #Add a directory of files to filelist + def addImportDirectory(self,dir): + #todo: test it is a valid directory + for file in os.listdir(dir): + if os.path.isdir(file): + print "BulkImport is not recursive - please select the final directory in which the history files are" + else: + blah = [dir+os.sep+file] + self.filelist = self.filelist + [dir+os.sep+file] + #Remove duplicates + set(self.filelist) + + #Run full import on filelist + def runImport(self): + for file in self.filelist: + print "Importing file: ", file + self.import_file_dict(file) + + #Run import on updated files, then store latest update time. + def runUpdated(self): + for file in self.filelist: + stat_info = os.stat(file) + if stat_info.st_mtime > self.updated: + self.import_file_dict(file) + self.updated = time() + + # This is now an internal function that should not be called directly. + def import_file_dict(self, file): starttime = time() last_read_hand=0 loc = 0 - if (self.caller.inputFile=="stdin"): + if (file=="stdin"): inputFile=sys.stdin else: - inputFile=open(self.caller.inputFile, "rU") - try: loc = self.pos_in_file[self.caller.inputFile] + inputFile=open(file, "rU") + try: loc = self.pos_in_file[file] except: pass # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[self.caller.inputFile] = inputFile.tell() + self.pos_in_file[file] = inputFile.tell() inputFile.close() firstline = self.lines[0] @@ -175,14 +217,14 @@ class Importer: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() @@ -219,7 +261,7 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:", self.caller.inputFile + print "Filename:", filename print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" print self.hand[0]