From 27ca82ca48b6ecce8eae80ea1174e475fcd07f05 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:12:30 +0800 Subject: [PATCH 1/9] Change api so object and settings are passed in at creation time. --- pyfpdb/CliFpdb.py | 4 +-- pyfpdb/GuiAutoImport.py | 4 +-- pyfpdb/GuiBulkImport.py | 8 ++--- pyfpdb/GuiTableViewer.py | 4 +-- pyfpdb/fpdb_import.py | 72 ++++++++++++++++++---------------------- 5 files changed, 42 insertions(+), 50 deletions(-) diff --git a/pyfpdb/CliFpdb.py b/pyfpdb/CliFpdb.py index 87c50025..e79a9d49 100755 --- a/pyfpdb/CliFpdb.py +++ b/pyfpdb/CliFpdb.py @@ -54,5 +54,5 @@ if __name__ == "__main__": (options, sys.argv) = parser.parse_args() settings={'imp-callFpdbHud':False, 'db-backend':2} - importer = fpdb_import.Importer() - importer.import_file_dict(options, settings) + importer = fpdb_import.Importer(options,settings) + importer.import_file_dict() diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index b155b9f4..79b25fee 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -57,7 +57,7 @@ class GuiAutoImport (threading.Thread): self.inputFile = os.path.join(self.path, file) stat_info = os.stat(self.inputFile) if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]: - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() self.import_files[self.inputFile] = stat_info.st_mtime print "GuiAutoImport.import_dir done" @@ -121,7 +121,7 @@ class GuiAutoImport (threading.Thread): def __init__(self, settings, debug=True): """Constructor for GuiAutoImport""" self.settings=settings - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) self.server=settings['db-host'] diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 4bd221cb..ed0a19b0 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -32,7 +32,7 @@ class GuiBulkImport (threading.Thread): print "BulkImport is not recursive - please select the final directory in which the history files are" else: self.inputFile=self.path+os.sep+file - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() print "GuiBulkImport.import_dir done" def load_clicked(self, widget, data=None): @@ -64,12 +64,10 @@ class GuiBulkImport (threading.Thread): else: self.failOnError=True - self.server, self.database, self.user, self.password=self.db.get_db_info() - if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() def get_vbox(self): """returns the vbox of this thread""" @@ -83,7 +81,7 @@ class GuiBulkImport (threading.Thread): def __init__(self, db, settings): self.db=db self.settings=settings - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self,self.settings) self.vbox=gtk.VBox(False,1) self.vbox.show() diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index 577be619..d7f3f8f2 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -255,9 +255,9 @@ class GuiTableViewer (threading.Thread): self.failOnError=False self.minPrint=0 self.handCount=0 - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self, self.settings) - self.last_read_hand_id=self.importer.import_file_dict(self, self.settings) + self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked def all_clicked(self, widget, data): diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 786aea59..46b97e7d 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -40,29 +40,32 @@ from time import time class Importer: - def __init__(self): + def __init__(self, options, settings): """Constructor""" - self.settings={'imp-callFpdbHud':False} + self.settings=settings + self.options=options self.db = None self.cursor = None - self.options = None self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file + if not self.settings.has_key('imp-callFpdbHud'): + self.settings['imp-callFpdbHud'] = False + self.dbConnect() - def dbConnect(self, options, settings): + def dbConnect(self): #connect to DB - if settings['db-backend'] == 2: + if self.settings['db-backend'] == 2: if not mysqlLibFound: raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file") - self.db = MySQLdb.connect(host = options.server, user = options.user, - passwd = options.password, db = options.database) - elif settings['db-backend'] == 3: + self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'], + self.settings['db-password'], self.settings['db-databaseName']) + elif self.settings['db-backend'] == 3: if not pgsqlLibFound: raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file") - self.db = psycopg2.connect(host = options.server, user = options.user, - password = options.password, database = options.database) - elif settings['db-backend'] == 4: + self.db = psycopg2.connect(self.settings['db-host'], self.settings['db-user'], + self.settings['db-password'], self.settings['db-databaseName']) + elif self.settings['db-backend'] == 4: pass else: pass @@ -71,32 +74,30 @@ class Importer: def setCallHud(self, value): self.callHud = value - def import_file_dict(self, options, settings): - self.options=options + def addImportFile(self, filename): + self.options.inputFile = filename + + def import_file_dict(self): starttime = time() last_read_hand=0 loc = 0 - if (options.inputFile=="stdin"): + if (self.options.inputFile=="stdin"): inputFile=sys.stdin else: - inputFile=open(options.inputFile, "rU") - try: loc = self.pos_in_file[options.inputFile] + inputFile=open(self.options.inputFile, "rU") + try: loc = self.pos_in_file[self.options.inputFile] except: pass - self.dbConnect(options,settings) - # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[options.inputFile] = inputFile.tell() + self.pos_in_file[self.options.inputFile] = inputFile.tell() inputFile.close() firstline = self.lines[0] if firstline.find("Tournament Summary")!=-1: print "TODO: implement importing tournament summaries" - self.cursor.close() - self.db.close() return 0 site=fpdb_simple.recogniseSite(firstline) @@ -151,41 +152,36 @@ class Importer: stored+=1 self.db.commit() # if settings['imp-callFpdbHud'] and self.callHud and os.sep=='/': - if settings['imp-callFpdbHud'] and self.callHud: + if self.settings['imp-callFpdbHud'] and self.callHud: #print "call to HUD here. handsId:",handsId #pipe the Hands.id out to the HUD -# options.pipe_to_hud.write("%s" % (handsId) + os.linesep) - options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + self.options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) except fpdb_simple.DuplicateError: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) - if (options.failOnError): + if (self.options.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. - self.cursor.close() - self.db.close() raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (options.failOnError): + if (self.options.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. - self.cursor.close() - self.db.close() raise - if (options.minPrint!=0): - if ((stored+duplicates+partial+errors)%options.minPrint==0): + if (self.options.minPrint!=0): + if ((stored+duplicates+partial+errors)%sielf.options.minPrint==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (options.handCount!=0): - if ((stored+duplicates+partial+errors)>=options.handCount): - if (not options.quiet): + if (self.options.handCount!=0): + if ((stored+duplicates+partial+errors)>=self.options.handCount): + if (not self.options.quiet): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) @@ -203,8 +199,6 @@ class Importer: handsId=0 #todo: this will cause return of an unstored hand number if the last hand was error or partial self.db.commit() - self.cursor.close() - self.db.close() return handsId #end def import_file_dict From ae9b70ea5d4caa476cf77f02a2c7319a435d89c2 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:19:57 +0800 Subject: [PATCH 2/9] Change variable name to indicate what it actually is. --- pyfpdb/fpdb_import.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 46b97e7d..bb4d21d3 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -40,10 +40,10 @@ from time import time class Importer: - def __init__(self, options, settings): + def __init__(self, caller, settings): """Constructor""" self.settings=settings - self.options=options + self.caller=caller self.db = None self.cursor = None self.callHud = False @@ -75,23 +75,23 @@ class Importer: self.callHud = value def addImportFile(self, filename): - self.options.inputFile = filename + self.caller.inputFile = filename def import_file_dict(self): starttime = time() last_read_hand=0 loc = 0 - if (self.options.inputFile=="stdin"): + if (self.caller.inputFile=="stdin"): inputFile=sys.stdin else: - inputFile=open(self.options.inputFile, "rU") - try: loc = self.pos_in_file[self.options.inputFile] + inputFile=open(self.caller.inputFile, "rU") + try: loc = self.pos_in_file[self.caller.inputFile] except: pass # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[self.options.inputFile] = inputFile.tell() + self.pos_in_file[self.caller.inputFile] = inputFile.tell() inputFile.close() firstline = self.lines[0] @@ -155,33 +155,33 @@ class Importer: if self.settings['imp-callFpdbHud'] and self.callHud: #print "call to HUD here. handsId:",handsId #pipe the Hands.id out to the HUD - self.options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) except fpdb_simple.DuplicateError: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) - if (self.options.failOnError): + if (self.caller.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (self.options.failOnError): + if (self.caller.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. raise - if (self.options.minPrint!=0): - if ((stored+duplicates+partial+errors)%sielf.options.minPrint==0): + if (self.caller.minPrint!=0): + if ((stored+duplicates+partial+errors)%self.caller.minPrint==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (self.options.handCount!=0): - if ((stored+duplicates+partial+errors)>=self.options.handCount): - if (not self.options.quiet): + if (self.caller.handCount!=0): + if ((stored+duplicates+partial+errors)>=self.caller.handCount): + if (not self.caller.quiet): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) @@ -204,7 +204,7 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:", self.options.inputFile + print "Filename:", self.caller.inputFile print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" print self.hand[0] From 1e8333ec5d65e0fd41064a77fd19cf9602a67f2e Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:42:08 +0800 Subject: [PATCH 3/9] Move minPrint variable from callling class to setting in Importer class --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/GuiBulkImport.py | 4 ++-- pyfpdb/GuiTableViewer.py | 2 +- pyfpdb/fpdb_import.py | 14 ++++++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 79b25fee..97653c77 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -123,6 +123,7 @@ class GuiAutoImport (threading.Thread): self.settings=settings self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) + self.importer.setMinPrint(30) self.server=settings['db-host'] self.user=settings['db-user'] @@ -130,7 +131,6 @@ class GuiAutoImport (threading.Thread): self.database=settings['db-databaseName'] self.quiet=False self.failOnError=False - self.minPrint=30 self.handCount=0 self.mainVBox=gtk.VBox(False,1) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index ed0a19b0..061e3848 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -48,9 +48,9 @@ class GuiBulkImport (threading.Thread): self.minPrint=self.min_print_tbuffer.get_text(self.min_print_tbuffer.get_start_iter(), self.min_print_tbuffer.get_end_iter()) if (self.minPrint=="never" or self.minPrint=="Never"): - self.minPrint=0 + self.importer.setMinPrint(0) else: - self.minPrint=int(self.minPrint) + self.importer.setMinPrint=int(self.minPrint) self.quiet=self.info_tbuffer.get_text(self.info_tbuffer.get_start_iter(), self.info_tbuffer.get_end_iter()) if (self.quiet=="yes"): diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index d7f3f8f2..2cccdbe5 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -253,9 +253,9 @@ class GuiTableViewer (threading.Thread): self.quiet=False self.failOnError=False - self.minPrint=0 self.handCount=0 self.importer = fpdb_import.Importer(self, self.settings) + self.importer.setMinPrint(0) self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index bb4d21d3..5a2aaab5 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -49,8 +49,11 @@ class Importer: self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file + #Set defaults if not self.settings.has_key('imp-callFpdbHud'): self.settings['imp-callFpdbHud'] = False + if not self.settings.has_key('minPrint'): + self.settings['minPrint'] = 30 self.dbConnect() def dbConnect(self): @@ -77,6 +80,9 @@ class Importer: def addImportFile(self, filename): self.caller.inputFile = filename + def setMinPrint(self, value): + self.settings['minPrint'] = int(value) + def import_file_dict(self): starttime = time() last_read_hand=0 @@ -163,7 +169,7 @@ class Importer: self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) if (self.caller.failOnError): - self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. + self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: errors+=1 @@ -173,10 +179,10 @@ class Importer: self.db.rollback() if (self.caller.failOnError): - self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. + self.db.commit() #dont remove this, in case hand processing was cancelled. raise - if (self.caller.minPrint!=0): - if ((stored+duplicates+partial+errors)%self.caller.minPrint==0): + if (self.settings['minPrint']!=0): + if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors if (self.caller.handCount!=0): From 638a6d6dabf2f122eb3c27c0c04f4be37ac2aeb6 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 02:14:06 +0800 Subject: [PATCH 4/9] Move more "options" from calling class into settings has of fpdb_import and fix all callers --- pyfpdb/GuiAutoImport.py | 6 +++--- pyfpdb/GuiBulkImport.py | 12 ++++++------ pyfpdb/GuiTableViewer.py | 6 +++--- pyfpdb/fpdb_import.py | 19 ++++++++++++++----- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 97653c77..2c4246dc 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -124,14 +124,14 @@ class GuiAutoImport (threading.Thread): self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) self.importer.setMinPrint(30) + self.importer.setQuiet(False) + self.importer.setFailOnError(False) + self.importer.setHandCount(0) self.server=settings['db-host'] self.user=settings['db-user'] self.password=settings['db-password'] self.database=settings['db-databaseName'] - self.quiet=False - self.failOnError=False - self.handCount=0 self.mainVBox=gtk.VBox(False,1) self.mainVBox.show() diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 061e3848..4f3d7ef6 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -40,9 +40,9 @@ class GuiBulkImport (threading.Thread): self.handCount=self.hand_count_tbuffer.get_text(self.hand_count_tbuffer.get_start_iter(), self.hand_count_tbuffer.get_end_iter()) if (self.handCount=="unlimited" or self.handCount=="Unlimited"): - self.handCount=0 + self.importer.setHandCount(0) else: - self.handCount=int(self.handCount) + self.importer.setHandCount(int(self.handCount)) self.errorFile="failed.txt" @@ -54,15 +54,15 @@ class GuiBulkImport (threading.Thread): self.quiet=self.info_tbuffer.get_text(self.info_tbuffer.get_start_iter(), self.info_tbuffer.get_end_iter()) if (self.quiet=="yes"): - self.quiet=False + self.importer.setQuiet(False) else: - self.quiet=True + self.importer.setQuiet(True) self.failOnError=self.fail_error_tbuffer.get_text(self.fail_error_tbuffer.get_start_iter(), self.fail_error_tbuffer.get_end_iter()) if (self.failOnError=="no"): - self.failOnError=False + self.importer.setFailOnError(False) else: - self.failOnError=True + self.importer.setFailOnError(True) if os.path.isdir(self.inputFile): self.import_dir() diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index 2cccdbe5..f2bfecd9 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -251,11 +251,11 @@ class GuiTableViewer (threading.Thread): self.user=self.db.user self.password=self.db.password - self.quiet=False - self.failOnError=False - self.handCount=0 self.importer = fpdb_import.Importer(self, self.settings) self.importer.setMinPrint(0) + self.importer.setQuiet(False) + self.importer.setFailOnError(False) + self.importer.setHandCount(0) self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5a2aaab5..9c2955eb 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -83,6 +83,15 @@ class Importer: def setMinPrint(self, value): self.settings['minPrint'] = int(value) + def setHandCount(self, value): + self.settings['handCount'] = int(value) + + def setQuiet(self, value): + self.settings['quiet'] = value + + def setFailOnError(self, value): + self.settings['failOnError'] = value + def import_file_dict(self): starttime = time() last_read_hand=0 @@ -168,7 +177,7 @@ class Importer: errors+=1 self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) - if (self.caller.failOnError): + if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: @@ -178,16 +187,16 @@ class Importer: #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (self.caller.failOnError): + if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise if (self.settings['minPrint']!=0): if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (self.caller.handCount!=0): - if ((stored+duplicates+partial+errors)>=self.caller.handCount): - if (not self.caller.quiet): + if (self.settings['handCount']!=0): + if ((stored+duplicates+partial+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) From 6c4e2f3eb9287d5ec2606d293805d5cc6dcd5375 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 15:49:09 +0800 Subject: [PATCH 5/9] Changes import_file_dict interface to Importer internal. Users of Importer should now addImportFile and addImportDirectory, set paramaters, then run either runImport - all files or runUpdated - modified files. Comments out postgres regression test until fixed Adds initial code for importing regression test files. --- pyfpdb/GuiAutoImport.py | 25 +++------------- pyfpdb/GuiBulkImport.py | 12 ++++---- pyfpdb/RegressionTest.py | 61 ++++++++++++++++++++++---------------- pyfpdb/fpdb_import.py | 64 +++++++++++++++++++++++++++++++++------- 4 files changed, 98 insertions(+), 64 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 2c4246dc..9cc10906 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -50,16 +50,7 @@ class GuiAutoImport (threading.Thread): def do_import(self): """Callback for timer to do an import iteration.""" - for file in os.listdir(self.path): - if os.path.isdir(file): - print "AutoImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile = os.path.join(self.path, file) - stat_info = os.stat(self.inputFile) - if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]: - self.importer.import_file_dict() - self.import_files[self.inputFile] = stat_info.st_mtime - + self.importer.runUpdated() print "GuiAutoImport.import_dir done" return True @@ -96,17 +87,9 @@ class GuiAutoImport (threading.Thread): # self.pipe_to_hud = os.popen(command, 'w') self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter()) -# Iniitally populate the self.import_files dict, which keeps mtimes for the files watched - - self.import_files = {} - for file in os.listdir(self.path): - if os.path.isdir(file): - pass # skip subdirs for now - else: - inputFile = os.path.join(self.path, file) - stat_info = os.stat(inputFile) - self.import_files[inputFile] = stat_info.st_mtime - +# Add directory to importer object and set the initial mtime reference. + self.importer.addImportDirectory(self.path) + self.importer.setWatchTime() self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 4f3d7ef6..c2347528 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -27,12 +27,8 @@ class GuiBulkImport (threading.Thread): def import_dir(self): """imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it""" self.path=self.inputFile - for file in os.listdir(self.path): - if os.path.isdir(file): - print "BulkImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile=self.path+os.sep+file - self.importer.import_file_dict() + self.importer.addImportDirectory(self.path) + self.importer.runImport() print "GuiBulkImport.import_dir done" def load_clicked(self, widget, data=None): @@ -67,7 +63,9 @@ class GuiBulkImport (threading.Thread): if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.import_file_dict() + self.importer.addImportFile() + self.importer.runImport() + self.importer.clearFileList() def get_vbox(self): """returns the vbox of this thread""" diff --git a/pyfpdb/RegressionTest.py b/pyfpdb/RegressionTest.py index 8a8facc5..b4dd1b7d 100644 --- a/pyfpdb/RegressionTest.py +++ b/pyfpdb/RegressionTest.py @@ -25,51 +25,62 @@ import os import sys import fpdb_db +import fpdb_import import FpdbSQLQueries import unittest class TestSequenceFunctions(unittest.TestCase): - def setUp(self): - """Configure MySQL settings/database and establish connection""" - self.mysql_settings={ 'db-host':"localhost", 'db-backend':2, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.mysql_db = fpdb_db.fpdb_db() - self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], - self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], - self.mysql_settings['db-password']) - self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + def setUp(self): + """Configure MySQL settings/database and establish connection""" + self.mysql_settings={ 'db-host':"localhost", + 'db-backend':2, + 'db-databaseName':"fpdbtest", + 'db-user':"fpdb", + 'db-password':"fpdb"} + self.mysql_db = fpdb_db.fpdb_db() + self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], + self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], + self.mysql_settings['db-password']) + self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + self.mysqlimporter = fpdb_import.Importer(self, self.mysql_settings) - """Configure Postgres settings/database and establish connection""" - self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.pg_db = fpdb_db.fpdb_db() - self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], - self.pg_settings['db-databaseName'], self.pg_settings['db-user'], - self.pg_settings['db-password']) - self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') +# """Configure Postgres settings/database and establish connection""" +# self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} +# self.pg_db = fpdb_db.fpdb_db() +# self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], +# self.pg_settings['db-databaseName'], self.pg_settings['db-user'], +# self.pg_settings['db-password']) +# self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') def testDatabaseConnection(self): """Test all supported DBs""" self.result = self.mysql_db.cursor.execute(self.mysqldict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - - print self.pgdict.query['list_tables'] - - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testMySQLRecreateTables(self): """Test droping then recreating fpdb table schema""" self.mysql_db.recreate_tables() self.result = self.mysql_db.cursor.execute("SHOW TABLES") self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - def testPostgresSQLRecreateTables(self): - """Test droping then recreating fpdb table schema""" - self.pg_db.recreate_tables() - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testImportHandHistoryFiles(self): + """Test import of single HH file""" + self.mysqlimporter.addImportFile("regression-test-files/hand-histories/ps-lhe-ring-3hands.txt") + self.mysqlimporter.runImport() + self.mysqlimporter.addImportDirectory("regression-test-files/hand-histories") + self.mysqlimporter.runImport() + +# def testPostgresSQLRecreateTables(self): +# """Test droping then recreating fpdb table schema""" +# self.pg_db.recreate_tables() +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) if __name__ == '__main__': unittest.main() diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 9c2955eb..c844c00d 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -46,6 +46,9 @@ class Importer: self.caller=caller self.db = None self.cursor = None + self.filelist = [] + self.queued = [] + self.updated = 0 #Time last import was run, used as mtime reference self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file @@ -74,12 +77,10 @@ class Importer: pass self.cursor = self.db.cursor() + #Set functions def setCallHud(self, value): self.callHud = value - def addImportFile(self, filename): - self.caller.inputFile = filename - def setMinPrint(self, value): self.settings['minPrint'] = int(value) @@ -92,21 +93,62 @@ class Importer: def setFailOnError(self, value): self.settings['failOnError'] = value - def import_file_dict(self): + def setWatchTime(self): + self.updated = time() + + def clearFileList(self): + self.filelist = [] + + #Add an individual file to filelist + def addImportFile(self, filename): + #todo: test it is a valid file + self.filelist = self.filelist + [filename] + print "Filelist in addImportFile: ", self.filelist + #Remove duplicates + set(filelist) + + #Add a directory of files to filelist + def addImportDirectory(self,dir): + #todo: test it is a valid directory + for file in os.listdir(dir): + if os.path.isdir(file): + print "BulkImport is not recursive - please select the final directory in which the history files are" + else: + blah = [dir+os.sep+file] + self.filelist = self.filelist + [dir+os.sep+file] + #Remove duplicates + set(self.filelist) + + #Run full import on filelist + def runImport(self): + for file in self.filelist: + print "Importing file: ", file + self.import_file_dict(file) + + #Run import on updated files, then store latest update time. + def runUpdated(self): + for file in self.filelist: + stat_info = os.stat(file) + if stat_info.st_mtime > self.updated: + self.import_file_dict(file) + self.updated = time() + + # This is now an internal function that should not be called directly. + def import_file_dict(self, file): starttime = time() last_read_hand=0 loc = 0 - if (self.caller.inputFile=="stdin"): + if (file=="stdin"): inputFile=sys.stdin else: - inputFile=open(self.caller.inputFile, "rU") - try: loc = self.pos_in_file[self.caller.inputFile] + inputFile=open(file, "rU") + try: loc = self.pos_in_file[file] except: pass # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[self.caller.inputFile] = inputFile.tell() + self.pos_in_file[file] = inputFile.tell() inputFile.close() firstline = self.lines[0] @@ -175,14 +217,14 @@ class Importer: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() @@ -219,7 +261,7 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:", self.caller.inputFile + print "Filename:", filename print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" print self.hand[0] From c2c37dc5ee095f8bf7bc4b7143b657b671d78ba4 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 18:21:42 +0800 Subject: [PATCH 6/9] Fix mtime time detection --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/fpdb_import.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 9cc10906..23493664 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -89,7 +89,6 @@ class GuiAutoImport (threading.Thread): # Add directory to importer object and set the initial mtime reference. self.importer.addImportDirectory(self.path) - self.importer.setWatchTime() self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) @@ -110,6 +109,7 @@ class GuiAutoImport (threading.Thread): self.importer.setQuiet(False) self.importer.setFailOnError(False) self.importer.setHandCount(0) + self.importer.setWatchTime() self.server=settings['db-host'] self.user=settings['db-user'] diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c844c00d..458b34d5 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -130,6 +130,7 @@ class Importer: for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: + # print "File: " + str(file) + " mtime: " + str(stat_info.st_mtime) + " update: " + str(self.updated) self.import_file_dict(file) self.updated = time() From 8ca5a5c9657c5f0ba01fb7ae0d5775679a2a54a3 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 01:26:04 +0800 Subject: [PATCH 7/9] Should fix detection of new hh files. --- pyfpdb/fpdb_import.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 458b34d5..73e141e7 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -47,7 +47,8 @@ class Importer: self.db = None self.cursor = None self.filelist = [] - self.queued = [] + self.dirlist = [] + self.monitor = False self.updated = 0 #Time last import was run, used as mtime reference self.callHud = False self.lines = None @@ -108,13 +109,16 @@ class Importer: set(filelist) #Add a directory of files to filelist - def addImportDirectory(self,dir): + def addImportDirectory(self,dir,monitor = False): #todo: test it is a valid directory + if monitor == True: + self.monitor = True + self.dirlist = self.dirlist + [dir] + for file in os.listdir(dir): if os.path.isdir(file): print "BulkImport is not recursive - please select the final directory in which the history files are" else: - blah = [dir+os.sep+file] self.filelist = self.filelist + [dir+os.sep+file] #Remove duplicates set(self.filelist) @@ -127,6 +131,13 @@ class Importer: #Run import on updated files, then store latest update time. def runUpdated(self): + #Check for new files in directory + #todo: make efficient - always checks for new file, should be able to use mtime of directory + # ^^ May not work on windows + for dir in self.dirlist: + for file in os.listdir(dir): + self.filelist = self.filelist + [dir+os.sep+file] + for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: From 9a60cf84c8cd90894dd0af14a09d88a57f6f3e6e Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 02:06:28 +0800 Subject: [PATCH 8/9] Real fix this time, modify GuiAutoImport to actually tell importer to monitor the directory --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/fpdb_import.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 23493664..6953d1f9 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -88,7 +88,7 @@ class GuiAutoImport (threading.Thread): self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter()) # Add directory to importer object and set the initial mtime reference. - self.importer.addImportDirectory(self.path) + self.importer.addImportDirectory(self.path, True) self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 73e141e7..fdf3ab3c 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -119,7 +119,7 @@ class Importer: if os.path.isdir(file): print "BulkImport is not recursive - please select the final directory in which the history files are" else: - self.filelist = self.filelist + [dir+os.sep+file] + self.filelist = self.filelist + [os.path.join(dir, file)] #Remove duplicates set(self.filelist) @@ -137,6 +137,7 @@ class Importer: for dir in self.dirlist: for file in os.listdir(dir): self.filelist = self.filelist + [dir+os.sep+file] + set(filelist) for file in self.filelist: stat_info = os.stat(file) From 3e9ef61fc893a01fb3441d0f173fff75bf089c31 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 23:34:18 +0800 Subject: [PATCH 9/9] Fix bug: incorrect use of set function to remove duplicates Now detects new tables without issues --- pyfpdb/fpdb_import.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fdf3ab3c..b652f204 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -104,9 +104,8 @@ class Importer: def addImportFile(self, filename): #todo: test it is a valid file self.filelist = self.filelist + [filename] - print "Filelist in addImportFile: ", self.filelist #Remove duplicates - set(filelist) + self.filelist = list(set(self.filelist)) #Add a directory of files to filelist def addImportDirectory(self,dir,monitor = False): @@ -121,12 +120,11 @@ class Importer: else: self.filelist = self.filelist + [os.path.join(dir, file)] #Remove duplicates - set(self.filelist) + self.filelist = list(set(self.filelist)) #Run full import on filelist def runImport(self): for file in self.filelist: - print "Importing file: ", file self.import_file_dict(file) #Run import on updated files, then store latest update time. @@ -137,12 +135,12 @@ class Importer: for dir in self.dirlist: for file in os.listdir(dir): self.filelist = self.filelist + [dir+os.sep+file] - set(filelist) + + self.filelist = list(set(self.filelist)) for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: - # print "File: " + str(file) + " mtime: " + str(stat_info.st_mtime) + " update: " + str(self.updated) self.import_file_dict(file) self.updated = time()