From 35625f6d29234229217c00d4246f71aaa8df5fdb Mon Sep 17 00:00:00 2001 From: steffen123 Date: Thu, 9 Oct 2008 21:21:40 +0100 Subject: [PATCH 01/15] p131 - insignificant changes to fpdb.py --- pyfpdb/fpdb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyfpdb/fpdb.py b/pyfpdb/fpdb.py index da58e6ef..07246ca8 100755 --- a/pyfpdb/fpdb.py +++ b/pyfpdb/fpdb.py @@ -351,7 +351,8 @@ class fpdb: #end def not_implemented def obtain_global_lock(self): - print "todo: implement obtain_global_lock (users: pls ignore this)" + #print "todo: implement obtain_global_lock (users: pls ignore this)" + pass #end def obtain_global_lock def quit(self, widget, data): @@ -421,7 +422,7 @@ This program is licensed under the AGPL3, see docs"""+os.sep+"agpl-3.0.txt") self.window = gtk.Window(gtk.WINDOW_TOPLEVEL) self.window.connect("delete_event", self.delete_event) self.window.connect("destroy", self.destroy) - self.window.set_title("Free Poker DB - version: alpha6+, p124 or higher") + self.window.set_title("Free Poker DB - version: alpha6+, p131 or higher") self.window.set_border_width(1) self.window.set_size_request(1020,400) self.window.set_resizable(True) From 4cc0eb539ee27e4abe2eea99ac7dde33e72f0278 Mon Sep 17 00:00:00 2001 From: steffen123 Date: Fri, 10 Oct 2008 04:14:26 +0100 Subject: [PATCH 02/15] p132 - fixed tv to new importer. added CliFpdb as initial CLI importer interface. fixed fpdb.printEmailError --- pyfpdb/CliFpdb.py | 83 +++++++++++++++++++ pyfpdb/GuiTableViewer.py | 2 +- pyfpdb/fpdb_import.py | 9 +- regression-test/ps-studhilo-ring-showdown.txt | 63 ++++++++++++++ regression-test/regression-test.sh | 8 +- 5 files changed, 156 insertions(+), 9 deletions(-) create mode 100755 pyfpdb/CliFpdb.py diff --git a/pyfpdb/CliFpdb.py b/pyfpdb/CliFpdb.py new file mode 100755 index 00000000..9b0fcc99 --- /dev/null +++ b/pyfpdb/CliFpdb.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +#Copyright 2008 Steffen Jobbagy-Felso +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, version 3 of the License. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . +#In the "official" distribution you can find the license in +#agpl-3.0.txt in the docs folder of the package. + +import os +import sys +import fpdb_simple +from optparse import OptionParser + +try: + import MySQLdb +except: + diaSQLLibMissing = gtk.Dialog(title="Fatal Error - SQL interface library missing", parent=None, flags=0, buttons=(gtk.STOCK_QUIT,gtk.RESPONSE_OK)) + + print "Please note that the CLI importer only works with MySQL, if you use PostgreSQL this error is expected." + +import fpdb_import +import fpdb_db + + +# def __init__(self, db, settings, debug=True): +# """Constructor for table_viewer""" +# self.debug=debug +# #print "start of table_viewer constructor" +# self.db=db +# self.cursor=db.cursor +# self.settings=settings + +if __name__ == "__main__": + failOnError=False + quiet=False + + #process CLI parameters + parser = OptionParser() + parser.add_option("-c", "--handCount", default="0", type="int", + help="Number of hands to import (default 0 means unlimited)") + parser.add_option("-d", "--database", default="fpdb", help="The MySQL database to use (default fpdb)") + parser.add_option("-e", "--errorFile", default="failed.txt", + help="File to store failed hands into. (default: failed.txt) Not implemented.") + parser.add_option("-f", "--inputFile", "--file", "--inputfile", default="stdin", + help="The file you want to import (remember to use quotes if necessary)") + parser.add_option("-m", "--minPrint", "--status", default="50", type="int", + help="How often to print a one-line status report (0 means never, default is 50)") + parser.add_option("-p", "--password", help="The password for the MySQL user") + parser.add_option("-q", "--quiet", action="store_true", + help="If this is passed it doesn't print a total at the end nor the opening line. Note that this purposely does NOT change --minPrint") + parser.add_option("-s", "--server", default="localhost", + help="Hostname/IP of the MySQL server (default localhost)") + parser.add_option("-u", "--user", default="fpdb", help="The MySQL username (default fpdb)") + parser.add_option("-x", "--failOnError", action="store_true", + help="If this option is passed it quits when it encounters any error") + + (options, sys.argv) = parser.parse_args() + + settings={'imp-callFpdbHud':False, 'db-backend':2} + + #self.inputFile=options.inputFile + + #self.server=options.server + #self.database=options.database + #self.user=options.user + #self.password=options.password + + #self.quiet=False + #self.failOnError=False + #self.minPrint=0 + #self.handCount=0 + importer = fpdb_import.Importer() + + importer.import_file_dict(options, settings) diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index 6b5cb398..577be619 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -257,7 +257,7 @@ class GuiTableViewer (threading.Thread): self.handCount=0 self.importer = fpdb_import.Importer() - self.last_read_hand_id=importer.import_file_dict(self, self.settings) + self.last_read_hand_id=self.importer.import_file_dict(self, self.settings) #end def table_viewer.import_clicked def all_clicked(self, widget, data): diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index b510f9dc..db15ed54 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -36,7 +36,6 @@ import os import datetime import fpdb_simple import fpdb_parse_logic -from optparse import OptionParser from time import time class Importer: @@ -72,6 +71,7 @@ class Importer: self.callHud = value def import_file_dict(self, options, settings): + self.options=options starttime = time() last_read_hand=0 if (options.inputFile=="stdin"): @@ -136,7 +136,8 @@ class Importer: if not isTourney: fpdb_simple.filterAnteBlindFold(site,hand) hand=fpdb_simple.filterCrap(site, hand, isTourney) - + self.hand=hand + try: handsId=fpdb_parse_logic.mainParser(self.db, self.cursor, site, category, hand) self.db.commit() @@ -203,9 +204,9 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:",options.inputFile + print "Filename:", self.options.inputFile print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" - print hand[0] + print self.hand[0] if __name__ == "__main__": diff --git a/regression-test/ps-studhilo-ring-showdown.txt b/regression-test/ps-studhilo-ring-showdown.txt index 7c69615b..ff1276e8 100644 --- a/regression-test/ps-studhilo-ring-showdown.txt +++ b/regression-test/ps-studhilo-ring-showdown.txt @@ -65,3 +65,66 @@ Seat 8: willowdale showed [3s 4d 5s 4s Ad 5h 2c] and won ($1.02) with HI: a stra +PokerStars Game #20711747191: Razz Limit ($1/$2) - 2008/09/26 14:36:50 ET +Table 'Siwa III' 8-max +Seat 2: dainmat ($63.15 in chips) +Seat 3: fnJ's ($30.75 in chips) +Seat 4: ambythegreat ($24.90 in chips) +Seat 5: jt studd ($39.90 in chips) +Seat 6: KyleHruby ($11.55 in chips) +Seat 7: nutOmatic ($71.30 in chips) +dainmat: posts the ante $0.10 +fnJ's: posts the ante $0.10 +ambythegreat: posts the ante $0.10 +jt studd: posts the ante $0.10 +KyleHruby: posts the ante $0.10 +nutOmatic: posts the ante $0.10 +*** 3rd STREET *** +Dealt to dainmat [Ts] +Dealt to fnJ's [3s] +Dealt to ambythegreat [5h] +Dealt to jt studd [Ad] +Dealt to KyleHruby [As] +Dealt to nutOmatic [6d Kd Th] +dainmat: brings in for $0.50 +fnJ's: calls $0.50 +ambythegreat: raises $0.50 to $1 +garnishgut joins the table at seat #1 +jt studd: folds +KyleHruby: folds +nutOmatic: folds +dainmat: folds +fnJ's: calls $0.50 +*** 4th STREET *** +Dealt to fnJ's [3s] [6h] +Dealt to ambythegreat [5h] [4c] +ambythegreat: bets $1 +fnJ's: calls $1 +*** 5th STREET *** +Dealt to fnJ's [3s 6h] [2h] +Dealt to ambythegreat [5h 4c] [Kh] +fnJ's: bets $2 +ambythegreat: calls $2 +*** 6th STREET *** +Dealt to fnJ's [3s 6h 2h] [2c] +Dealt to ambythegreat [5h 4c Kh] [5s] +fnJ's: bets $2 +ambythegreat: calls $2 +*** RIVER *** +fnJ's: bets $2 +ambythegreat: calls $2 +*** SHOW DOWN *** +fnJ's: shows [9d 5c 3s 6h 2h 2c 4h] (Lo: 6,5,4,3,2) +ambythegreat: mucks hand +fnJ's collected $16.35 from pot +*** SUMMARY *** +Total pot $17.10 | Rake $0.75 +Seat 2: dainmat folded on the 3rd Street +Seat 3: fnJ's showed [9d 5c 3s 6h 2h 2c 4h] and won ($16.35) with Lo: 6,5,4,3,2 +Seat 4: ambythegreat mucked [6s 7h 5h 4c Kh 5s Jc] +Seat 5: jt studd folded on the 3rd Street (didn't bet) +Seat 6: KyleHruby folded on the 3rd Street (didn't bet) +Seat 7: nutOmatic folded on the 3rd Street (didn't bet) + + + diff --git a/regression-test/regression-test.sh b/regression-test/regression-test.sh index 3cc6078e..44a1a1f7 100755 --- a/regression-test/regression-test.sh +++ b/regression-test/regression-test.sh @@ -18,8 +18,8 @@ echo "Please note for this to work you need to work on an empty database, otherwise some info (the id fields) will be off" rm *.found.txt -../pyfpdb/fpdb_import.py -p$1 --file=ps-lhe-ring-3hands.txt -x -../pyfpdb/fpdb_import.py -p$1 --file=ps-lhe-ring-3hands.txt -x +../pyfpdb/CliFpdb.py -p$1 --file=ps-lhe-ring-3hands.txt -x +../pyfpdb/CliFpdb.py -p$1 --file=ps-lhe-ring-3hands.txt -x echo "it should've reported first that it stored 3, then that it had 3 duplicates" @@ -31,12 +31,12 @@ echo "it should've reported first that it stored 3, then that it had 3 duplicate ./PrintPlayerHudData.py -p$1 -nPlayer_5 -oB > ps-flags-B-1hands.found.txt && colordiff ps-flags-B-1hands.found.txt ps-flags-B-1hands.expected.txt -../pyfpdb/fpdb_import.py -p$1 --file=ps-lhe-ring-call-3B-preflop-cb-no2b.txt -x +../pyfpdb/CliFpdb.py -p$1 --file=ps-lhe-ring-call-3B-preflop-cb-no2b.txt -x echo "it should've now reported another successful store of 1 hand" ./PrintPlayerHudData.py -p$1 -nplayer3 -oE -e10 -b50 > ps-flags-CBflop.found.txt && colordiff ps-flags-CBflop.found.txt ps-flags-CBflop.expected.txt -../pyfpdb/fpdb_import.py -p$1 --file=ps-studhilo-ring-showdown.txt -x +../pyfpdb/CliFpdb.py -p$1 --file=ps-studhilo-ring-showdown.txt -x echo "it should've now reported another successful store of 1 hand" ./PrintHand.py -p$1 --hand=15043388146 > ps.15043388146.found.txt && colordiff ps.15043388146.found.txt ps.15043388146.expected.txt ./PrintPlayerHudData.py -p$1 -nbr1an -o0 -e6 -b20 -cstudhilo> ps-flags-studhilo.found.txt && colordiff ps-flags-studhilo.found.txt ps-flags-studhilo.expected.txt From c871ce067cfafed3ada6051c312ff1966f1ba628 Mon Sep 17 00:00:00 2001 From: steffen123 Date: Fri, 10 Oct 2008 04:29:12 +0100 Subject: [PATCH 03/15] p133 - cleaned CliFpdb.py --- pyfpdb/CliFpdb.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pyfpdb/CliFpdb.py b/pyfpdb/CliFpdb.py index 9b0fcc99..87c50025 100755 --- a/pyfpdb/CliFpdb.py +++ b/pyfpdb/CliFpdb.py @@ -30,19 +30,7 @@ except: import fpdb_import import fpdb_db - -# def __init__(self, db, settings, debug=True): -# """Constructor for table_viewer""" -# self.debug=debug -# #print "start of table_viewer constructor" -# self.db=db -# self.cursor=db.cursor -# self.settings=settings - if __name__ == "__main__": - failOnError=False - quiet=False - #process CLI parameters parser = OptionParser() parser.add_option("-c", "--handCount", default="0", type="int", @@ -66,18 +54,5 @@ if __name__ == "__main__": (options, sys.argv) = parser.parse_args() settings={'imp-callFpdbHud':False, 'db-backend':2} - - #self.inputFile=options.inputFile - - #self.server=options.server - #self.database=options.database - #self.user=options.user - #self.password=options.password - - #self.quiet=False - #self.failOnError=False - #self.minPrint=0 - #self.handCount=0 importer = fpdb_import.Importer() - importer.import_file_dict(options, settings) From 27ca82ca48b6ecce8eae80ea1174e475fcd07f05 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:12:30 +0800 Subject: [PATCH 04/15] Change api so object and settings are passed in at creation time. --- pyfpdb/CliFpdb.py | 4 +-- pyfpdb/GuiAutoImport.py | 4 +-- pyfpdb/GuiBulkImport.py | 8 ++--- pyfpdb/GuiTableViewer.py | 4 +-- pyfpdb/fpdb_import.py | 72 ++++++++++++++++++---------------------- 5 files changed, 42 insertions(+), 50 deletions(-) diff --git a/pyfpdb/CliFpdb.py b/pyfpdb/CliFpdb.py index 87c50025..e79a9d49 100755 --- a/pyfpdb/CliFpdb.py +++ b/pyfpdb/CliFpdb.py @@ -54,5 +54,5 @@ if __name__ == "__main__": (options, sys.argv) = parser.parse_args() settings={'imp-callFpdbHud':False, 'db-backend':2} - importer = fpdb_import.Importer() - importer.import_file_dict(options, settings) + importer = fpdb_import.Importer(options,settings) + importer.import_file_dict() diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index b155b9f4..79b25fee 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -57,7 +57,7 @@ class GuiAutoImport (threading.Thread): self.inputFile = os.path.join(self.path, file) stat_info = os.stat(self.inputFile) if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]: - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() self.import_files[self.inputFile] = stat_info.st_mtime print "GuiAutoImport.import_dir done" @@ -121,7 +121,7 @@ class GuiAutoImport (threading.Thread): def __init__(self, settings, debug=True): """Constructor for GuiAutoImport""" self.settings=settings - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) self.server=settings['db-host'] diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 4bd221cb..ed0a19b0 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -32,7 +32,7 @@ class GuiBulkImport (threading.Thread): print "BulkImport is not recursive - please select the final directory in which the history files are" else: self.inputFile=self.path+os.sep+file - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() print "GuiBulkImport.import_dir done" def load_clicked(self, widget, data=None): @@ -64,12 +64,10 @@ class GuiBulkImport (threading.Thread): else: self.failOnError=True - self.server, self.database, self.user, self.password=self.db.get_db_info() - if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.import_file_dict(self, self.settings) + self.importer.import_file_dict() def get_vbox(self): """returns the vbox of this thread""" @@ -83,7 +81,7 @@ class GuiBulkImport (threading.Thread): def __init__(self, db, settings): self.db=db self.settings=settings - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self,self.settings) self.vbox=gtk.VBox(False,1) self.vbox.show() diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index 577be619..d7f3f8f2 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -255,9 +255,9 @@ class GuiTableViewer (threading.Thread): self.failOnError=False self.minPrint=0 self.handCount=0 - self.importer = fpdb_import.Importer() + self.importer = fpdb_import.Importer(self, self.settings) - self.last_read_hand_id=self.importer.import_file_dict(self, self.settings) + self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked def all_clicked(self, widget, data): diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 786aea59..46b97e7d 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -40,29 +40,32 @@ from time import time class Importer: - def __init__(self): + def __init__(self, options, settings): """Constructor""" - self.settings={'imp-callFpdbHud':False} + self.settings=settings + self.options=options self.db = None self.cursor = None - self.options = None self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file + if not self.settings.has_key('imp-callFpdbHud'): + self.settings['imp-callFpdbHud'] = False + self.dbConnect() - def dbConnect(self, options, settings): + def dbConnect(self): #connect to DB - if settings['db-backend'] == 2: + if self.settings['db-backend'] == 2: if not mysqlLibFound: raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file") - self.db = MySQLdb.connect(host = options.server, user = options.user, - passwd = options.password, db = options.database) - elif settings['db-backend'] == 3: + self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'], + self.settings['db-password'], self.settings['db-databaseName']) + elif self.settings['db-backend'] == 3: if not pgsqlLibFound: raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file") - self.db = psycopg2.connect(host = options.server, user = options.user, - password = options.password, database = options.database) - elif settings['db-backend'] == 4: + self.db = psycopg2.connect(self.settings['db-host'], self.settings['db-user'], + self.settings['db-password'], self.settings['db-databaseName']) + elif self.settings['db-backend'] == 4: pass else: pass @@ -71,32 +74,30 @@ class Importer: def setCallHud(self, value): self.callHud = value - def import_file_dict(self, options, settings): - self.options=options + def addImportFile(self, filename): + self.options.inputFile = filename + + def import_file_dict(self): starttime = time() last_read_hand=0 loc = 0 - if (options.inputFile=="stdin"): + if (self.options.inputFile=="stdin"): inputFile=sys.stdin else: - inputFile=open(options.inputFile, "rU") - try: loc = self.pos_in_file[options.inputFile] + inputFile=open(self.options.inputFile, "rU") + try: loc = self.pos_in_file[self.options.inputFile] except: pass - self.dbConnect(options,settings) - # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[options.inputFile] = inputFile.tell() + self.pos_in_file[self.options.inputFile] = inputFile.tell() inputFile.close() firstline = self.lines[0] if firstline.find("Tournament Summary")!=-1: print "TODO: implement importing tournament summaries" - self.cursor.close() - self.db.close() return 0 site=fpdb_simple.recogniseSite(firstline) @@ -151,41 +152,36 @@ class Importer: stored+=1 self.db.commit() # if settings['imp-callFpdbHud'] and self.callHud and os.sep=='/': - if settings['imp-callFpdbHud'] and self.callHud: + if self.settings['imp-callFpdbHud'] and self.callHud: #print "call to HUD here. handsId:",handsId #pipe the Hands.id out to the HUD -# options.pipe_to_hud.write("%s" % (handsId) + os.linesep) - options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + self.options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) except fpdb_simple.DuplicateError: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) - if (options.failOnError): + if (self.options.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. - self.cursor.close() - self.db.close() raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (options.failOnError): + if (self.options.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. - self.cursor.close() - self.db.close() raise - if (options.minPrint!=0): - if ((stored+duplicates+partial+errors)%options.minPrint==0): + if (self.options.minPrint!=0): + if ((stored+duplicates+partial+errors)%sielf.options.minPrint==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (options.handCount!=0): - if ((stored+duplicates+partial+errors)>=options.handCount): - if (not options.quiet): + if (self.options.handCount!=0): + if ((stored+duplicates+partial+errors)>=self.options.handCount): + if (not self.options.quiet): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) @@ -203,8 +199,6 @@ class Importer: handsId=0 #todo: this will cause return of an unstored hand number if the last hand was error or partial self.db.commit() - self.cursor.close() - self.db.close() return handsId #end def import_file_dict From ae9b70ea5d4caa476cf77f02a2c7319a435d89c2 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:19:57 +0800 Subject: [PATCH 05/15] Change variable name to indicate what it actually is. --- pyfpdb/fpdb_import.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 46b97e7d..bb4d21d3 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -40,10 +40,10 @@ from time import time class Importer: - def __init__(self, options, settings): + def __init__(self, caller, settings): """Constructor""" self.settings=settings - self.options=options + self.caller=caller self.db = None self.cursor = None self.callHud = False @@ -75,23 +75,23 @@ class Importer: self.callHud = value def addImportFile(self, filename): - self.options.inputFile = filename + self.caller.inputFile = filename def import_file_dict(self): starttime = time() last_read_hand=0 loc = 0 - if (self.options.inputFile=="stdin"): + if (self.caller.inputFile=="stdin"): inputFile=sys.stdin else: - inputFile=open(self.options.inputFile, "rU") - try: loc = self.pos_in_file[self.options.inputFile] + inputFile=open(self.caller.inputFile, "rU") + try: loc = self.pos_in_file[self.caller.inputFile] except: pass # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[self.options.inputFile] = inputFile.tell() + self.pos_in_file[self.caller.inputFile] = inputFile.tell() inputFile.close() firstline = self.lines[0] @@ -155,33 +155,33 @@ class Importer: if self.settings['imp-callFpdbHud'] and self.callHud: #print "call to HUD here. handsId:",handsId #pipe the Hands.id out to the HUD - self.options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) + self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep) except fpdb_simple.DuplicateError: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) - if (self.options.failOnError): + if (self.caller.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.options.inputFile, hand[0]) + self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (self.options.failOnError): + if (self.caller.failOnError): self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. raise - if (self.options.minPrint!=0): - if ((stored+duplicates+partial+errors)%sielf.options.minPrint==0): + if (self.caller.minPrint!=0): + if ((stored+duplicates+partial+errors)%self.caller.minPrint==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (self.options.handCount!=0): - if ((stored+duplicates+partial+errors)>=self.options.handCount): - if (not self.options.quiet): + if (self.caller.handCount!=0): + if ((stored+duplicates+partial+errors)>=self.caller.handCount): + if (not self.caller.quiet): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) @@ -204,7 +204,7 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:", self.options.inputFile + print "Filename:", self.caller.inputFile print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" print self.hand[0] From 1e8333ec5d65e0fd41064a77fd19cf9602a67f2e Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 01:42:08 +0800 Subject: [PATCH 06/15] Move minPrint variable from callling class to setting in Importer class --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/GuiBulkImport.py | 4 ++-- pyfpdb/GuiTableViewer.py | 2 +- pyfpdb/fpdb_import.py | 14 ++++++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 79b25fee..97653c77 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -123,6 +123,7 @@ class GuiAutoImport (threading.Thread): self.settings=settings self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) + self.importer.setMinPrint(30) self.server=settings['db-host'] self.user=settings['db-user'] @@ -130,7 +131,6 @@ class GuiAutoImport (threading.Thread): self.database=settings['db-databaseName'] self.quiet=False self.failOnError=False - self.minPrint=30 self.handCount=0 self.mainVBox=gtk.VBox(False,1) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index ed0a19b0..061e3848 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -48,9 +48,9 @@ class GuiBulkImport (threading.Thread): self.minPrint=self.min_print_tbuffer.get_text(self.min_print_tbuffer.get_start_iter(), self.min_print_tbuffer.get_end_iter()) if (self.minPrint=="never" or self.minPrint=="Never"): - self.minPrint=0 + self.importer.setMinPrint(0) else: - self.minPrint=int(self.minPrint) + self.importer.setMinPrint=int(self.minPrint) self.quiet=self.info_tbuffer.get_text(self.info_tbuffer.get_start_iter(), self.info_tbuffer.get_end_iter()) if (self.quiet=="yes"): diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index d7f3f8f2..2cccdbe5 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -253,9 +253,9 @@ class GuiTableViewer (threading.Thread): self.quiet=False self.failOnError=False - self.minPrint=0 self.handCount=0 self.importer = fpdb_import.Importer(self, self.settings) + self.importer.setMinPrint(0) self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index bb4d21d3..5a2aaab5 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -49,8 +49,11 @@ class Importer: self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file + #Set defaults if not self.settings.has_key('imp-callFpdbHud'): self.settings['imp-callFpdbHud'] = False + if not self.settings.has_key('minPrint'): + self.settings['minPrint'] = 30 self.dbConnect() def dbConnect(self): @@ -77,6 +80,9 @@ class Importer: def addImportFile(self, filename): self.caller.inputFile = filename + def setMinPrint(self, value): + self.settings['minPrint'] = int(value) + def import_file_dict(self): starttime = time() last_read_hand=0 @@ -163,7 +169,7 @@ class Importer: self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) if (self.caller.failOnError): - self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. + self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: errors+=1 @@ -173,10 +179,10 @@ class Importer: self.db.rollback() if (self.caller.failOnError): - self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends. + self.db.commit() #dont remove this, in case hand processing was cancelled. raise - if (self.caller.minPrint!=0): - if ((stored+duplicates+partial+errors)%self.caller.minPrint==0): + if (self.settings['minPrint']!=0): + if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors if (self.caller.handCount!=0): From 638a6d6dabf2f122eb3c27c0c04f4be37ac2aeb6 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 02:14:06 +0800 Subject: [PATCH 07/15] Move more "options" from calling class into settings has of fpdb_import and fix all callers --- pyfpdb/GuiAutoImport.py | 6 +++--- pyfpdb/GuiBulkImport.py | 12 ++++++------ pyfpdb/GuiTableViewer.py | 6 +++--- pyfpdb/fpdb_import.py | 19 ++++++++++++++----- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 97653c77..2c4246dc 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -124,14 +124,14 @@ class GuiAutoImport (threading.Thread): self.importer = fpdb_import.Importer(self,self.settings) self.importer.setCallHud(True) self.importer.setMinPrint(30) + self.importer.setQuiet(False) + self.importer.setFailOnError(False) + self.importer.setHandCount(0) self.server=settings['db-host'] self.user=settings['db-user'] self.password=settings['db-password'] self.database=settings['db-databaseName'] - self.quiet=False - self.failOnError=False - self.handCount=0 self.mainVBox=gtk.VBox(False,1) self.mainVBox.show() diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 061e3848..4f3d7ef6 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -40,9 +40,9 @@ class GuiBulkImport (threading.Thread): self.handCount=self.hand_count_tbuffer.get_text(self.hand_count_tbuffer.get_start_iter(), self.hand_count_tbuffer.get_end_iter()) if (self.handCount=="unlimited" or self.handCount=="Unlimited"): - self.handCount=0 + self.importer.setHandCount(0) else: - self.handCount=int(self.handCount) + self.importer.setHandCount(int(self.handCount)) self.errorFile="failed.txt" @@ -54,15 +54,15 @@ class GuiBulkImport (threading.Thread): self.quiet=self.info_tbuffer.get_text(self.info_tbuffer.get_start_iter(), self.info_tbuffer.get_end_iter()) if (self.quiet=="yes"): - self.quiet=False + self.importer.setQuiet(False) else: - self.quiet=True + self.importer.setQuiet(True) self.failOnError=self.fail_error_tbuffer.get_text(self.fail_error_tbuffer.get_start_iter(), self.fail_error_tbuffer.get_end_iter()) if (self.failOnError=="no"): - self.failOnError=False + self.importer.setFailOnError(False) else: - self.failOnError=True + self.importer.setFailOnError(True) if os.path.isdir(self.inputFile): self.import_dir() diff --git a/pyfpdb/GuiTableViewer.py b/pyfpdb/GuiTableViewer.py index 2cccdbe5..f2bfecd9 100644 --- a/pyfpdb/GuiTableViewer.py +++ b/pyfpdb/GuiTableViewer.py @@ -251,11 +251,11 @@ class GuiTableViewer (threading.Thread): self.user=self.db.user self.password=self.db.password - self.quiet=False - self.failOnError=False - self.handCount=0 self.importer = fpdb_import.Importer(self, self.settings) self.importer.setMinPrint(0) + self.importer.setQuiet(False) + self.importer.setFailOnError(False) + self.importer.setHandCount(0) self.last_read_hand_id=self.importer.import_file_dict() #end def table_viewer.import_clicked diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 5a2aaab5..9c2955eb 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -83,6 +83,15 @@ class Importer: def setMinPrint(self, value): self.settings['minPrint'] = int(value) + def setHandCount(self, value): + self.settings['handCount'] = int(value) + + def setQuiet(self, value): + self.settings['quiet'] = value + + def setFailOnError(self, value): + self.settings['failOnError'] = value + def import_file_dict(self): starttime = time() last_read_hand=0 @@ -168,7 +177,7 @@ class Importer: errors+=1 self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) - if (self.caller.failOnError): + if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: @@ -178,16 +187,16 @@ class Importer: #fe.printStackTrace() #todo: get stacktrace self.db.rollback() - if (self.caller.failOnError): + if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise if (self.settings['minPrint']!=0): if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0): print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors - if (self.caller.handCount!=0): - if ((stored+duplicates+partial+errors)>=self.caller.handCount): - if (not self.caller.quiet): + if (self.settings['handCount']!=0): + if ((stored+duplicates+partial+errors)>=self.settings['handCount']): + if (not self.settings['quiet']): print "quitting due to reaching the amount of hands to be imported" print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime) sys.exit(0) From 6c4e2f3eb9287d5ec2606d293805d5cc6dcd5375 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 15:49:09 +0800 Subject: [PATCH 08/15] Changes import_file_dict interface to Importer internal. Users of Importer should now addImportFile and addImportDirectory, set paramaters, then run either runImport - all files or runUpdated - modified files. Comments out postgres regression test until fixed Adds initial code for importing regression test files. --- pyfpdb/GuiAutoImport.py | 25 +++------------- pyfpdb/GuiBulkImport.py | 12 ++++---- pyfpdb/RegressionTest.py | 61 ++++++++++++++++++++++---------------- pyfpdb/fpdb_import.py | 64 +++++++++++++++++++++++++++++++++------- 4 files changed, 98 insertions(+), 64 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 2c4246dc..9cc10906 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -50,16 +50,7 @@ class GuiAutoImport (threading.Thread): def do_import(self): """Callback for timer to do an import iteration.""" - for file in os.listdir(self.path): - if os.path.isdir(file): - print "AutoImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile = os.path.join(self.path, file) - stat_info = os.stat(self.inputFile) - if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]: - self.importer.import_file_dict() - self.import_files[self.inputFile] = stat_info.st_mtime - + self.importer.runUpdated() print "GuiAutoImport.import_dir done" return True @@ -96,17 +87,9 @@ class GuiAutoImport (threading.Thread): # self.pipe_to_hud = os.popen(command, 'w') self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter()) -# Iniitally populate the self.import_files dict, which keeps mtimes for the files watched - - self.import_files = {} - for file in os.listdir(self.path): - if os.path.isdir(file): - pass # skip subdirs for now - else: - inputFile = os.path.join(self.path, file) - stat_info = os.stat(inputFile) - self.import_files[inputFile] = stat_info.st_mtime - +# Add directory to importer object and set the initial mtime reference. + self.importer.addImportDirectory(self.path) + self.importer.setWatchTime() self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 4f3d7ef6..c2347528 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -27,12 +27,8 @@ class GuiBulkImport (threading.Thread): def import_dir(self): """imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it""" self.path=self.inputFile - for file in os.listdir(self.path): - if os.path.isdir(file): - print "BulkImport is not recursive - please select the final directory in which the history files are" - else: - self.inputFile=self.path+os.sep+file - self.importer.import_file_dict() + self.importer.addImportDirectory(self.path) + self.importer.runImport() print "GuiBulkImport.import_dir done" def load_clicked(self, widget, data=None): @@ -67,7 +63,9 @@ class GuiBulkImport (threading.Thread): if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.import_file_dict() + self.importer.addImportFile() + self.importer.runImport() + self.importer.clearFileList() def get_vbox(self): """returns the vbox of this thread""" diff --git a/pyfpdb/RegressionTest.py b/pyfpdb/RegressionTest.py index 8a8facc5..b4dd1b7d 100644 --- a/pyfpdb/RegressionTest.py +++ b/pyfpdb/RegressionTest.py @@ -25,51 +25,62 @@ import os import sys import fpdb_db +import fpdb_import import FpdbSQLQueries import unittest class TestSequenceFunctions(unittest.TestCase): - def setUp(self): - """Configure MySQL settings/database and establish connection""" - self.mysql_settings={ 'db-host':"localhost", 'db-backend':2, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.mysql_db = fpdb_db.fpdb_db() - self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], - self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], - self.mysql_settings['db-password']) - self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + def setUp(self): + """Configure MySQL settings/database and establish connection""" + self.mysql_settings={ 'db-host':"localhost", + 'db-backend':2, + 'db-databaseName':"fpdbtest", + 'db-user':"fpdb", + 'db-password':"fpdb"} + self.mysql_db = fpdb_db.fpdb_db() + self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'], + self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'], + self.mysql_settings['db-password']) + self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB') + self.mysqlimporter = fpdb_import.Importer(self, self.mysql_settings) - """Configure Postgres settings/database and establish connection""" - self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} - self.pg_db = fpdb_db.fpdb_db() - self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], - self.pg_settings['db-databaseName'], self.pg_settings['db-user'], - self.pg_settings['db-password']) - self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') +# """Configure Postgres settings/database and establish connection""" +# self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"} +# self.pg_db = fpdb_db.fpdb_db() +# self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'], +# self.pg_settings['db-databaseName'], self.pg_settings['db-user'], +# self.pg_settings['db-password']) +# self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL') def testDatabaseConnection(self): """Test all supported DBs""" self.result = self.mysql_db.cursor.execute(self.mysqldict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - - print self.pgdict.query['list_tables'] - - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testMySQLRecreateTables(self): """Test droping then recreating fpdb table schema""" self.mysql_db.recreate_tables() self.result = self.mysql_db.cursor.execute("SHOW TABLES") self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) - def testPostgresSQLRecreateTables(self): - """Test droping then recreating fpdb table schema""" - self.pg_db.recreate_tables() - self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) - self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) + def testImportHandHistoryFiles(self): + """Test import of single HH file""" + self.mysqlimporter.addImportFile("regression-test-files/hand-histories/ps-lhe-ring-3hands.txt") + self.mysqlimporter.runImport() + self.mysqlimporter.addImportDirectory("regression-test-files/hand-histories") + self.mysqlimporter.runImport() + +# def testPostgresSQLRecreateTables(self): +# """Test droping then recreating fpdb table schema""" +# self.pg_db.recreate_tables() +# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables']) +# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result)) if __name__ == '__main__': unittest.main() diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 9c2955eb..c844c00d 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -46,6 +46,9 @@ class Importer: self.caller=caller self.db = None self.cursor = None + self.filelist = [] + self.queued = [] + self.updated = 0 #Time last import was run, used as mtime reference self.callHud = False self.lines = None self.pos_in_file = {} # dict to remember how far we have read in the file @@ -74,12 +77,10 @@ class Importer: pass self.cursor = self.db.cursor() + #Set functions def setCallHud(self, value): self.callHud = value - def addImportFile(self, filename): - self.caller.inputFile = filename - def setMinPrint(self, value): self.settings['minPrint'] = int(value) @@ -92,21 +93,62 @@ class Importer: def setFailOnError(self, value): self.settings['failOnError'] = value - def import_file_dict(self): + def setWatchTime(self): + self.updated = time() + + def clearFileList(self): + self.filelist = [] + + #Add an individual file to filelist + def addImportFile(self, filename): + #todo: test it is a valid file + self.filelist = self.filelist + [filename] + print "Filelist in addImportFile: ", self.filelist + #Remove duplicates + set(filelist) + + #Add a directory of files to filelist + def addImportDirectory(self,dir): + #todo: test it is a valid directory + for file in os.listdir(dir): + if os.path.isdir(file): + print "BulkImport is not recursive - please select the final directory in which the history files are" + else: + blah = [dir+os.sep+file] + self.filelist = self.filelist + [dir+os.sep+file] + #Remove duplicates + set(self.filelist) + + #Run full import on filelist + def runImport(self): + for file in self.filelist: + print "Importing file: ", file + self.import_file_dict(file) + + #Run import on updated files, then store latest update time. + def runUpdated(self): + for file in self.filelist: + stat_info = os.stat(file) + if stat_info.st_mtime > self.updated: + self.import_file_dict(file) + self.updated = time() + + # This is now an internal function that should not be called directly. + def import_file_dict(self, file): starttime = time() last_read_hand=0 loc = 0 - if (self.caller.inputFile=="stdin"): + if (file=="stdin"): inputFile=sys.stdin else: - inputFile=open(self.caller.inputFile, "rU") - try: loc = self.pos_in_file[self.caller.inputFile] + inputFile=open(file, "rU") + try: loc = self.pos_in_file[file] except: pass # Read input file into class and close file inputFile.seek(loc) self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines()) - self.pos_in_file[self.caller.inputFile] = inputFile.tell() + self.pos_in_file[file] = inputFile.tell() inputFile.close() firstline = self.lines[0] @@ -175,14 +217,14 @@ class Importer: duplicates+=1 except (ValueError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) if (self.settings['failOnError']): self.db.commit() #dont remove this, in case hand processing was cancelled. raise except (fpdb_simple.FpdbError), fe: errors+=1 - self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0]) + self.printEmailErrorMessage(errors, file, hand[0]) #fe.printStackTrace() #todo: get stacktrace self.db.rollback() @@ -219,7 +261,7 @@ class Importer: def printEmailErrorMessage(self, errors, filename, line): print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it." - print "Filename:", self.caller.inputFile + print "Filename:", filename print "Here is the first line so you can identify it. Please mention that the error was a ValueError:" print self.hand[0] From c2c37dc5ee095f8bf7bc4b7143b657b671d78ba4 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 12 Oct 2008 18:21:42 +0800 Subject: [PATCH 09/15] Fix mtime time detection --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/fpdb_import.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 9cc10906..23493664 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -89,7 +89,6 @@ class GuiAutoImport (threading.Thread): # Add directory to importer object and set the initial mtime reference. self.importer.addImportDirectory(self.path) - self.importer.setWatchTime() self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) @@ -110,6 +109,7 @@ class GuiAutoImport (threading.Thread): self.importer.setQuiet(False) self.importer.setFailOnError(False) self.importer.setHandCount(0) + self.importer.setWatchTime() self.server=settings['db-host'] self.user=settings['db-user'] diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c844c00d..458b34d5 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -130,6 +130,7 @@ class Importer: for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: + # print "File: " + str(file) + " mtime: " + str(stat_info.st_mtime) + " update: " + str(self.updated) self.import_file_dict(file) self.updated = time() From 8ca5a5c9657c5f0ba01fb7ae0d5775679a2a54a3 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 01:26:04 +0800 Subject: [PATCH 10/15] Should fix detection of new hh files. --- pyfpdb/fpdb_import.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 458b34d5..73e141e7 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -47,7 +47,8 @@ class Importer: self.db = None self.cursor = None self.filelist = [] - self.queued = [] + self.dirlist = [] + self.monitor = False self.updated = 0 #Time last import was run, used as mtime reference self.callHud = False self.lines = None @@ -108,13 +109,16 @@ class Importer: set(filelist) #Add a directory of files to filelist - def addImportDirectory(self,dir): + def addImportDirectory(self,dir,monitor = False): #todo: test it is a valid directory + if monitor == True: + self.monitor = True + self.dirlist = self.dirlist + [dir] + for file in os.listdir(dir): if os.path.isdir(file): print "BulkImport is not recursive - please select the final directory in which the history files are" else: - blah = [dir+os.sep+file] self.filelist = self.filelist + [dir+os.sep+file] #Remove duplicates set(self.filelist) @@ -127,6 +131,13 @@ class Importer: #Run import on updated files, then store latest update time. def runUpdated(self): + #Check for new files in directory + #todo: make efficient - always checks for new file, should be able to use mtime of directory + # ^^ May not work on windows + for dir in self.dirlist: + for file in os.listdir(dir): + self.filelist = self.filelist + [dir+os.sep+file] + for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: From 9a60cf84c8cd90894dd0af14a09d88a57f6f3e6e Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 02:06:28 +0800 Subject: [PATCH 11/15] Real fix this time, modify GuiAutoImport to actually tell importer to monitor the directory --- pyfpdb/GuiAutoImport.py | 2 +- pyfpdb/fpdb_import.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index 23493664..6953d1f9 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -88,7 +88,7 @@ class GuiAutoImport (threading.Thread): self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter()) # Add directory to importer object and set the initial mtime reference. - self.importer.addImportDirectory(self.path) + self.importer.addImportDirectory(self.path, True) self.do_import() interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter())) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 73e141e7..fdf3ab3c 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -119,7 +119,7 @@ class Importer: if os.path.isdir(file): print "BulkImport is not recursive - please select the final directory in which the history files are" else: - self.filelist = self.filelist + [dir+os.sep+file] + self.filelist = self.filelist + [os.path.join(dir, file)] #Remove duplicates set(self.filelist) @@ -137,6 +137,7 @@ class Importer: for dir in self.dirlist: for file in os.listdir(dir): self.filelist = self.filelist + [dir+os.sep+file] + set(filelist) for file in self.filelist: stat_info = os.stat(file) From 3e9ef61fc893a01fb3441d0f173fff75bf089c31 Mon Sep 17 00:00:00 2001 From: Worros Date: Mon, 13 Oct 2008 23:34:18 +0800 Subject: [PATCH 12/15] Fix bug: incorrect use of set function to remove duplicates Now detects new tables without issues --- pyfpdb/fpdb_import.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fdf3ab3c..b652f204 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -104,9 +104,8 @@ class Importer: def addImportFile(self, filename): #todo: test it is a valid file self.filelist = self.filelist + [filename] - print "Filelist in addImportFile: ", self.filelist #Remove duplicates - set(filelist) + self.filelist = list(set(self.filelist)) #Add a directory of files to filelist def addImportDirectory(self,dir,monitor = False): @@ -121,12 +120,11 @@ class Importer: else: self.filelist = self.filelist + [os.path.join(dir, file)] #Remove duplicates - set(self.filelist) + self.filelist = list(set(self.filelist)) #Run full import on filelist def runImport(self): for file in self.filelist: - print "Importing file: ", file self.import_file_dict(file) #Run import on updated files, then store latest update time. @@ -137,12 +135,12 @@ class Importer: for dir in self.dirlist: for file in os.listdir(dir): self.filelist = self.filelist + [dir+os.sep+file] - set(filelist) + + self.filelist = list(set(self.filelist)) for file in self.filelist: stat_info = os.stat(file) if stat_info.st_mtime > self.updated: - # print "File: " + str(file) + " mtime: " + str(stat_info.st_mtime) + " update: " + str(self.updated) self.import_file_dict(file) self.updated = time() From 155b2e93b0f1a457b6db7c4726d41273f0250720 Mon Sep 17 00:00:00 2001 From: steffen123 Date: Mon, 13 Oct 2008 23:09:27 +0100 Subject: [PATCH 13/15] p134 - updated CliFpdb for new importer --- pyfpdb/CliFpdb.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pyfpdb/CliFpdb.py b/pyfpdb/CliFpdb.py index e79a9d49..75dd1251 100755 --- a/pyfpdb/CliFpdb.py +++ b/pyfpdb/CliFpdb.py @@ -54,5 +54,12 @@ if __name__ == "__main__": (options, sys.argv) = parser.parse_args() settings={'imp-callFpdbHud':False, 'db-backend':2} - importer = fpdb_import.Importer(options,settings) - importer.import_file_dict() + settings['db-host']=options.server + settings['db-user']=options.user + settings['db-password']=options.password + settings['db-databaseName']=options.database + settings['handCount']=options.handCount + + importer = fpdb_import.Importer(options, settings) + importer.addImportFile(options.inputFile) + importer.runImport() From c7ba32c755e23733ae479333d33125da58afb556 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 14 Oct 2008 22:35:05 +0800 Subject: [PATCH 14/15] Fix graph message to include site and player name --- pyfpdb/GuiGraphViewer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyfpdb/GuiGraphViewer.py b/pyfpdb/GuiGraphViewer.py index 5b3fa081..58c0cc31 100644 --- a/pyfpdb/GuiGraphViewer.py +++ b/pyfpdb/GuiGraphViewer.py @@ -51,8 +51,10 @@ class GuiGraphViewer (threading.Thread): if site=="PS": site=2 + sitename="PokerStars: " elif site=="FTP": site=1 + sitename="Full Tilt: " else: print "invalid text in site selection in graph, defaulting to PS" site=2 @@ -69,7 +71,9 @@ class GuiGraphViewer (threading.Thread): self.ax.set_xlabel("Hands", fontsize = 12) self.ax.set_ylabel("$", fontsize = 12) self.ax.grid(color='g', linestyle=':', linewidth=0.2) - self.ax.annotate ("All Hands, Site %s", (61,25), xytext =(0.1, 0.9) , textcoords ="axes fraction" ,) + text = "All Hands, " + sitename + str(name) + + self.ax.annotate (text, (61,25), xytext =(0.1, 0.9) , textcoords ="axes fraction" ,) #Get graph data from DB line = self.getRingProfitGraph(name, site) From 725f0589f807d1d174eeaec2b430e1d19e797f40 Mon Sep 17 00:00:00 2001 From: Worros Date: Wed, 15 Oct 2008 00:15:01 +0800 Subject: [PATCH 15/15] Fix importing single files again - missing arguement --- pyfpdb/GuiBulkImport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index c2347528..978f328d 100644 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -63,7 +63,7 @@ class GuiBulkImport (threading.Thread): if os.path.isdir(self.inputFile): self.import_dir() else: - self.importer.addImportFile() + self.importer.addImportFile(self.inputFile) self.importer.runImport() self.importer.clearFileList()