From e9fd3541150aa50625be830037ee7150c021437d Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Wed, 18 Mar 2009 16:52:18 +0000 Subject: [PATCH 1/6] handinfo --- pyfpdb/FulltiltToFpdb.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index fb2bf6ea..0c21762f 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -113,10 +113,30 @@ follow : whether to tail -f the input""" def readHandInfo(self, hand): m = self.re_HandInfo.search(hand.handText,re.DOTALL) - #print m.groups() + + if(m == None): + logging.info("Didn't match re_HandInfo") + logging.info(hand.handText) + return None + hand.handid = m.group('HID') hand.tablename = m.group('TABLE') hand.starttime = time.strptime(m.group('DATETIME'), "%H:%M:%S ET - %Y/%m/%d") + + # attributes + # (deep 6) + # (6 max) + # + # be goodd idea to log those what we don't know idea + hand.maxseats = 8 # assume 8-max until we see otherwise + # stud tables are usually 8-max? + # + atts = m.group('ATTRIBUTES') + if atts: + # TODO: parse these + pass + + # These work, but the info is already in the Hand class - should be used for tourneys though. # m.group('SB') # m.group('BB') From cbba16feaf44a911f9d85d766a21ecd2f5a1b30c Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Thu, 19 Mar 2009 13:41:05 +0000 Subject: [PATCH 2/6] comment --- pyfpdb/fpdb_save_to_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index ba734e80..1089a5de 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -32,6 +32,7 @@ saveActions=True # set this to False to avoid storing action data # Pros: speeds up imports # Cons: no action data is saved, so you need to keep the hand histories # variance not available on stats page + # no graphs #stores a stud/razz hand into the database def ring_stud(backend, db, cursor, base, category, site_hand_no, gametype_id, hand_start_time From 451a9b3ab12e8c3ec0b0a88851ef83df569f5e01 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 21 Mar 2009 14:27:49 +0000 Subject: [PATCH 3/6] GuiBulkImport: import directories from cmdline. Basically now does what CliFpdb should do, but reads most settings from xml config. --- pyfpdb/GuiBulkImport.py | 41 ++++++++++++++++++++++++----------------- pyfpdb/fpdb_import.py | 10 +++++++++- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 509c9f61..2d086428 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -60,8 +60,8 @@ class GuiBulkImport(): self.importer.setFailOnError(self.chk_fail.get_active()) self.importer.setThreads(int(self.spin_threads.get_text())) self.importer.setHandsInDB(self.n_hands_in_db) - cb_model = self.cb.get_model() - cb_index = self.cb.get_active() + cb_model = self.cb_dropindexes.get_model() + cb_index = self.cb_dropindexes.get_active() if cb_index: self.importer.setDropIndexes(cb_model[cb_index][0]) else: @@ -159,13 +159,13 @@ class GuiBulkImport(): self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT) # ComboBox - drop indexes - self.cb = gtk.combo_box_new_text() - self.cb.append_text('auto') - self.cb.append_text("don't drop") - self.cb.append_text('drop') - self.cb.set_active(0) - self.table.attach(self.cb, 4, 5, 1, 2, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK) - self.cb.show() + self.cb_dropindexes = gtk.combo_box_new_text() + self.cb_dropindexes.append_text('auto') + self.cb_dropindexes.append_text("don't drop") + self.cb_dropindexes.append_text('drop') + self.cb_dropindexes.set_active(0) + self.table.attach(self.cb_dropindexes, 4, 5, 1, 2, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK) + self.cb_dropindexes.show() # label - filter self.lab_filter = gtk.Label("Site filter:") @@ -195,13 +195,13 @@ class GuiBulkImport(): # see how many hands are in the db and adjust accordingly tcursor = db.db.cursor() - tcursor.execute("Select max(id) from Hands;") + tcursor.execute("Select count(1) from Hands;") row = tcursor.fetchone() tcursor.close() self.n_hands_in_db = row[0] if self.n_hands_in_db == 0: - self.cb.set_active(2) - self.cb.set_sensitive(False) + self.cb_dropindexes.set_active(2) + self.cb_dropindexes.set_sensitive(False) self.lab_drop.set_sensitive(False) if __name__ == '__main__': @@ -211,9 +211,16 @@ if __name__ == '__main__': gtk.main_quit() parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", help="Input file in quiet mode", metavar="FILE") - parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui") - parser.add_option("-x", "--convert", dest="filtername", help="Conversion filter", default="passthrough") + parser.add_option("-f", "--file", dest="filename", metavar="FILE", + help="Input file in quiet mode") + parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, + help="don't start gui") + parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", + help="Conversion filter (*passthrough, FullTiltToFpdb, PokerStarsToFpdb, EverleafToFpdb)") + parser.add_option("-x", "--failOnError", action="store_true", default=False, + help="If this option is passed it quits when it encounters any error") + #parser.add_option("-m", "--minPrint", "--status", default="0", type="int", + #help="How often to print a one-line status report (0 (default) means never)") (options, sys.argv) = parser.parse_args() config = Configuration.Config() @@ -239,8 +246,8 @@ if __name__ == '__main__': #Do something useful importer = fpdb_import.Importer(False,settings, config) importer.setDropIndexes("auto") - importer.setFailOnError(True) - importer.addImportFile(options.filename, filter=options.filtername) + importer.setFailOnError(options.failOnError) + importer.addBulkImportImportFileOrDir(options.filename, filter=options.filtername) importer.setCallHud(False) importer.runImport() importer.clearFileList() diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 12575b4f..fb602e63 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -172,7 +172,15 @@ class Importer: def calculate_auto(self): """An heuristic to determine a reasonable value of drop/don't drop""" - if len(self.filelist) == 1: return "don't drop" + if len(self.filelist) == 1: return "don't drop" + if 'handsInDB' not in self.settings: + try: + tmpcursor = self.fdb.db.cursor() + tmpcursor.execute("Select count(1) from Hands;") + self.settings['handsInDB'] = tmpcursor.fetchone()[0] + tmpcursor.close() + except: + pass # if this fails we're probably doomed anyway if self.settings['handsInDB'] < 5000: return "drop" if len(self.filelist) < 50: return "don't drop" if self.settings['handsInDB'] > 50000: return "don't drop" From 239330ae7bb20b05981090d7f541931f9a1d40ca Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 21 Mar 2009 15:34:23 +0000 Subject: [PATCH 4/6] filterCrap after checking if hand is in db refactor GuiBulkImport : add main() so that it can be called from interactive shell (for profiling purposes). --- pyfpdb/GuiAutoImport.py | 6 ++++-- pyfpdb/GuiBulkImport.py | 33 ++++++++++++++++++++++++--------- pyfpdb/fpdb_import.py | 3 +-- pyfpdb/fpdb_parse_logic.py | 2 ++ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index ee633d4f..7212862b 100755 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -45,7 +45,7 @@ class GuiAutoImport (threading.Thread): self.importer = fpdb_import.Importer(self,self.settings, self.config) self.importer.setCallHud(True) - self.importer.setMinPrint(30) + self.importer.setMinPrint(settings['minPrint']) self.importer.setQuiet(False) self.importer.setFailOnError(False) self.importer.setHandCount(0) @@ -227,13 +227,15 @@ if __name__== "__main__": parser = OptionParser() parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui") - + parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int", + help="How often to print a one-line status report (0 (default) means never)") (options, sys.argv) = parser.parse_args() config = Configuration.Config() # db = fpdb_db.fpdb_db() settings = {} + settings['minPrint'] = options.minPrint if os.name == 'nt': settings['os'] = 'windows' else: settings['os'] = 'linuxmac' diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 2d086428..6745e9fc 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -204,29 +204,36 @@ class GuiBulkImport(): self.cb_dropindexes.set_sensitive(False) self.lab_drop.set_sensitive(False) -if __name__ == '__main__': - +def main(argv=None): + """main can also be called in the python interpreter, by supplying the command line as the argument. +>>>import GuiBulkImport +>>>GuiBulkImport.main("-f ~/data/hands")""" + if argv is None: + argv = sys.argv[1:] + else: + argv = argv.split(" ") def destroy(*args): # call back for terminating the main eventloop gtk.main_quit() parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", metavar="FILE", + parser.add_option("-f", "--file", dest="filename", metavar="FILE", default=None, help="Input file in quiet mode") parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, - help="don't start gui") + help="don't start gui; deprecated (just give a filename with -f).") parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER", help="Conversion filter (*passthrough, FullTiltToFpdb, PokerStarsToFpdb, EverleafToFpdb)") parser.add_option("-x", "--failOnError", action="store_true", default=False, help="If this option is passed it quits when it encounters any error") - #parser.add_option("-m", "--minPrint", "--status", default="0", type="int", - #help="How often to print a one-line status report (0 (default) means never)") - (options, sys.argv) = parser.parse_args() + parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int", + help="How often to print a one-line status report (0 (default) means never)") + (options, sys.argv) = parser.parse_args(args = argv) config = Configuration.Config() db = fpdb_db.fpdb_db() settings = {} + settings['minPrint'] = options.minPrint if os.name == 'nt': settings['os'] = 'windows' else: settings['os'] = 'linuxmac' @@ -235,7 +242,10 @@ if __name__ == '__main__': settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) - if(options.gui == True): + if not options.gui: + print """-q is deprecated. Just use "-f filename" instead""" + # This is because -q on its own causes an error, so -f is necessary and sufficient for cmd line use + if not options.filename: i = GuiBulkImport(db, settings, config) main_window = gtk.Window() main_window.connect('destroy', destroy) @@ -247,7 +257,12 @@ if __name__ == '__main__': importer = fpdb_import.Importer(False,settings, config) importer.setDropIndexes("auto") importer.setFailOnError(options.failOnError) - importer.addBulkImportImportFileOrDir(options.filename, filter=options.filtername) + importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), filter=options.filtername) importer.setCallHud(False) importer.runImport() importer.clearFileList() + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index fb602e63..49e180df 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -115,6 +115,7 @@ class Importer: # Called from GuiBulkImport to add a file or directory. def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"): """Add a file or directory for bulk import""" + # Bulk import never monitors # if directory, add all files in it. Otherwise add single file. # TODO: only add sane files? @@ -124,7 +125,6 @@ class Importer: self.addImportFile(os.path.join(inputPath, subdir[0], file), site="default", filter=filter) else: self.addImportFile(inputPath, site="default", filter=filter) - #Add a directory of files to filelist #Only one import directory per site supported. #dirlist is a hash of lists: @@ -355,7 +355,6 @@ class Importer: isTourney=fpdb_simple.isTourney(hand[0]) if not isTourney: fpdb_simple.filterAnteBlindFold(site,hand) - hand=fpdb_simple.filterCrap(site, hand, isTourney) self.hand=hand try: diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index 9dd54f83..8550a102 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -65,6 +65,8 @@ def mainParser(backend, db, cursor, site, category, hand, config): tourneyTypeId=fpdb_simple.recogniseTourneyTypeId(cursor, siteID, buyin, fee, knockout, rebuyOrAddon) fpdb_simple.isAlreadyInDB(cursor, gametypeID, siteHandNo) + hand=fpdb_simple.filterCrap(site, hand, isTourney) + #part 2: classify lines by type (e.g. cards, action, win, sectionchange) and street fpdb_simple.classifyLines(hand, category, lineTypes, lineStreets) From 39db65b65dfe438744690124226f5baca142a3b4 Mon Sep 17 00:00:00 2001 From: Ray Date: Sat, 21 Mar 2009 11:38:17 -0400 Subject: [PATCH 5/6] Fix bug in defaults of import params + bug in printing same. --- pyfpdb/Configuration.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index 81645694..19e62038 100755 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -32,12 +32,12 @@ import shutil import xml.dom.minidom from xml.dom.minidom import Node -def fix_tf(x): +def fix_tf(x, default = True): if x == "1" or x == 1 or string.lower(x) == "true" or string.lower(x) == "t": return True if x == "0" or x == 0 or string.lower(x) == "false" or string.lower(x) == "f": return False - return False + return default class Layout: def __init__(self, node): @@ -212,18 +212,12 @@ class Import: self.interval = node.getAttribute("interval") self.callFpdbHud = node.getAttribute("callFpdbHud") self.hhArchiveBase = node.getAttribute("hhArchiveBase") - if node.hasAttribute("saveActions"): - self.saveActions = fix_tf(node.getAttribute("saveActions")) - else: - self.saveActions = True - if node.hasAttribute("fastStoreHudCache"): - self.fastStoreHudCache = fix_tf(node.getAttribute("fastStoreHudCache")) - else: - self.fastStoreHudCache = False + self.saveActions = fix_tf(node.getAttribute("saveActions"), True) + self.fastStoreHudCache = fix_tf(node.getAttribute("fastStoreHudCache"), True) def __str__(self): return " interval = %s\n callFpdbHud = %s\n hhArchiveBase = %s\n saveActions = %s\n fastStoreHudCache = %s\n" \ - % (self.interval, self.callFpdbHud, self.hhArchiveBase, self.saveActions, self.saveActions) + % (self.interval, self.callFpdbHud, self.hhArchiveBase, self.saveActions, self.fastStoreHudCache) class Tv: def __init__(self, node): From 86d350984e07a87edf39e8a7640be1d44639f1a4 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 22 Mar 2009 01:31:36 +0900 Subject: [PATCH 6/6] Dodgy 6 max fix to FTP converter --- pyfpdb/FulltiltToFpdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index fb2bf6ea..b28d4065 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -117,6 +117,9 @@ follow : whether to tail -f the input""" hand.handid = m.group('HID') hand.tablename = m.group('TABLE') hand.starttime = time.strptime(m.group('DATETIME'), "%H:%M:%S ET - %Y/%m/%d") + if m.group('TABLEATTRIBUTES'): + m2 = re.search("(\d+) max", m.group('TABLEATTRIBUTES')) + hand.maxseats = int(m2.group(1)) # These work, but the info is already in the Hand class - should be used for tourneys though. # m.group('SB') # m.group('BB')