From f7616c6ee883c8b65a56fc367f5e6145af5254df Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 17 Dec 2009 02:24:57 +0800 Subject: [PATCH 1/7] [NEWIMPORT] Move hud call to after database commit HUD still doesn't quite work, but getting closer - suspect hud_cache rebuild isn't happening --- pyfpdb/fpdb_import.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 3fc35339..7b3dd4f1 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -432,18 +432,21 @@ class Importer: #This code doesn't do anything yet handlist = hhc.getProcessedHands() self.pos_in_file[file] = hhc.getLastCharacterRead() + to_hud = [] for hand in handlist: #try, except duplicates here? hand.prepInsert(self.database) hand.insert(self.database) if self.callHud and hand.dbid_hands != 0: - #print "DEBUG: call to HUD: handsId: %s" % hand.dbid_hands - #pipe the Hands.id out to the HUD - print "fpdb_import: sending hand to hud", hand.dbid_hands, "pipe =", self.caller.pipe_to_hud - self.caller.pipe_to_hud.stdin.write("%s" % (hand.dbid_hands) + os.linesep) + to_hud.append(hand.dbid_hands) self.database.commit() + #pipe the Hands.id out to the HUD + for hid in to_hud: + print "fpdb_import: sending hand to hud", hand.dbid_hands, "pipe =", self.caller.pipe_to_hud + self.caller.pipe_to_hud.stdin.write("%s" % (hid) + os.linesep) + errors = getattr(hhc, 'numErrors') stored = getattr(hhc, 'numHands') else: From 1fb99e459568dc0448dc7b346f6cbc334361b57f Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 17 Dec 2009 15:59:29 +0800 Subject: [PATCH 2/7] Add test hand - Hand cancelled --- ...LO8-6max-USD-0.05-0.10-20090315.Hand-cancelled.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pyfpdb/regression-test-files/cash/Stars/Flop/LO8-6max-USD-0.05-0.10-20090315.Hand-cancelled.txt diff --git a/pyfpdb/regression-test-files/cash/Stars/Flop/LO8-6max-USD-0.05-0.10-20090315.Hand-cancelled.txt b/pyfpdb/regression-test-files/cash/Stars/Flop/LO8-6max-USD-0.05-0.10-20090315.Hand-cancelled.txt new file mode 100644 index 00000000..9959180c --- /dev/null +++ b/pyfpdb/regression-test-files/cash/Stars/Flop/LO8-6max-USD-0.05-0.10-20090315.Hand-cancelled.txt @@ -0,0 +1,11 @@ +PokerStars Game #25979907808: Omaha Pot Limit ($0.05/$0.10 USD) - 2009/03/15 6:20:33 ET +Table 'Waterman' 6-max Seat #1 is the button +Seat 1: s0rrow ($11.65 in chips) +s0rrow: posts small blind $0.05 +ritalinIV: is sitting out +Hand cancelled +*** SUMMARY *** +Seat 1: s0rrow (button) collected ($0) + + + From 22f98ab1c5203af4df5e38477f3dd7e41b112d74 Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 17 Dec 2009 15:53:12 +0800 Subject: [PATCH 3/7] Add some code to kinda detect hand cancellation hhc.readHandInfo(self) hhc.readPlayerStacks(self) hhc.compilePlayerRegexs(self) hhc.markStreets(self) Is the order, the first correctly failing regex is markStreets --- pyfpdb/Hand.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 6901340e..32140256 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -54,6 +54,7 @@ class Hand(object): self.starttime = 0 self.handText = handText self.handid = 0 + self.cancelled = False self.dbid_hands = 0 self.dbid_pids = None self.dbid_gt = 0 @@ -263,6 +264,8 @@ If a player has None chips he won't be added.""" log.debug("markStreets:\n"+ str(self.streets)) else: log.error("markstreets didn't match") + log.error(" - Assuming hand cancelled") + self.cancelled = True def checkPlayerExists(self,player): if player not in [p[1] for p in self.players]: @@ -613,6 +616,8 @@ class HoldemOmahaHand(Hand): hhc.readPlayerStacks(self) hhc.compilePlayerRegexs(self) hhc.markStreets(self) + if self.cancelled: + return hhc.readBlinds(self) hhc.readAntes(self) hhc.readButton(self) From ba8f61d14ac5af3a180ba5815ab52b8b403775ef Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 17 Dec 2009 18:42:50 +0800 Subject: [PATCH 4/7] Add ability to import Stars archive files. PokerStars support can provide a HH archive. The format is similar but not the same as a a standard hh format as it contains an additional line "Hand #X" between each hand. Patch adds an option -s to GuiBulkImport, which when specified will strip these lines out and continue parsing. --- pyfpdb/GuiBulkImport.py | 4 ++++ pyfpdb/Hand.py | 1 + pyfpdb/HandHistoryConverter.py | 8 +++++++- pyfpdb/fpdb_import.py | 19 +++++++++++++------ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index 7db420c7..16131ab2 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -326,6 +326,8 @@ def main(argv=None): help="How often to print a one-line status report (0 (default) means never)") parser.add_option("-u", "--usage", action="store_true", dest="usage", default=False, help="Print some useful one liners") + parser.add_option("-s", "--starsarchive", action="store_true", dest="starsArchive", default=False, + help="Do the required conversion for Stars Archive format (ie. as provided by support") (options, argv) = parser.parse_args(args = argv) if options.usage == True: @@ -369,6 +371,8 @@ def main(argv=None): importer.setThreads(-1) importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), site=options.filtername) importer.setCallHud(False) + if options.starsArchive: + importer.setStarsArchive(True) (stored, dups, partial, errs, ttime) = importer.runImport() importer.clearFileList() print 'GuiBulkImport done: Stored: %d \tDuplicates: %d \tPartial: %d \tErrors: %d in %s seconds - %.0f/sec'\ diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 32140256..3467216a 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -266,6 +266,7 @@ If a player has None chips he won't be added.""" log.error("markstreets didn't match") log.error(" - Assuming hand cancelled") self.cancelled = True + raise FpdbParseError def checkPlayerExists(self,player): if player not in [p[1] for p in self.players]: diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index 27bb9b1a..a18797df 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -57,7 +57,7 @@ class HandHistoryConverter(): codepage = "cp1252" - def __init__(self, in_path = '-', out_path = '-', follow=False, index=0, autostart=True): + def __init__(self, in_path = '-', out_path = '-', follow=False, index=0, autostart=True, starsArchive=False): """\ in_path (default '-' = sys.stdin) out_path (default '-' = sys.stdout) @@ -66,6 +66,7 @@ follow : whether to tail -f the input""" log.info("HandHistory init - %s subclass, in_path '%s'; out_path '%s'" % (self.sitename, in_path, out_path) ) self.index = 0 + self.starsArchive = starsArchive self.in_path = in_path self.out_path = out_path @@ -254,6 +255,11 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py. self.readFile() self.obs = self.obs.strip() self.obs = self.obs.replace('\r\n', '\n') + if self.starsArchive == True: + log.debug("Converting starsArchive format to readable") + m = re.compile('^Hand #\d+', re.MULTILINE) + self.obs = m.sub('', self.obs) + if self.obs is None or self.obs == "": log.info("Read no hands.") return [] diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 7b3dd4f1..8921d9d8 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -91,6 +91,7 @@ class Importer: self.settings.setdefault("writeQMaxWait", 10) # not used self.settings.setdefault("dropIndexes", "don't drop") self.settings.setdefault("dropHudCache", "don't drop") + self.settings.setdefault("starsArchive", False) self.writeq = None self.database = Database.Database(self.config, sql = self.sql) @@ -134,6 +135,9 @@ class Importer: def setDropHudCache(self, value): self.settings['dropHudCache'] = value + def setStarsArchive(self, value): + self.settings['starsArchive'] = value + # def setWatchTime(self): # self.updated = time() @@ -425,7 +429,7 @@ class Importer: mod = __import__(filter) obj = getattr(mod, filter_name, None) if callable(obj): - hhc = obj(in_path = file, out_path = out_path, index = 0) # Index into file 0 until changeover + hhc = obj(in_path = file, out_path = out_path, index = 0, starsArchive = self.settings['starsArchive']) # Index into file 0 until changeover if hhc.getStatus() and self.NEWIMPORT == False: (stored, duplicates, partial, errors, ttime) = self.import_fpdb_file(db, out_path, site, q) elif hhc.getStatus() and self.NEWIMPORT == True: @@ -435,11 +439,14 @@ class Importer: to_hud = [] for hand in handlist: - #try, except duplicates here? - hand.prepInsert(self.database) - hand.insert(self.database) - if self.callHud and hand.dbid_hands != 0: - to_hud.append(hand.dbid_hands) + if hand is not None: + #try, except duplicates here? + hand.prepInsert(self.database) + hand.insert(self.database) + if self.callHud and hand.dbid_hands != 0: + to_hud.append(hand.dbid_hands) + else: + log.error("Hand processed but empty") self.database.commit() #pipe the Hands.id out to the HUD From a2947c87078ee8778c5503432fbf5cfcd4c76793 Mon Sep 17 00:00:00 2001 From: Worros Date: Fri, 18 Dec 2009 10:27:43 +0800 Subject: [PATCH 5/7] [NEWIMPORT] Add stubbed variable to insert --- pyfpdb/Database.py | 3 ++- pyfpdb/SQL.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 9d1f3f85..72b41336 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1651,7 +1651,8 @@ class Database: pdata[p]['street2CheckCallRaiseDone'], pdata[p]['street3CheckCallRaiseChance'], pdata[p]['street3CheckCallRaiseDone'], - pdata[p]['street4CheckCallRaiseChance'] + pdata[p]['street4CheckCallRaiseChance'], + pdata[p]['street4CheckCallRaiseDone'] ) ) q = self.sql.query['store_hands_players'] diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index c0319dfe..56c1f388 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -3404,10 +3404,11 @@ class Sql: street2CheckCallRaiseDone, street3CheckCallRaiseChance, street3CheckCallRaiseDone, - street4CheckCallRaiseChance + street4CheckCallRaiseChance, + street4CheckCallRaiseDone ) VALUES ( - %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, From 9012c74db7daa55c8e2fe1c429b9e18c42e7146b Mon Sep 17 00:00:00 2001 From: Worros Date: Fri, 18 Dec 2009 13:32:09 +0800 Subject: [PATCH 6/7] [NEWIMPORT] Fix syntax to be 2.5 compatible. Python 2.6 enumerate() function contains a useful 'start' paramater, apparently this did not exist in 2.5. Patch frim Mika Bostrom --- pyfpdb/DerivedStats.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 1a34db1e..5943a10f 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -168,8 +168,10 @@ class DerivedStats(): for player in hand.players: hcs = hand.join_holecards(player[1], asList=True) hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x'] - for i, card in enumerate(hcs[:7], 1): - self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card) + #for i, card in enumerate(hcs[:7], 1): #Python 2.6 syntax + # self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card) + for i, card in enumerate(hcs[:7]): + self.handsplayers[player[1]]['card%s' % (i+1)] = Card.encodeCard(card) # position, @@ -266,13 +268,17 @@ class DerivedStats(): # Then no bets before the player with initiatives first action on current street # ie. if player on street-1 had initiative # and no donkbets occurred - for i, street in enumerate(hand.actionStreets[2:], start=1): - name = self.lastBetOrRaiser(hand.actionStreets[i]) + + # XXX: enumerate(list, start=x) is python 2.6 syntax; 'start' + # came there + #for i, street in enumerate(hand.actionStreets[2:], start=1): + for i, street in enumerate(hand.actionStreets[2:]: + name = self.lastBetOrRaiser(hand.actionStreets[i+1]) if name: - chance = self.noBetsBefore(hand.actionStreets[i+1], name) - self.handsplayers[name]['street%dCBChance' %i] = True + chance = self.noBetsBefore(hand.actionStreets[i+2], name) + self.handsplayers[name]['street%dCBChance' % (i+1)] = True if chance == True: - self.handsplayers[name]['street%dCBDone' %i] = self.betStreet(hand.actionStreets[i+1], name) + self.handsplayers[name]['street%dCBDone' % (i+1)] = self.betStreet(hand.actionStreets[i+2], name) def seen(self, hand, i): pas = set() From d5de93da28723f580242c050bfc9254e953b42f3 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 19 Dec 2009 10:07:53 +0800 Subject: [PATCH 7/7] [NEWIMPORT] Syntax fix --- pyfpdb/DerivedStats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 5943a10f..dbd655a3 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -272,7 +272,7 @@ class DerivedStats(): # XXX: enumerate(list, start=x) is python 2.6 syntax; 'start' # came there #for i, street in enumerate(hand.actionStreets[2:], start=1): - for i, street in enumerate(hand.actionStreets[2:]: + for i, street in enumerate(hand.actionStreets[2:]): name = self.lastBetOrRaiser(hand.actionStreets[i+1]) if name: chance = self.noBetsBefore(hand.actionStreets[i+2], name)