From cbac52ccbbdb68a1bd10993bcf615360a3ded0b6 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 13 Dec 2008 00:52:31 +0000 Subject: [PATCH 01/11] Autoimport toggle button --- pyfpdb/GuiAutoImport.py | 67 +++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index bb5e0386..e74c3888 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -71,7 +71,8 @@ class GuiAutoImport (threading.Thread): self.addSites(self.mainVBox) - self.startButton=gtk.Button("Start Autoimport") + self.doAutoImportBool = False + self.startButton=gtk.ToggleButton("Start Autoimport") self.startButton.connect("clicked", self.startClicked, "start clicked") self.mainVBox.add(self.startButton) self.startButton.show() @@ -103,7 +104,7 @@ class GuiAutoImport (threading.Thread): """Callback for timer to do an import iteration.""" self.importer.runUpdated() print "GuiAutoImport.import_dir done" - return True + return self.doAutoImportBool def startClicked(self, widget, data): """runs when user clicks start on auto import tab""" @@ -117,34 +118,42 @@ class GuiAutoImport (threading.Thread): # That is not correct. It should open another dir for importing while piping the # results to the same pipe. This means that self.path should be a a list of dirs # to watch. - try: #uhhh, I don't this this is the best way to check for the existence of an attr - getattr(self, "pipe_to_hud") - except AttributeError: - if os.name == 'nt': - command = "python HUD_main.py" + " %s" % (self.database) - bs = 0 # windows is not happy with line buffing here - self.pipe_to_hud = subprocess.Popen(command, bufsize = bs, stdin = subprocess.PIPE, - universal_newlines=True) - else: - cwd = os.getcwd() - command = os.path.join(cwd, 'HUD_main.py') - bs = 1 - self.pipe_to_hud = subprocess.Popen((command, self.database), bufsize = bs, stdin = subprocess.PIPE, - universal_newlines=True) -# self.pipe_to_hud = subprocess.Popen((command, self.database), bufsize = bs, stdin = subprocess.PIPE, -# universal_newlines=True) -# command = command + " %s" % (self.database) -# print "command = ", command -# self.pipe_to_hud = os.popen(command, 'w') + if widget.get_active(): # toggled on + self.doAutoImportBool = True + widget.set_label(u'Stop Autoimport') + try: #uhhh, I don't this this is the best way to check for the existence of an attr + getattr(self, "pipe_to_hud") + except AttributeError: + if os.name == 'nt': + command = "python HUD_main.py" + " %s" % (self.database) + bs = 0 # windows is not happy with line buffing here + self.pipe_to_hud = subprocess.Popen(command, bufsize = bs, stdin = subprocess.PIPE, + universal_newlines=True) + else: + cwd = os.getcwd() + command = os.path.join(cwd, 'HUD_main.py') + bs = 1 + self.pipe_to_hud = subprocess.Popen((command, self.database), bufsize = bs, stdin = subprocess.PIPE, + universal_newlines=True) + # self.pipe_to_hud = subprocess.Popen((command, self.database), bufsize = bs, stdin = subprocess.PIPE, + # universal_newlines=True) + # command = command + " %s" % (self.database) + # print "command = ", command + # self.pipe_to_hud = os.popen(command, 'w') -# Add directories to importer object. - for site in self.input_settings: - self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1]) - print "Adding import directories - Site: " + site + " dir: "+ str(self.input_settings[site][0]) - self.do_import() - - interval=int(self.intervalEntry.get_text()) - gobject.timeout_add(interval*1000, self.do_import) + # Add directories to importer object. + for site in self.input_settings: + self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1]) + print "Adding import directories - Site: " + site + " dir: "+ str(self.input_settings[site][0]) + self.do_import() + + interval=int(self.intervalEntry.get_text()) + gobject.timeout_add(interval*1000, self.do_import) + else: # toggled off + self.doAutoImportBool = False # do_import will return this and stop the gobject callback timer + #TODO: other clean up, such as killing HUD + print "Stopping autoimport" + widget.set_label(u'Start Autoimport') #end def GuiAutoImport.startClicked def get_vbox(self): From 24805700dad5f460d53b0c828dc7db3e79a20431 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sat, 13 Dec 2008 01:06:26 +0000 Subject: [PATCH 02/11] StopAutoImport kills HUD --- pyfpdb/GuiAutoImport.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyfpdb/GuiAutoImport.py b/pyfpdb/GuiAutoImport.py index e74c3888..ba09ab1e 100644 --- a/pyfpdb/GuiAutoImport.py +++ b/pyfpdb/GuiAutoImport.py @@ -40,6 +40,8 @@ class GuiAutoImport (threading.Thread): self.input_settings = {} + self.pipe_to_hud = None + self.importer = fpdb_import.Importer(self,self.settings, self.config) self.importer.setCallHud(True) self.importer.setMinPrint(30) @@ -121,9 +123,7 @@ class GuiAutoImport (threading.Thread): if widget.get_active(): # toggled on self.doAutoImportBool = True widget.set_label(u'Stop Autoimport') - try: #uhhh, I don't this this is the best way to check for the existence of an attr - getattr(self, "pipe_to_hud") - except AttributeError: + if self.pipe_to_hud is None: if os.name == 'nt': command = "python HUD_main.py" + " %s" % (self.database) bs = 0 # windows is not happy with line buffing here @@ -153,6 +153,8 @@ class GuiAutoImport (threading.Thread): self.doAutoImportBool = False # do_import will return this and stop the gobject callback timer #TODO: other clean up, such as killing HUD print "Stopping autoimport" + self.pipe_to_hud.communicate('\n') # waits for process to terminate + self.pipe_to_hud = None widget.set_label(u'Start Autoimport') #end def GuiAutoImport.startClicked From a429cbb6e1f3c9e0d886890584de5f43a9ae419c Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 02:23:40 +0000 Subject: [PATCH 03/11] added new routines to drop and recreate indexes and foreign keys. These could be called from any combination of standalone menu options, as part of the database re-create option or as part of the bulk import option --- pyfpdb/fpdb_simple.py | 303 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 302 insertions(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index e03666fc..8bfd3ba9 100755 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -27,6 +27,307 @@ MYSQL_INNODB=2 PGSQL=3 SQLITE=4 +# Data Structures for index and foreign key creation +# drop_code is an int with possible values: 0 - don't drop for bulk import +# 1 - drop during bulk import +# db differences: +# - note that mysql automatically creates indexes on constrained columns when +# foreign keys are created, while postgres does not. Hence the much longer list +# of indexes is required for postgres. +# all primary keys are left on all the time +# +# table column drop_code + +indexes = [ + [ ] # no db with index 0 + , [ ] # no db with index 1 + , [ # indexes for mysql (list index 2) + {'tab':'Players', 'col':'name', 'drop':0} + , {'tab':'Hands', 'col':'siteHandNo', 'drop':0} + , {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} + ] + , [ # indexes for postgres (list index 3) + {'tab':'Boardcards', 'col':'handId', 'drop':0} + , {'tab':'Gametypes', 'col':'siteId', 'drop':0} + , {'tab':'Hands', 'col':'gametypeId', 'drop':1} + , {'tab':'Hands', 'col':'siteHandNo', 'drop':0} + , {'tab':'HandsActions', 'col':'handplayerId', 'drop':0} + , {'tab':'HandsPlayers', 'col':'handId', 'drop':1} + , {'tab':'HandsPlayers', 'col':'playerId', 'drop':1} + , {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0} + , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} + , {'tab':'HudCache', 'col':'playerId', 'drop':0} + , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'Players', 'col':'siteId', 'drop':1} + , {'tab':'Players', 'col':'name', 'drop':0} + , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} + , {'tab':'Tourneys', 'col':'siteTourneyNo', 'drop':0} + , {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0} + , {'tab':'TourneysPlayers', 'col':'tourneyId', 'drop':0} + , {'tab':'TourneyTypes', 'col':'siteId', 'drop':0} + ] + ] + +foreignKeys = [ + [ ] # no db with index 0 + , [ ] # no db with index 1 + , [ # foreign keys for mysql + {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} + , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'handPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1} + , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + ] + , [ # foreign keys for postgres + {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} + , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'handPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1} + , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + ] + ] + + +# MySQL Notes: +# "FOREIGN KEY (handId) REFERENCES Hands(id)" - requires index on Hands.id +# - creates index handId on .handId +# alter table t drop foreign key fk +# alter table t add foreign key (fkcol) references tab(rcol) +# alter table t add constraint c foreign key (fkcol) references tab(rcol) +# (fkcol is used for foreigh key name) + +# mysql to list indexes: +# SELECT table_name, index_name, non_unique, column_name +# FROM INFORMATION_SCHEMA.STATISTICS +# WHERE table_name = 'tbl_name' +# AND table_schema = 'db_name' +# ORDER BY table_name, index_name, seq_in_index +# +# ALTER TABLE Tourneys ADD INDEX siteTourneyNo(siteTourneyNo) +# ALTER TABLE tab DROP INDEX idx + +# mysql to list fks: +# SELECT constraint_name, table_name, column_name, referenced_table_name, referenced_column_name +# FROM information_schema.KEY_COLUMN_USAGE +# WHERE REFERENCED_TABLE_SCHEMA = (your schema name here) +# AND REFERENCED_TABLE_NAME is not null +# ORDER BY TABLE_NAME, COLUMN_NAME; + +# this may indicate missing object +# _mysql_exceptions.OperationalError: (1025, "Error on rename of '.\\fpdb\\hands' to '.\\fpdb\\#sql2-7f0-1b' (errno: 152)") + + +# PG notes: + +# To add a foreign key constraint to a table: +# ALTER TABLE tab ADD CONSTRAINT c FOREIGN KEY (col) REFERENCES t2(col2) MATCH FULL; +# ALTER TABLE tab DROP CONSTRAINT zipchk +# +# Note: index names must be unique across a schema +# CREATE INDEX idx ON tab(col) +# DROP INDEX idx + +def prepareBulkImport(fdb): + """Drop some indexes/foreign keys to prepare for bulk import. + Currently keeping the standalone indexes as needed to import quickly""" + # fdb is a fpdb_db object including backend, db, cursor, sql variables + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(0) # allow table/index operations to work + for fk in foreignKeys[fdb.backend]: + if fk['drop'] == 1: + if fdb.backend == MYSQL_INNODB: + fdb.cursor.execute("SELECT constraint_name " + + "FROM information_schema.KEY_COLUMN_USAGE " + + #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb' + "WHERE 1=1 " + + "AND table_name = %s AND column_name = %s " + + "AND referenced_table_name = %s " + + "AND referenced_column_name = %s ", + (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) ) + cons = fdb.cursor.fetchone() + print "preparebulk: cons=", cons + if cons: + print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol'] + try: + fdb.cursor.execute("alter table " + fk['fktab'] + " drop foreign key " + cons[0]) + except: + pass + elif fdb.backend == PGSQL: + print "dropping pg fk", fk['fktab'], fk['fkcol'] + try: + fdb.cursor.execute("alter table " + fk['fktab'] + " drop constraint " + + fk['fktab'] + '_' + fk['fkcol'] + '_fkey') + except: + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + + for idx in indexes[fdb.backend]: + if idx['drop'] == 1: + if fdb.backend == MYSQL_INNODB: + print "dropping mysql index ", idx['tab'], idx['col'] + try: + fdb.cursor.execute( "alter table %s drop index %s", (idx['tab'],idx['col']) ) + except: + pass + elif fdb.backend == PGSQL: + print "dropping pg index ", idx['tab'], idx['col'] + # mod to use tab_col for index name? + try: + fdb.cursor.execute( "drop index %s_%s_idx" % (idx['tab'],idx['col']) ) + except: + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(1) # go back to normal isolation level + fdb.db.commit() # seems to clear up errors if there were any in postgres +#end def prepareBulkImport + +def afterBulkImport(fdb): + """Re-create any dropped indexes/foreign keys after bulk import""" + # fdb is a fpdb_db object including backend, db, cursor, sql variables + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(0) # allow table/index operations to work + for fk in foreignKeys[fdb.backend]: + if fk['drop'] == 1: + if fdb.backend == MYSQL_INNODB: + fdb.cursor.execute("SELECT constraint_name " + + "FROM information_schema.KEY_COLUMN_USAGE " + + #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb' + "WHERE 1=1 " + + "AND table_name = %s AND column_name = %s " + + "AND referenced_table_name = %s " + + "AND referenced_column_name = %s ", + (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) ) + cons = fdb.cursor.fetchone() + print "afterbulk: cons=", cons + if cons: + pass + else: + print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol'] + try: + fdb.cursor.execute("alter table " + fk['fktab'] + " add foreign key (" + + fk['fkcol'] + ") references " + fk['rtab'] + "(" + + fk['rcol'] + ")") + except: + pass + elif fdb.backend == PGSQL: + print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol'] + try: + fdb.cursor.execute("alter table " + fk['fktab'] + " add constraint " + + fk['fktab'] + '_' + fk['fkcol'] + '_fkey' + + " foreign key (" + fk['fkcol'] + + ") references " + fk['rtab'] + "(" + fk['rcol'] + ")") + except: + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + + for idx in indexes[fdb.backend]: + if idx['drop'] == 1: + if fdb.backend == MYSQL_INNODB: + print "creating mysql index ", idx['tab'], idx['col'] + try: + fdb.cursor.execute( "alter table %s add index %s(%s)" + , (idx['tab'],idx['col'],idx['col']) ) + except: + pass + elif fdb.backend == PGSQL: + # mod to use tab_col for index name? + print "creating pg index ", idx['tab'], idx['col'] + try: + print "create index %s_%s_idx on %s(%s)" % (idx['tab'], idx['col'], idx['tab'], idx['col']) + fdb.cursor.execute( "create index %s_%s_idx on %s(%s)" + % (idx['tab'], idx['col'], idx['tab'], idx['col']) ) + except: + print " ERROR! :-(" + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(1) # go back to normal isolation level + fdb.db.commit() # seems to clear up errors if there were any in postgres +#end def afterBulkImport + +def createAllIndexes(fdb): + """Create new indexes""" + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(0) # allow table/index operations to work + for idx in indexes[fdb.backend]: + if fdb.backend == MYSQL_INNODB: + print "creating mysql index ", idx['tab'], idx['col'] + try: + fdb.cursor.execute( "alter table %s add index %s(%s)" + , (idx['tab'],idx['col'],idx['col']) ) + except: + pass + elif fdb.backend == PGSQL: + # mod to use tab_col for index name? + print "creating pg index ", idx['tab'], idx['col'] + try: + print "create index %s_%s_idx on %s(%s)" % (idx['tab'], idx['col'], idx['tab'], idx['col']) + fdb.cursor.execute( "create index %s_%s_idx on %s(%s)" + % (idx['tab'], idx['col'], idx['tab'], idx['col']) ) + except: + print " ERROR! :-(" + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(1) # go back to normal isolation level +#end def createAllIndexes + +def dropAllIndexes(fdb): + """Drop all standalone indexes (i.e. not including primary keys or foreign keys) + using list of indexes in indexes data structure""" + # maybe upgrade to use data dictionary?? (but take care to exclude PK and FK) + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(0) # allow table/index operations to work + for idx in indexes[fdb.backend]: + if fdb.backend == MYSQL_INNODB: + print "dropping mysql index ", idx['tab'], idx['col'] + try: + fdb.cursor.execute( "alter table %s drop index %s" + , (idx['tab'],idx['col']) ) + except: + pass + elif fdb.backend == PGSQL: + print "dropping pg index ", idx['tab'], idx['col'] + # mod to use tab_col for index name? + try: + fdb.cursor.execute( "drop index %s_%s_idx" + % (idx['tab'],idx['col']) ) + except: + pass + else: + print "Only MySQL and Postgres supported so far" + return -1 + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(1) # go back to normal isolation level +#end def dropAllIndexes + +def analyzeDB(fdb): + """Do whatever the DB can offer to update index/table statistics""" + if fdb.backend == PGSQL: + fdb.db.set_isolation_level(0) # allow vacuum to work + try: + fdb.cursor.execute("vacuum analyze") + except: + print "Error during vacuum" + fdb.db.set_isolation_level(1) # go back to normal isolation level +#end def analyzeDB class DuplicateError(Exception): def __init__(self, value): @@ -39,7 +340,7 @@ class FpdbError(Exception): self.value = value def __str__(self): return repr(self.value) - + # gets value for last auto-increment key generated # returns -1 if a problem occurs def getLastInsertId(backend, conn, cursor): From d4e03424bfa5966f4f07ba8974f207237d012011 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 02:30:19 +0000 Subject: [PATCH 04/11] call routines to drop and recreate some indexes and foreign keys before and after bulk import --- pyfpdb/fpdb_import.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 306c0880..2d35a97a 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -116,8 +116,11 @@ class Importer: #Run full import on filelist def runImport(self): + fpdb_simple.prepareBulkImport(self.fdb) for file in self.filelist: self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + fpdb_simple.afterBulkImport(self.fdb) + fpdb_simple.analyzeDB(self.fdb) #Run import on updated files, then store latest update time. def runUpdated(self): From fb6b8e5a5bd7827af5003c47dca40b31e9dc2f1a Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 02:38:09 +0000 Subject: [PATCH 05/11] add variable at top of file to control whether actions are saved or not and commented out timing debug statement --- pyfpdb/fpdb_save_to_db.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index 3a7989ca..a7e75e7f 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -22,6 +22,11 @@ from time import time import fpdb_simple +saveActions=False # set this to False to avoid storing action data + # Pros: speeds up imports + # Cons: no action data is saved, need to keep the hand histories + # variance not available on stats page + #stores a stud/razz hand into the database def ring_stud(backend, db, cursor, base, category, site_hand_no, gametype_id, hand_start_time ,names, player_ids, start_cashes, antes, card_values, card_suits, winnings, rakes @@ -39,8 +44,9 @@ def ring_stud(backend, db, cursor, base, category, site_hand_no, gametype_id, ha fpdb_simple.storeHudCache(cursor, base, category, gametype_id, player_ids, hudImportData) - fpdb_simple.storeActions(cursor, hands_players_ids, action_types - ,allIns, action_amounts, actionNos) + if saveActions: + fpdb_simple.storeActions(cursor, hands_players_ids, action_types + ,allIns, action_amounts, actionNos) return hands_id #end def ring_stud @@ -66,10 +72,10 @@ def ring_holdem_omaha(backend, db, cursor, base, category, site_hand_no, gametyp t5 = time() fpdb_simple.store_board_cards(cursor, hands_id, board_values, board_suits) t6 = time() - fpdb_simple.storeActions(cursor, hands_players_ids, action_types, allIns, action_amounts, actionNos) + if saveActions: + fpdb_simple.storeActions(cursor, hands_players_ids, action_types, allIns, action_amounts, actionNos) t7 = time() - print "cards=%4.3f board=%4.3f hands=%4.3f plyrs=%4.3f hudcache=%4.3f board=%4.3f actions=%4.3f" \ - % (t1-t0, t2-t1, t3-t2, t4-t3, t5-t4, t6-t5, t7-t6) + #print "fills=(%4.3f) saves=(%4.3f,%4.3f,%4.3f,%4.3f)" % (t2-t0, t3-t2, t4-t3, t5-t4, t6-t5) return hands_id #end def ring_holdem_omaha @@ -98,7 +104,8 @@ def tourney_holdem_omaha(backend, db, cursor, base, category, siteTourneyNo, buy fpdb_simple.store_board_cards(cursor, hands_id, board_values, board_suits) - fpdb_simple.storeActions(cursor, hands_players_ids, action_types, allIns, action_amounts, actionNos) + if saveActions: + fpdb_simple.storeActions(cursor, hands_players_ids, action_types, allIns, action_amounts, actionNos) return hands_id #end def tourney_holdem_omaha @@ -122,6 +129,7 @@ def tourney_stud(backend, db, cursor, base, category, siteTourneyNo, buyin, fee, fpdb_simple.storeHudCache(cursor, base, category, gametypeId, playerIds, hudImportData) - fpdb_simple.storeActions(cursor, hands_players_ids, actionTypes, allIns, actionAmounts, actionNos) + if saveActions: + fpdb_simple.storeActions(cursor, hands_players_ids, actionTypes, allIns, actionAmounts, actionNos) return hands_id #end def tourney_stud From 26506b3421c5a23497ce56ec7c741b7f391ab12b Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 02:42:07 +0000 Subject: [PATCH 06/11] use new routine in fpdb_simple to create indexes --- pyfpdb/fpdb_db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index e1c61c6d..e51538ee 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -108,9 +108,9 @@ class fpdb_db: self.cursor.execute(self.sql.query['createHandsPlayersTable']) self.cursor.execute(self.sql.query['createHandsActionsTable']) self.cursor.execute(self.sql.query['createHudCacheTable']) - self.cursor.execute(self.sql.query['addTourneyIndex']) - self.cursor.execute(self.sql.query['addHandsIndex']) - self.cursor.execute(self.sql.query['addPlayersIndex']) + #self.cursor.execute(self.sql.query['addTourneyIndex']) + #self.cursor.execute(self.sql.query['addHandsIndex']) + #self.cursor.execute(self.sql.query['addPlayersIndex']) self.fillDefaultData() self.db.commit() #end def disconnect @@ -185,6 +185,7 @@ class fpdb_db: self.drop_tables() self.create_tables() + fpdb_simple.createAllIndexes(self) self.db.commit() print "Finished recreating tables" #end def recreate_tables From 791068d24f008817857a5a1c61f403754f02f36e Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 02:50:09 +0000 Subject: [PATCH 07/11] refine column headings, handle null stats and remove debug message --- pyfpdb/GuiPlayerStats.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pyfpdb/GuiPlayerStats.py b/pyfpdb/GuiPlayerStats.py index 8220e204..5a53813f 100644 --- a/pyfpdb/GuiPlayerStats.py +++ b/pyfpdb/GuiPlayerStats.py @@ -60,7 +60,7 @@ class GuiPlayerStats (threading.Thread): vbox.add(self.stats_table) # Create header row - titles = ("Game", "Hands", "VPIP", "PFR", "saw_f", "sawsd", "wtsdwsf", "wmsd", "FlAFq", "TuAFq", "RvAFq", "PFAFq", "Net($)", "BB/100", "$/hand", "Variance") + titles = ("Game", "Hands", "VPIP", "PFR", "Saw_F", "SawSD", "WtSDwsF", "W$SD", "FlAFq", "TuAFq", "RvAFq", "PoFAFq", "Net($)", "BB/100", "$/hand", "Variance") col = 0 row = 0 @@ -71,14 +71,17 @@ class GuiPlayerStats (threading.Thread): col +=1 for row in range(rows-1): + if(row%2 == 0): + bgcolor = "white" + else: + bgcolor = "lightgrey" for col in range(cols): - if(row%2 == 0): - bgcolor = "white" - else: - bgcolor = "lightgrey" eb = gtk.EventBox() eb.modify_bg(gtk.STATE_NORMAL, gtk.gdk.color_parse(bgcolor)) - l = gtk.Label(result[row][col]) + if result[row][col]: + l = gtk.Label(result[row][col]) + else: + l = gtk.Label(' ') if col == 0: l.set_alignment(xalign=0.0, yalign=0.5) else: @@ -127,7 +130,6 @@ class GuiPlayerStats (threading.Thread): def __set_hero_name(self, w, site): self.heroes[site] = w.get_text() - print "DEBUG: settings heroes[%s]: %s"%(site, self.heroes[site]) def __init__(self, db, config, querylist, debug=True): self.debug=debug From 2ae8c792a63642418ec2f6b60b8aaf55f5393f6f Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 03:07:05 +0000 Subject: [PATCH 08/11] playerstats: round value for variance before displaying, correct calculation of $/hand --- pyfpdb/FpdbSQLQueries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyfpdb/FpdbSQLQueries.py b/pyfpdb/FpdbSQLQueries.py index 13899707..d233f2d2 100644 --- a/pyfpdb/FpdbSQLQueries.py +++ b/pyfpdb/FpdbSQLQueries.py @@ -694,7 +694,7 @@ class FpdbSQLQueries: AS BBlPer100 ,hprof2.profitperhand AS Profitperhand */ - ,hprof2.variance as Variance + ,format(hprof2.variance,2) AS Variance FROM (select /* stats from hudcache */ gt.base @@ -793,7 +793,7 @@ class FpdbSQLQueries: AS BBper100 ,hprof2.profitperhand AS Profitperhand */ - ,hprof2.variance as Variance + ,round(hprof2.variance,2) AS Variance FROM (select gt.base ,gt.category @@ -825,10 +825,10 @@ class FpdbSQLQueries: else to_char(100.0*(sum(street1Aggr)+sum(street2Aggr)+sum(street3Aggr)) /(sum(street1Seen)+sum(street2Seen)+sum(street3Seen)),'90D0') end AS PoFAFq - ,to_char(sum(totalProfit)/100.0,'9G999G990D00') AS Net + ,round(sum(totalProfit)/100.0,2) AS Net ,to_char((sum(totalProfit)/(gt.bigBlind+0.0)) / (sum(HDs)/100.0), '990D00') AS BBper100 - ,to_char(sum(totalProfit) / (sum(HDs)+0.0), '990D0000') AS Profitperhand + ,to_char(sum(totalProfit/100.0) / (sum(HDs)+0.0), '990D0000') AS Profitperhand from Gametypes gt inner join Sites s on s.Id = gt.siteId inner join HudCache hc on hc.gameTypeId = gt.Id From f1be7c2ec0e19b693e3dc37dea8daf9752ab4977 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sun, 14 Dec 2008 11:31:47 +0000 Subject: [PATCH 09/11] saveActions setting was supposed to be True in 'official' code --- pyfpdb/fpdb_save_to_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index a7e75e7f..c8d5b208 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -22,9 +22,9 @@ from time import time import fpdb_simple -saveActions=False # set this to False to avoid storing action data +saveActions=True # set this to False to avoid storing action data # Pros: speeds up imports - # Cons: no action data is saved, need to keep the hand histories + # Cons: no action data is saved, so you need to keep the hand histories # variance not available on stats page #stores a stud/razz hand into the database From 7926ac9def71f9ef4c2179123d2d8d8cd1430bb8 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sun, 14 Dec 2008 19:25:04 +0000 Subject: [PATCH 10/11] multiple 'collected pots' handles side pots, rake calculated from totalbets - totalcollected. --- pyfpdb/EverleafToFpdb.py | 8 +- pyfpdb/Hand.py | 411 +++++++++++++++++++++++++++++++++ pyfpdb/HandHistoryConverter.py | 359 +--------------------------- 3 files changed, 417 insertions(+), 361 deletions(-) create mode 100644 pyfpdb/Hand.py diff --git a/pyfpdb/EverleafToFpdb.py b/pyfpdb/EverleafToFpdb.py index 68e4ce5b..ce7bddd2 100755 --- a/pyfpdb/EverleafToFpdb.py +++ b/pyfpdb/EverleafToFpdb.py @@ -193,18 +193,16 @@ class Everleaf(HandHistoryConverter): hand.addShownCards(cards, shows.group('PNAME')) def readCollectPot(self,hand): - m = self.rexx.collect_pot_re.search(hand.string) - if m is not None: + for m in self.rexx.collect_pot_re.finditer(hand.string): if m.group('HAND') is not None: re_card = re.compile('(?P[0-9tjqka][schd])') # copied from earlier cards = set([hand.card(card.group('CARD')) for card in re_card.finditer(m.group('HAND'))]) hand.addShownCards(cards=None, player=m.group('PNAME'), holeandboard=cards) hand.addCollectPot(player=m.group('PNAME'),pot=m.group('POT')) - else: - print "WARNING: Unusual, no one collected; can happen if it's folded to big blind with a dead small blind." + def getRake(self, hand): - hand.rake = hand.totalpot * Decimal('0.05') # probably not quite right + hand.rake = hand.totalpot - hand.totalcollected # * Decimal('0.05') # probably not quite right if __name__ == "__main__": c = Configuration.Config() diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py new file mode 100644 index 00000000..5f5d0fd3 --- /dev/null +++ b/pyfpdb/Hand.py @@ -0,0 +1,411 @@ +#!/usr/bin/python + +#Copyright 2008 Carl Gherardi +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, version 3 of the License. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . +#In the "official" distribution you can find the license in +#agpl-3.0.txt in the docs folder of the package. + +import Configuration +import FpdbRegex +import Hand +import re +import sys +import traceback +import os +import os.path +import xml.dom.minidom +import codecs +from decimal import Decimal +import operator +from time import time + +class Hand: +# def __init__(self, sitename, gametype, sb, bb, string): + + UPS = {'a':'A', 't':'T', 'j':'J', 'q':'Q', 'k':'K'} + def __init__(self, sitename, gametype, string): + self.sitename = sitename + self.gametype = gametype + self.string = string + + self.streetList = ['BLINDS','PREFLOP','FLOP','TURN','RIVER'] # a list of the observed street names in order + + self.handid = 0 + self.sb = gametype[3] + self.bb = gametype[4] + self.tablename = "Slartibartfast" + self.hero = "Hiro" + self.maxseats = 10 + self.counted_seats = 0 + self.buttonpos = 0 + self.seating = [] + self.players = [] + self.posted = [] + self.involved = True + + + # + # Collections indexed by street names + # + + # A MatchObject using a groupnames to identify streets. + # filled by markStreets() + self.streets = None + + # dict from street names to lists of tuples, such as + # [['mct','bets','$10'],['mika','folds'],['carlg','raises','$20']] + # actually they're clearly lists but they probably should be tuples. + self.actions = {} + + # dict from street names to community cards + self.board = {} + + + # + # Collections indexed by player names + # + + # dict from player names to lists of hole cards + self.holecards = {} + + # dict from player names to amounts collected + self.collected = {} + + # Sets of players + self.shown = set() + self.folded = set() + + self.action = [] + self.totalpot = None + self.totalcollected = None + self.rake = None + + self.bets = {} + self.lastBet = {} + for street in self.streetList: + self.bets[street] = {} + self.lastBet[street] = 0 + + def addPlayer(self, seat, name, chips): + """\ +Adds a player to the hand, and initialises data structures indexed by player. +seat (int) indicating the seat +name (string) player name +chips (string) the chips the player has at the start of the hand (can be None) +If a player has None chips he won't be added.""" + if chips is not None: + self.players.append([seat, name, chips]) + self.holecards[name] = [] + for street in self.streetList: + self.bets[street][name] = [] + + + def addHoleCards(self, cards, player): + """\ +Assigns observed holecards to a player. +cards list of card bigrams e.g. ['2h','jc'] +player (string) name of player +hand +Note, will automatically uppercase the rank letter. +""" + try: + self.checkPlayerExists(player) + self.holecards[player] = set([self.card(c) for c in cards]) + except FpdbParseError, e: + print "Tried to add holecards for unknown player: %s" % (player,) + + def addShownCards(self, cards, player, holeandboard=None): + """\ +For when a player shows cards for any reason (for showdown or out of choice). +""" + if cards is not None: + self.shown.add(player) + self.addHoleCards(cards,player) + elif holeandboard is not None: + board = set([c for s in self.board.values() for c in s]) + #print board + #print holeandboard + #print holeandboard.difference(board) + self.addHoleCards(holeandboard.difference(board),player) + + + def checkPlayerExists(self,player): + if player not in [p[1] for p in self.players]: + raise FpdbParseError + + def discardHoleCards(self, cards, player): + try: + self.checkPlayerExists(player) + for card in cards: + self.holecards[player].remove(card) + except FpdbParseError, e: + pass + except ValueError: + print "tried to discard a card %s didn't have" % (player,) + + def setCommunityCards(self, street, cards): + self.board[street] = [self.card(c) for c in cards] + + def card(self,c): + """upper case the ranks but not suits, 'atjqk' => 'ATJQK'""" + for k,v in self.UPS.items(): + c = c.replace(k,v) + return c + + def addBlind(self, player, amount): + # if player is None, it's a missing small blind. + if player is not None: + self.bets['PREFLOP'][player].append(Decimal(amount)) + self.lastBet['PREFLOP'] = Decimal(amount) + self.posted += [player] + + + def addCall(self, street, player=None, amount=None): + # Potentially calculate the amount of the call if not supplied + # corner cases include if player would be all in + if amount is not None: + self.bets[street][player].append(Decimal(amount)) + #self.lastBet[street] = Decimal(amount) + self.actions[street] += [[player, 'calls', amount]] + + def addRaiseTo(self, street, player, amountTo): + """\ +Add a raise on [street] by [player] to [amountTo] +""" + #Given only the amount raised to, the amount of the raise can be calculated by + # working out how much this player has already in the pot + # (which is the sum of self.bets[street][player]) + # and how much he needs to call to match the previous player + # (which is tracked by self.lastBet) + self.checkPlayerExists(player) + committedThisStreet = reduce(operator.add, self.bets[street][player], 0) + amountToCall = self.lastBet[street] - committedThisStreet + self.lastBet[street] = Decimal(amountTo) + amountBy = Decimal(amountTo) - amountToCall + self.bets[street][player].append(amountBy+amountToCall) + self.actions[street] += [[player, 'raises', amountBy, amountTo]] + + def addBet(self, street, player, amount): + self.checkPlayerExists(player) + self.bets[street][player].append(Decimal(amount)) + self.actions[street] += [[player, 'bets', amount]] + + def addFold(self, street, player): + self.checkPlayerExists(player) + self.folded.add(player) + self.actions[street] += [[player, 'folds']] + + def addCheck(self, street, player): + self.checkPlayerExists(player) + self.actions[street] += [[player, 'checks']] + + def addCollectPot(self,player, pot): + self.checkPlayerExists(player) + if player not in self.collected: + self.collected[player] = pot + else: + # possibly lines like "p collected $ from pot" appear during the showdown + # but they are usually unique in the summary, so it's best to try to get them from there. + print "%s collected pot more than once; avoidable by reading winnings only from summary lines?" + + + def totalPot(self): + """If all bets and blinds have been added, totals up the total pot size +Known bug: doesn't take into account side pots""" + if self.totalpot is None: + self.totalpot = 0 + + # player names: + # print [x[1] for x in self.players] + for player in [x[1] for x in self.players]: + for street in self.streetList: + #print street, self.bets[street][player] + self.totalpot += reduce(operator.add, self.bets[street][player], 0) + + if self.totalcollected is None: + self.totalcollected = 0; + for amount in self.collected.values(): + self.totalcollected += Decimal(amount) + + + def getGameTypeAsString(self): + """\ +Map the tuple self.gametype onto the pokerstars string describing it +""" + # currently it appears to be something like ["ring", "hold", "nl", sb, bb]: + gs = {"hold" : "Hold'em", + "omahahi" : "FIXME", + "omahahilo" : "FIXME", + "razz" : "Razz", + "studhi" : "FIXME", + "studhilo" : "FIXME", + "fivedraw" : "5 Card Draw", + "27_1draw" : "FIXME", + "27_3draw" : "Triple Draw 2-7 Lowball", + "badugi" : "FIXME" + } + ls = {"nl" : "No Limit", + "pl" : "Pot Limit", + "fl" : "Limit", + "cn" : "Cap No Limit", + "cp" : "Cap Pot Limit" + } + + string = "%s %s" %(gs[self.gametype[1]], ls[self.gametype[2]]) + + return string + + def printHand(self): + # PokerStars format. + print "\n### Pseudo stars format ###" + print "%s Game #%s: %s ($%s/$%s) - %s" %(self.sitename, self.handid, self.getGameTypeAsString(), self.sb, self.bb, self.starttime) + print "Table '%s' %d-max Seat #%s is the button" %(self.tablename, self.maxseats, self.buttonpos) + for player in self.players: + print "Seat %s: %s ($%s)" %(player[0], player[1], player[2]) + + if(self.posted[0] is None): + print "No small blind posted" + else: + print "%s: posts small blind $%s" %(self.posted[0], self.sb) + + #May be more than 1 bb posting + for a in self.posted[1:]: + print "%s: posts big blind $%s" %(self.posted[1], self.bb) + + # What about big & small blinds? + + print "*** HOLE CARDS ***" + if self.involved: + print "Dealt to %s [%s]" %(self.hero , " ".join(self.holecards[self.hero])) + + if 'PREFLOP' in self.actions: + for act in self.actions['PREFLOP']: + self.printActionLine(act) + + if 'FLOP' in self.actions: + print "*** FLOP *** [%s]" %( " ".join(self.board['Flop'])) + for act in self.actions['FLOP']: + self.printActionLine(act) + + if 'TURN' in self.actions: + print "*** TURN *** [%s] [%s]" %( " ".join(self.board['Flop']), " ".join(self.board['Turn'])) + for act in self.actions['TURN']: + self.printActionLine(act) + + if 'RIVER' in self.actions: + print "*** RIVER *** [%s] [%s]" %(" ".join(self.board['Flop']+self.board['Turn']), " ".join(self.board['River']) ) + for act in self.actions['RIVER']: + self.printActionLine(act) + + + #Some sites don't have a showdown section so we have to figure out if there should be one + # The logic for a showdown is: at the end of river action there are at least two players in the hand + # we probably don't need a showdown section in pseudo stars format for our filtering purposes + if 'SHOWDOWN' in self.actions: + print "*** SHOW DOWN ***" + print "what do they show" + + print "*** SUMMARY ***" + print "Total pot $%s | Rake $%.2f" % (self.totalcollected, self.rake) # TODO side pots + board = [] + for s in self.board.values(): + board += s + if board: # sometimes hand ends preflop without a board + print "Board [%s]" % (" ".join(board)) + + + for player in self.players: + seatnum = player[0] + name = player[1] + if name in self.collected and self.holecards[name]: + print "Seat %d: %s showed [%s] and won ($%s)" % (seatnum, name, " ".join(self.holecards[name]), self.collected[name]) + elif name in self.collected: + print "Seat %d: %s collected ($%s)" % (seatnum, name, self.collected[name]) + elif player[1] in self.shown: + print "Seat %d: %s showed [%s]" % (seatnum, name, " ".join(self.holecards[name])) + elif player[1] in self.folded: + print "Seat %d: %s folded" % (seatnum, name) + else: + print "Seat %d: %s mucked" % (seatnum, name) + + print + # TODO: + # logic for side pots + # logic for which players get to showdown + # I'm just not sure we need to do this so heavily.. and if we do, it's probably better to use pokerlib + #if self.holecards[player[1]]: # empty list default is false + #hole = self.holecards[player[1]] + ##board = [] + ##for s in self.board.values(): + ##board += s + ##playerhand = self.bestHand('hi', board+hole) + ##print "Seat %d: %s showed %s and won/lost with %s" % (player[0], player[1], hole, playerhand) + #print "Seat %d: %s showed %s" % (player[0], player[1], hole) + #else: + #print "Seat %d: %s mucked or folded" % (player[0], player[1]) + + + def printActionLine(self, act): + if act[1] == 'folds' or act[1] == 'checks': + print "%s: %s " %(act[0], act[1]) + if act[1] == 'calls': + print "%s: %s $%s" %(act[0], act[1], act[2]) + if act[1] == 'bets': + print "%s: %s $%s" %(act[0], act[1], act[2]) + if act[1] == 'raises': + print "%s: %s $%s to $%s" %(act[0], act[1], act[2], act[3]) + + # going to use pokereval to figure out hands at some point. + # these functions are copied from pokergame.py + def bestHand(self, side, cards): + return HandHistoryConverter.eval.best('hi', cards, []) + + # from pokergame.py + def bestHandValue(self, side, serial): + (value, cards) = self.bestHand(side, serial) + return value + + # from pokergame.py + # got rid of the _ for internationalisation + def readableHandValueLong(self, side, value, cards): + if value == "NoPair": + if side == "low": + if cards[0][0] == '5': + return ("The wheel") + else: + return join(map(lambda card: card[0], cards), ", ") + else: + return ("High card %(card)s") % { 'card' : (letter2name[cards[0][0]]) } + elif value == "OnePair": + return ("A pair of %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[2][0]]) } + elif value == "TwoPair": + return ("Two pairs %(card1)s and %(card2)s") % { 'card1' : (letter2names[cards[0][0]]), 'card2' : _(letter2names[cards[2][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[4][0]]) } + elif value == "Trips": + return ("Three of a kind %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[3][0]]) } + elif value == "Straight": + return ("Straight %(card1)s to %(card2)s") % { 'card1' : (letter2name[cards[0][0]]), 'card2' : (letter2name[cards[4][0]]) } + elif value == "Flush": + return ("Flush %(card)s high") % { 'card' : (letter2name[cards[0][0]]) } + elif value == "FlHouse": + return ("%(card1)ss full of %(card2)ss") % { 'card1' : (letter2name[cards[0][0]]), 'card2' : (letter2name[cards[3][0]]) } + elif value == "Quads": + return _("Four of a kind %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[4][0]]) } + elif value == "StFlush": + if letter2name[cards[0][0]] == 'Ace': + return ("Royal flush") + else: + return ("Straight flush %(card)s high") % { 'card' : (letter2name[cards[0][0]]) } + return value + + +class FpdbParseError(Exception): pass diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index 77d76eb3..a00df74d 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -17,6 +17,7 @@ import Configuration import FpdbRegex +import Hand import re import sys import traceback @@ -176,7 +177,7 @@ class HandHistoryConverter: def getRake(self, hand): abstract def sanityCheck(self): - sane = True + sane = False base_w = False #Check if hhbase exists and is writable #Note: Will not try to create the base HH directory @@ -208,7 +209,7 @@ class HandHistoryConverter: list.pop() #Last entry is empty for l in list: # print "'" + l + "'" - hands = hands + [Hand(self.sitename, self.gametype, l)] + hands = hands + [Hand.Hand(self.sitename, self.gametype, l)] return hands def readFile(self, filename): @@ -241,357 +242,3 @@ class HandHistoryConverter: result*=100 return result #end def float2int - -class Hand: -# def __init__(self, sitename, gametype, sb, bb, string): - - UPS = {'a':'A', 't':'T', 'j':'J', 'q':'Q', 'k':'K'} - def __init__(self, sitename, gametype, string): - self.sitename = sitename - self.gametype = gametype - self.string = string - - self.streetList = ['BLINDS','PREFLOP','FLOP','TURN','RIVER'] # a list of the observed street names in order - - self.handid = 0 - self.sb = gametype[3] - self.bb = gametype[4] - self.tablename = "Slartibartfast" - self.hero = "Hiro" - self.maxseats = 10 - self.counted_seats = 0 - self.buttonpos = 0 - self.seating = [] - self.players = [] - self.posted = [] - self.involved = True - - - # - # Collections indexed by street names - # - - # A MatchObject using a groupnames to identify streets. - # filled by markStreets() - self.streets = None - - # dict from street names to lists of tuples, such as - # [['mct','bets','$10'],['mika','folds'],['carlg','raises','$20']] - # actually they're clearly lists but they probably should be tuples. - self.actions = {} - - # dict from street names to community cards - self.board = {} - - - # - # Collections indexed by player names - # - - # dict from player names to lists of hole cards - self.holecards = {} - - # dict from player names to amounts collected - self.collected = {} - - # Sets of players - self.shown = set() - self.folded = set() - - self.action = [] - self.totalpot = None - self.rake = None - - self.bets = {} - self.lastBet = {} - for street in self.streetList: - self.bets[street] = {} - self.lastBet[street] = 0 - - def addPlayer(self, seat, name, chips): - """\ -Adds a player to the hand, and initialises data structures indexed by player. -seat (int) indicating the seat -name (string) player name -chips (string) the chips the player has at the start of the hand (can be None) -If a player has None chips he won't be added.""" - if chips is not None: - self.players.append([seat, name, chips]) - self.holecards[name] = [] - for street in self.streetList: - self.bets[street][name] = [] - - - def addHoleCards(self, cards, player): - """\ -Assigns observed holecards to a player. -cards list of card bigrams e.g. ['2h','jc'] -player (string) name of player -hand -Note, will automatically uppercase the rank letter. -""" - try: - self.checkPlayerExists(player) - self.holecards[player] = set([self.card(c) for c in cards]) - except FpdbParseError, e: - print "Tried to add holecards for unknown player: %s" % (player,) - - def addShownCards(self, cards, player, holeandboard=None): - """\ -For when a player shows cards for any reason (for showdown or out of choice). -""" - if cards is not None: - self.shown.add(player) - self.addHoleCards(cards,player) - elif holeandboard is not None: - board = set([c for s in self.board.values() for c in s]) - #print board - #print holeandboard - #print holeandboard.difference(board) - self.addHoleCards(holeandboard.difference(board),player) - - - def checkPlayerExists(self,player): - if player not in [p[1] for p in self.players]: - raise FpdbParseError - - def discardHoleCards(self, cards, player): - try: - self.checkPlayerExists(player) - for card in cards: - self.holecards[player].remove(card) - except FpdbParseError, e: - pass - except ValueError: - print "tried to discard a card %s didn't have" % (player,) - - def setCommunityCards(self, street, cards): - self.board[street] = [self.card(c) for c in cards] - - def card(self,c): - """upper case the ranks but not suits, 'atjqk' => 'ATJQK'""" - for k,v in self.UPS.items(): - c = c.replace(k,v) - return c - - def addBlind(self, player, amount): - # if player is None, it's a missing small blind. - if player is not None: - self.bets['PREFLOP'][player].append(Decimal(amount)) - self.lastBet['PREFLOP'] = Decimal(amount) - self.posted += [player] - - - def addCall(self, street, player=None, amount=None): - # Potentially calculate the amount of the call if not supplied - # corner cases include if player would be all in - if amount is not None: - self.bets[street][player].append(Decimal(amount)) - #self.lastBet[street] = Decimal(amount) - self.actions[street] += [[player, 'calls', amount]] - - def addRaiseTo(self, street, player, amountTo): - """\ -Add a raise on [street] by [player] to [amountTo] -""" - #Given only the amount raised to, the amount of the raise can be calculated by - # working out how much this player has already in the pot - # (which is the sum of self.bets[street][player]) - # and how much he needs to call to match the previous player - # (which is tracked by self.lastBet) - self.checkPlayerExists(player) - committedThisStreet = reduce(operator.add, self.bets[street][player], 0) - amountToCall = self.lastBet[street] - committedThisStreet - self.lastBet[street] = Decimal(amountTo) - amountBy = Decimal(amountTo) - amountToCall - self.bets[street][player].append(amountBy+amountToCall) - self.actions[street] += [[player, 'raises', amountBy, amountTo]] - - def addBet(self, street, player, amount): - self.checkPlayerExists(player) - self.bets[street][player].append(Decimal(amount)) - self.actions[street] += [[player, 'bets', amount]] - - def addFold(self, street, player): - self.checkPlayerExists(player) - self.folded.add(player) - self.actions[street] += [[player, 'folds']] - - def addCheck(self, street, player): - self.checkPlayerExists(player) - self.actions[street] += [[player, 'checks']] - - def addCollectPot(self,player, pot): - self.checkPlayerExists(player) - if player not in self.collected: - self.collected[player] = pot - else: - # possibly lines like "p collected $ from pot" appear during the showdown - # but they are usually unique in the summary, so it's best to try to get them from there. - print "%s collected pot more than once; avoidable by reading winnings only from summary lines?" - - - def totalPot(self): - """If all bets and blinds have been added, totals up the total pot size -Known bug: doesn't take into account side pots""" - if self.totalpot is None: - self.totalpot = 0 - - # player names: - # print [x[1] for x in self.players] - for player in [x[1] for x in self.players]: - for street in self.streetList: - #print street, self.bets[street][player] - self.totalpot += reduce(operator.add, self.bets[street][player], 0) - - def getGameTypeAsString(self): - """\ -Map the tuple self.gametype onto the pokerstars string describing it -""" - # currently it appears to be something like ["ring", "hold", "nl", sb, bb]: - return "Hold'em No Limit" - - def printHand(self): - # PokerStars format. - print "\n### Pseudo stars format ###" - print "%s Game #%s: %s ($%s/$%s) - %s" %(self.sitename, self.handid, self.getGameTypeAsString(), self.sb, self.bb, self.starttime) - print "Table '%s' %d-max Seat #%s is the button" %(self.tablename, self.maxseats, self.buttonpos) - for player in self.players: - print "Seat %s: %s ($%s)" %(player[0], player[1], player[2]) - - if(self.posted[0] is None): - print "No small blind posted" - else: - print "%s: posts small blind $%s" %(self.posted[0], self.sb) - - #May be more than 1 bb posting - for a in self.posted[1:]: - print "%s: posts big blind $%s" %(self.posted[1], self.bb) - - # What about big & small blinds? - - print "*** HOLE CARDS ***" - if self.involved: - print "Dealt to %s [%s]" %(self.hero , " ".join(self.holecards[self.hero])) - - if 'PREFLOP' in self.actions: - for act in self.actions['PREFLOP']: - self.printActionLine(act) - - if 'FLOP' in self.actions: - print "*** FLOP *** [%s]" %( " ".join(self.board['Flop'])) - for act in self.actions['FLOP']: - self.printActionLine(act) - - if 'TURN' in self.actions: - print "*** TURN *** [%s] [%s]" %( " ".join(self.board['Flop']), " ".join(self.board['Turn'])) - for act in self.actions['TURN']: - self.printActionLine(act) - - if 'RIVER' in self.actions: - print "*** RIVER *** [%s] [%s]" %(" ".join(self.board['Flop']+self.board['Turn']), " ".join(self.board['River']) ) - for act in self.actions['RIVER']: - self.printActionLine(act) - - - #Some sites don't have a showdown section so we have to figure out if there should be one - # The logic for a showdown is: at the end of river action there are at least two players in the hand - # we probably don't need a showdown section in pseudo stars format for our filtering purposes - if 'SHOWDOWN' in self.actions: - print "*** SHOW DOWN ***" - print "what do they show" - - print "*** SUMMARY ***" - print "Total pot $%s | Rake $%.2f)" % (self.totalpot, self.rake) # TODO side pots - board = [] - for s in self.board.values(): - board += s - if board: # sometimes hand ends preflop without a board - print "Board [%s]" % (" ".join(board)) - - - for player in self.players: - seatnum = player[0] - name = player[1] - if name in self.collected and self.holecards[name]: - print "Seat %d: %s showed [%s] and won ($%s)" % (seatnum, name, " ".join(self.holecards[name]), self.collected[name]) - elif name in self.collected: - print "Seat %d: %s collected ($%s)" % (seatnum, name, self.collected[name]) - elif player[1] in self.shown: - print "Seat %d: %s showed [%s]" % (seatnum, name, " ".join(self.holecards[name])) - elif player[1] in self.folded: - print "Seat %d: %s folded" % (seatnum, name) - else: - print "Seat %d: %s mucked" % (seatnum, name) - - print - # TODO: - # logic for side pots - # logic for which players get to showdown - # I'm just not sure we need to do this so heavily.. and if we do, it's probably better to use pokerlib - #if self.holecards[player[1]]: # empty list default is false - #hole = self.holecards[player[1]] - ##board = [] - ##for s in self.board.values(): - ##board += s - ##playerhand = self.bestHand('hi', board+hole) - ##print "Seat %d: %s showed %s and won/lost with %s" % (player[0], player[1], hole, playerhand) - #print "Seat %d: %s showed %s" % (player[0], player[1], hole) - #else: - #print "Seat %d: %s mucked or folded" % (player[0], player[1]) - - - def printActionLine(self, act): - if act[1] == 'folds' or act[1] == 'checks': - print "%s: %s " %(act[0], act[1]) - if act[1] == 'calls': - print "%s: %s $%s" %(act[0], act[1], act[2]) - if act[1] == 'bets': - print "%s: %s $%s" %(act[0], act[1], act[2]) - if act[1] == 'raises': - print "%s: %s $%s to $%s" %(act[0], act[1], act[2], act[3]) - - # going to use pokereval to figure out hands at some point. - # these functions are copied from pokergame.py - def bestHand(self, side, cards): - return HandHistoryConverter.eval.best('hi', cards, []) - - # from pokergame.py - def bestHandValue(self, side, serial): - (value, cards) = self.bestHand(side, serial) - return value - - # from pokergame.py - # got rid of the _ for internationalisation - def readableHandValueLong(self, side, value, cards): - if value == "NoPair": - if side == "low": - if cards[0][0] == '5': - return ("The wheel") - else: - return join(map(lambda card: card[0], cards), ", ") - else: - return ("High card %(card)s") % { 'card' : (letter2name[cards[0][0]]) } - elif value == "OnePair": - return ("A pair of %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[2][0]]) } - elif value == "TwoPair": - return ("Two pairs %(card1)s and %(card2)s") % { 'card1' : (letter2names[cards[0][0]]), 'card2' : _(letter2names[cards[2][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[4][0]]) } - elif value == "Trips": - return ("Three of a kind %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[3][0]]) } - elif value == "Straight": - return ("Straight %(card1)s to %(card2)s") % { 'card1' : (letter2name[cards[0][0]]), 'card2' : (letter2name[cards[4][0]]) } - elif value == "Flush": - return ("Flush %(card)s high") % { 'card' : (letter2name[cards[0][0]]) } - elif value == "FlHouse": - return ("%(card1)ss full of %(card2)ss") % { 'card1' : (letter2name[cards[0][0]]), 'card2' : (letter2name[cards[3][0]]) } - elif value == "Quads": - return _("Four of a kind %(card)s") % { 'card' : (letter2names[cards[0][0]]) } + (", %(card)s kicker") % { 'card' : (letter2name[cards[4][0]]) } - elif value == "StFlush": - if letter2name[cards[0][0]] == 'Ace': - return ("Royal flush") - else: - return ("Straight flush %(card)s high") % { 'card' : (letter2name[cards[0][0]]) } - return value - - -class FpdbParseError(Exception): pass \ No newline at end of file From 3dbb0e83760dbbed2519c77ec2d2def3bf823968 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Sun, 14 Dec 2008 22:05:51 +0000 Subject: [PATCH 11/11] Everleaf uncalled bets; pot total; rake --- pyfpdb/EverleafToFpdb.py | 25 +++++------- pyfpdb/Hand.py | 74 ++++++++++++++++++++++++++++++---- pyfpdb/HandHistoryConverter.py | 9 +++-- 3 files changed, 83 insertions(+), 25 deletions(-) diff --git a/pyfpdb/EverleafToFpdb.py b/pyfpdb/EverleafToFpdb.py index ce7bddd2..188e4ade 100755 --- a/pyfpdb/EverleafToFpdb.py +++ b/pyfpdb/EverleafToFpdb.py @@ -75,6 +75,7 @@ class Everleaf(HandHistoryConverter): self.rexx.setPlayerInfoRegex('Seat (?P[0-9]+): (?P.*) \(\s+(\$ (?P[.0-9]+) USD|new player|All-in) \)') self.rexx.setPostSbRegex('.*\n(?P.*): posts small blind \[\$? (?P[.0-9]+)') self.rexx.setPostBbRegex('.*\n(?P.*): posts big blind \[\$? (?P[.0-9]+)') + self.rexx.setPostBothRegex('.*\n(?P.*): posts small \& big blinds \[\$? (?P[.0-9]+)') # mct : what about posting small & big blinds simultaneously? self.rexx.setHeroCardsRegex('.*\nDealt\sto\s(?P.*)\s\[ (?P\S\S), (?P\S\S) \]') self.rexx.setActionStepRegex('.*\n(?P.*)(?P: bets| checks| raises| calls| folds)(\s\[\$ (?P[.\d]+) USD\])?') @@ -131,7 +132,8 @@ class Everleaf(HandHistoryConverter): r"(\*\* Dealing Turn \*\* \[ \S\S \](?P.+(?=\*\* Dealing River \*\*)|.+))?" r"(\*\* Dealing River \*\* \[ \S\S \](?P.+))?", hand.string,re.DOTALL) - hand.streets = m + hand.addStreets(m) + def readCommunityCards(self, hand): # currently regex in wrong place pls fix my brain's fried @@ -146,15 +148,13 @@ class Everleaf(HandHistoryConverter): def readBlinds(self, hand): try: m = self.rexx.small_blind_re.search(hand.string) - hand.addBlind(m.group('PNAME'), m.group('SB')) - #hand.posted = [m.group('PNAME')] - except: - hand.addBlind(None, 0) - #hand.posted = ["FpdbNBP"] - m = self.rexx.big_blind_re.finditer(hand.string) - for a in m: - hand.addBlind(a.group('PNAME'), a.group('BB')) - #hand.posted = hand.posted + [a.group('PNAME')] + hand.addBlind(m.group('PNAME'), 'small blind', m.group('SB')) + except: # no small blind + hand.addBlind(None, None, None) + for a in self.rexx.big_blind_re.finditer(hand.string): + hand.addBlind(a.group('PNAME'), 'big blind', a.group('BB')) + for a in self.rexx.both_blinds_re.finditer(hand.string): + hand.addBlind(a.group('PNAME'), 'small & big blinds', a.group('SBBB')) def readHeroCards(self, hand): m = self.rexx.hero_cards_re.search(hand.string) @@ -167,7 +167,6 @@ class Everleaf(HandHistoryConverter): def readAction(self, hand, street): m = self.rexx.action_re.finditer(hand.streets.group(street)) - hand.actions[street] = [] for action in m: if action.group('ATYPE') == ' raises': hand.addRaiseTo( street, action.group('PNAME'), action.group('BET') ) @@ -182,6 +181,7 @@ class Everleaf(HandHistoryConverter): else: print "DEBUG: unimplemented readAction: %s %s" %(action.group('PNAME'),action.group('ATYPE'),) #hand.actions[street] += [[action.group('PNAME'), action.group('ATYPE')]] + # TODO: Everleaf does not record uncalled bets. def readShowdownActions(self, hand): @@ -201,9 +201,6 @@ class Everleaf(HandHistoryConverter): hand.addCollectPot(player=m.group('PNAME'),pot=m.group('POT')) - def getRake(self, hand): - hand.rake = hand.totalpot - hand.totalcollected # * Decimal('0.05') # probably not quite right - if __name__ == "__main__": c = Configuration.Config() e = Everleaf(c, "regression-test-files/everleaf/Speed_Kuala_full.txt") diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 5f5d0fd3..809c1be7 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -88,6 +88,7 @@ class Hand: self.action = [] self.totalpot = None self.totalcollected = None + self.rake = None self.bets = {} @@ -110,6 +111,17 @@ If a player has None chips he won't be added.""" self.bets[street][name] = [] + def addStreets(self, match): + # go through m and initialise actions to empty list for each street. + if match is not None: + self.streets = match + for street in match.groupdict(): + if match.group(street) is not None: + self.actions[street] = [] + + else: + print "empty markStreets match" # better to raise exception and put process hand in a try block + def addHoleCards(self, cards, player): """\ Assigns observed holecards to a player. @@ -162,12 +174,18 @@ For when a player shows cards for any reason (for showdown or out of choice). c = c.replace(k,v) return c - def addBlind(self, player, amount): + def addBlind(self, player, blindtype, amount): # if player is None, it's a missing small blind. if player is not None: self.bets['PREFLOP'][player].append(Decimal(amount)) - self.lastBet['PREFLOP'] = Decimal(amount) + self.actions['PREFLOP'] += [(player, 'posts', blindtype, amount)] + if blindtype == 'big blind': + self.lastBet['PREFLOP'] = Decimal(amount) + elif blindtype == 'small & big blinds': + # extra small blind is 'dead' + self.lastBet['PREFLOP'] = Decimal(self.bb) self.posted += [player] + def addCall(self, street, player=None, amount=None): @@ -176,7 +194,7 @@ For when a player shows cards for any reason (for showdown or out of choice). if amount is not None: self.bets[street][player].append(Decimal(amount)) #self.lastBet[street] = Decimal(amount) - self.actions[street] += [[player, 'calls', amount]] + self.actions[street] += [(player, 'calls', amount)] def addRaiseTo(self, street, player, amountTo): """\ @@ -193,21 +211,22 @@ Add a raise on [street] by [player] to [amountTo] self.lastBet[street] = Decimal(amountTo) amountBy = Decimal(amountTo) - amountToCall self.bets[street][player].append(amountBy+amountToCall) - self.actions[street] += [[player, 'raises', amountBy, amountTo]] + self.actions[street] += [(player, 'raises', amountBy, amountTo, amountToCall)] def addBet(self, street, player, amount): self.checkPlayerExists(player) self.bets[street][player].append(Decimal(amount)) - self.actions[street] += [[player, 'bets', amount]] + self.actions[street] += [(player, 'bets', amount)] + self.lastBet[street] = Decimal(amount) def addFold(self, street, player): self.checkPlayerExists(player) self.folded.add(player) - self.actions[street] += [[player, 'folds']] + self.actions[street] += [(player, 'folds')] def addCheck(self, street, player): self.checkPlayerExists(player) - self.actions[street] += [[player, 'checks']] + self.actions[street] += [(player, 'checks')] def addCollectPot(self,player, pot): self.checkPlayerExists(player) @@ -231,12 +250,50 @@ Known bug: doesn't take into account side pots""" for street in self.streetList: #print street, self.bets[street][player] self.totalpot += reduce(operator.add, self.bets[street][player], 0) + print "conventional totalpot:", self.totalpot + self.totalpot = 0 + + print self.actions + for street in self.actions: + uncalled = 0 + calls = [0] + for act in self.actions[street]: + if act[1] == 'bets': # [name, 'bets', amount] + self.totalpot += Decimal(act[2]) + uncalled = Decimal(act[2]) # only the last bet or raise can be uncalled + calls = [0] + print "uncalled: ", uncalled + elif act[1] == 'raises': # [name, 'raises', amountby, amountto, amountcalled] + print "calls %s and raises %s to %s" % (act[4],act[2],act[3]) + self.totalpot += Decimal(act[2]) + Decimal(act[4]) + calls = [0] + uncalled = Decimal(act[2]) + print "uncalled: ", uncalled + elif act[1] == 'calls': # [name, 'calls', amount] + self.totalpot += Decimal(act[2]) + calls = calls + [Decimal(act[2])] + print "calls:", calls + if act[1] == ('posts'): + self.totalpot += Decimal(act[3]) + uncalled = Decimal(act[3]) + if uncalled > 0 and max(calls+[0]) < uncalled: + + print "returning some bet, calls:", calls + print "returned: %.2f from %.2f" % ((uncalled - max(calls)), self.totalpot,) + self.totalpot -= (uncalled - max(calls)) + print "new totalpot:", self.totalpot if self.totalcollected is None: self.totalcollected = 0; for amount in self.collected.values(): self.totalcollected += Decimal(amount) + # TODO: Some sites (Everleaf) don't record uncalled bets. Figure out if a bet is uncalled and subtract it from self.totalcollected. + # remember that portions of bets may be uncalled, so: + # bet followed by no call is an uncalled bet + # bet x followed by call y where y < x has x-y uncalled (and second player all in) + + def getGameTypeAsString(self): """\ @@ -316,7 +373,8 @@ Map the tuple self.gametype onto the pokerstars string describing it print "what do they show" print "*** SUMMARY ***" - print "Total pot $%s | Rake $%.2f" % (self.totalcollected, self.rake) # TODO side pots + print "Total pot $%s | Rake $%.2f" % (self.totalcollected, self.rake) # TODO: side pots + board = [] for s in self.board.values(): board += s diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index a00df74d..68d00e96 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -161,7 +161,8 @@ class HandHistoryConverter: def readPlayerStacks(self, hand): abstract # Needs to return a MatchObject with group names identifying the streets into the Hand object - # that is, pulls the chunks of preflop, flop, turn and river text into hand.streets MatchObject. + # so groups are called by street names 'PREFLOP', 'FLOP', 'STREET2' etc + # blinds are done seperately def markStreets(self, hand): abstract #Needs to return a list in the format @@ -173,8 +174,10 @@ class HandHistoryConverter: def readCollectPot(self, hand): abstract # Some sites don't report the rake. This will be called at the end of the hand after the pot total has been calculated - # so that an inheriting class can calculate it for the specific site if need be. - def getRake(self, hand): abstract + # an inheriting class can calculate it for the specific site if need be. + def getRake(self, hand): + hand.rake = hand.totalpot - hand.totalcollected # * Decimal('0.05') # probably not quite right + def sanityCheck(self): sane = False