From f9a805f5cdd1087a5dc09eec45ac06f5219aa6a5 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Wed, 17 Dec 2008 23:03:17 +0000 Subject: [PATCH 1/5] add do_commit(config) method to fpdb_db.py to make creating a db connection easier. Used this in guiplayerstats.py to create a separate connection and added commit to stop locks here blocking hand imports. --- pyfpdb/GuiPlayerStats.py | 13 +++++++++---- pyfpdb/fpdb_db.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pyfpdb/GuiPlayerStats.py b/pyfpdb/GuiPlayerStats.py index 5a53813f..1f1967e9 100644 --- a/pyfpdb/GuiPlayerStats.py +++ b/pyfpdb/GuiPlayerStats.py @@ -45,13 +45,13 @@ class GuiPlayerStats (threading.Thread): tmp = self.sql.query['playerStats'] result = self.cursor.execute(self.sql.query['getPlayerId'], (self.heroes[self.activesite],)) - result = self.db.cursor.fetchall() + result = self.cursor.fetchall() if not result == (): pid = result[0][0] pid = result[0][0] tmp = tmp.replace("", "(" + str(pid) + ")") self.cursor.execute(tmp) - result = self.db.cursor.fetchall() + result = self.cursor.fetchall() cols = 16 rows = len(result)+1 # +1 for title row self.stats_table = gtk.Table(rows, cols, False) @@ -90,6 +90,8 @@ class GuiPlayerStats (threading.Thread): self.stats_table.attach(eb, col, col+1, row+1, row+2) l.show() eb.show() + self.fdb.db.commit() + #end def fillStatsFrame(self, vbox): def fillPlayerFrame(self, vbox): for site in self.conf.supported_sites.keys(): @@ -133,9 +135,12 @@ class GuiPlayerStats (threading.Thread): def __init__(self, db, config, querylist, debug=True): self.debug=debug - self.db=db - self.cursor=db.cursor self.conf=config + + # create new db connection to avoid conflicts with other threads + self.fdb = fpdb_db.fpdb_db() + self.fdb.do_connect(self.conf) + self.cursor=self.fdb.cursor self.sql = querylist diff --git a/pyfpdb/fpdb_db.py b/pyfpdb/fpdb_db.py index e51538ee..9d619e9a 100644 --- a/pyfpdb/fpdb_db.py +++ b/pyfpdb/fpdb_db.py @@ -30,6 +30,25 @@ class fpdb_db: self.PGSQL=3 self.SQLITE=4 #end def __init__ + + def do_connect(self, config=None): + """Connects a database using information in config""" + if config is None: + raise FpdbError('Configuration not defined') + + self.settings = {} + if (os.sep=="/"): + self.settings['os']="linuxmac" + else: + self.settings['os']="windows" + + self.settings.update(config.get_db_parameters()) + self.connect(self.settings['db-backend'], + self.settings['db-host'], + self.settings['db-databaseName'], + self.settings['db-user'], + self.settings['db-password']) + #end def do_connect def connect(self, backend=None, host=None, database=None, user=None, password=None): From 257dd35c93ff1d228d50e273e8384e81ee422627 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 3 Jan 2009 17:59:48 +0000 Subject: [PATCH 2/5] let fpdb_db routine work out which bits of config to use for connection --- pyfpdb/fpdb_import.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 2d35a97a..be44c3cd 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -66,11 +66,7 @@ class Importer: #TODO: Is this value in the xml file? self.settings['handCount'] = 0 self.fdb = fpdb_db.fpdb_db() # sets self.fdb.db self.fdb.cursor and self.fdb.sql - self.fdb.connect(self.settings['db-backend'], - self.settings['db-host'], - self.settings['db-databaseName'], - self.settings['db-user'], - self.settings['db-password']) + self.fdb.do_connect(self.config) #Set functions def setCallHud(self, value): From cc31f8626960f53323892e051faa313acbe55ba1 Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 3 Jan 2009 18:00:44 +0000 Subject: [PATCH 3/5] add profit/100 hands stat --- pyfpdb/Stats.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pyfpdb/Stats.py b/pyfpdb/Stats.py index 80b3c7f1..6507719d 100644 --- a/pyfpdb/Stats.py +++ b/pyfpdb/Stats.py @@ -203,6 +203,28 @@ def wmsd(stat_dict, player): '% won money at showdown' ) +def profit100_0(stat_dict, player): + """ Profit won per 100 hands (no decimal places).""" + stat = 0.0 + try: + stat = float(stat_dict[player]['net'])/float(stat_dict[player]['n']) + return (stat, + '%.0f' % (100.0*stat), + 'p=%.0f' % (100.0*stat), + 'p/100=%.0f' % (100.0*stat), + '%d/%d' % (stat_dict[player]['net'], stat_dict[player]['n']), + 'profit/100hands' + ) + except: + print "exception calcing p/100: 100 * %d / %d" % (stat_dict[player]['net'], stat_dict[player]['n']) + return (stat, + '%.0f' % (0), + 'p=%.0f' % (0), + 'p/100=%.0f' % (0), + '(%d/%d)' % (0, 0), + 'profit/100hands' + ) + def saw_f(stat_dict, player): """ Saw flop/4th.""" try: @@ -690,6 +712,7 @@ if __name__== "__main__": print "player = ", player, do_stat(stat_dict, player = player, stat = 'pfr') print "player = ", player, do_stat(stat_dict, player = player, stat = 'pfr_0') print "player = ", player, do_stat(stat_dict, player = player, stat = 'wtsd') + print "player = ", player, do_stat(stat_dict, player = player, stat = 'profit100_0') print "player = ", player, do_stat(stat_dict, player = player, stat = 'saw_f') print "player = ", player, do_stat(stat_dict, player = player, stat = 'n') print "player = ", player, do_stat(stat_dict, player = player, stat = 'fold_f') From ab1e8cf670622ae1678276a03244a95a1316c4aa Mon Sep 17 00:00:00 2001 From: sqlcoder Date: Sat, 3 Jan 2009 19:55:54 +0000 Subject: [PATCH 4/5] added new faster version of storeHudCache routine, storeHudCache2 Set the fastStoreHudCache variable to True to test this out. In the best case it only does one DB operation (update), if this fails because the record does not exist it does an insert. --- pyfpdb/fpdb_save_to_db.py | 18 +++- pyfpdb/fpdb_simple.py | 169 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/pyfpdb/fpdb_save_to_db.py b/pyfpdb/fpdb_save_to_db.py index c8d5b208..ba734e80 100644 --- a/pyfpdb/fpdb_save_to_db.py +++ b/pyfpdb/fpdb_save_to_db.py @@ -22,6 +22,12 @@ from time import time import fpdb_simple +MYSQL_INNODB=2 +PGSQL=3 +SQLITE=4 + +fastStoreHudCache=False # set this to True to test the new storeHudCache routine + saveActions=True # set this to False to avoid storing action data # Pros: speeds up imports # Cons: no action data is saved, so you need to keep the hand histories @@ -68,7 +74,11 @@ def ring_holdem_omaha(backend, db, cursor, base, category, site_hand_no, gametyp backend, db, cursor, category, hands_id, player_ids, start_cashes , positions, card_values, card_suits, winnings, rakes, seatNos) t4 = time() - fpdb_simple.storeHudCache(cursor, base, category, gametype_id, player_ids, hudImportData) + #print "ring holdem, backend=%d" % backend + if fastStoreHudCache: + fpdb_simple.storeHudCache2(backend, cursor, base, category, gametype_id, player_ids, hudImportData) + else: + fpdb_simple.storeHudCache(cursor, base, category, gametype_id, player_ids, hudImportData) t5 = time() fpdb_simple.store_board_cards(cursor, hands_id, board_values, board_suits) t6 = time() @@ -100,7 +110,11 @@ def tourney_holdem_omaha(backend, db, cursor, base, category, siteTourneyNo, buy backend, db, cursor, category, hands_id, player_ids, start_cashes, positions , card_values, card_suits, winnings, rakes, seatNos, tourneys_players_ids) - fpdb_simple.storeHudCache(cursor, base, category, gametype_id, player_ids, hudImportData) + #print "tourney holdem, backend=%d" % backend + if fastStoreHudCache: + fpdb_simple.storeHudCache2(backend, cursor, base, category, gametype_id, player_ids, hudImportData) + else: + fpdb_simple.storeHudCache(cursor, base, category, gametype_id, player_ids, hudImportData) fpdb_simple.store_board_cards(cursor, hands_id, board_values, board_suits) diff --git a/pyfpdb/fpdb_simple.py b/pyfpdb/fpdb_simple.py index e1e0f737..099432c0 100644 --- a/pyfpdb/fpdb_simple.py +++ b/pyfpdb/fpdb_simple.py @@ -2486,6 +2486,175 @@ WHERE gametypeId=%s AND playerId=%s AND activeSeats=%s AND position=%s AND tourn # print "todo: implement storeHudCache for stud base" #end def storeHudCache +def storeHudCache2(backend, cursor, base, category, gametypeId, playerIds, hudImportData): + """Modified version aiming for more speed ...""" +# if (category=="holdem" or category=="omahahi" or category=="omahahilo"): + + #print "storeHudCache, len(playerIds)=", len(playerIds), " len(vpip)=" \ + #, len(hudImportData['street0VPI']), " len(totprof)=", len(hudImportData['totalProfit']) + for player in range (len(playerIds)): + + # Set up a clean row + row=[] + row.append(0)#blank for id + row.append(gametypeId) + row.append(playerIds[player]) + row.append(len(playerIds))#seats + for i in range(len(hudImportData)+2): + row.append(0) + + if base=="hold": + row[4]=hudImportData['position'][player] + else: + row[4]=0 + row[5]=1 #tourneysGametypeId + row[6]+=1 #HDs + if hudImportData['street0VPI'][player]: row[7]+=1 + if hudImportData['street0Aggr'][player]: row[8]+=1 + if hudImportData['street0_3B4BChance'][player]: row[9]+=1 + if hudImportData['street0_3B4BDone'][player]: row[10]+=1 + if hudImportData['street1Seen'][player]: row[11]+=1 + if hudImportData['street2Seen'][player]: row[12]+=1 + if hudImportData['street3Seen'][player]: row[13]+=1 + if hudImportData['street4Seen'][player]: row[14]+=1 + if hudImportData['sawShowdown'][player]: row[15]+=1 + if hudImportData['street1Aggr'][player]: row[16]+=1 + if hudImportData['street2Aggr'][player]: row[17]+=1 + if hudImportData['street3Aggr'][player]: row[18]+=1 + if hudImportData['street4Aggr'][player]: row[19]+=1 + if hudImportData['otherRaisedStreet1'][player]: row[20]+=1 + if hudImportData['otherRaisedStreet2'][player]: row[21]+=1 + if hudImportData['otherRaisedStreet3'][player]: row[22]+=1 + if hudImportData['otherRaisedStreet4'][player]: row[23]+=1 + if hudImportData['foldToOtherRaisedStreet1'][player]: row[24]+=1 + if hudImportData['foldToOtherRaisedStreet2'][player]: row[25]+=1 + if hudImportData['foldToOtherRaisedStreet3'][player]: row[26]+=1 + if hudImportData['foldToOtherRaisedStreet4'][player]: row[27]+=1 + if hudImportData['wonWhenSeenStreet1'][player]!=0.0: row[28]+=hudImportData['wonWhenSeenStreet1'][player] + if hudImportData['wonAtSD'][player]!=0.0: row[29]+=hudImportData['wonAtSD'][player] + if hudImportData['stealAttemptChance'][player]: row[30]+=1 + if hudImportData['stealAttempted'][player]: row[31]+=1 + if hudImportData['foldBbToStealChance'][player]: row[32]+=1 + if hudImportData['foldedBbToSteal'][player]: row[33]+=1 + if hudImportData['foldSbToStealChance'][player]: row[34]+=1 + if hudImportData['foldedSbToSteal'][player]: row[35]+=1 + + if hudImportData['street1CBChance'][player]: row[36]+=1 + if hudImportData['street1CBDone'][player]: row[37]+=1 + if hudImportData['street2CBChance'][player]: row[38]+=1 + if hudImportData['street2CBDone'][player]: row[39]+=1 + if hudImportData['street3CBChance'][player]: row[40]+=1 + if hudImportData['street3CBDone'][player]: row[41]+=1 + if hudImportData['street4CBChance'][player]: row[42]+=1 + if hudImportData['street4CBDone'][player]: row[43]+=1 + + if hudImportData['foldToStreet1CBChance'][player]: row[44]+=1 + if hudImportData['foldToStreet1CBDone'][player]: row[45]+=1 + if hudImportData['foldToStreet2CBChance'][player]: row[46]+=1 + if hudImportData['foldToStreet2CBDone'][player]: row[47]+=1 + if hudImportData['foldToStreet3CBChance'][player]: row[48]+=1 + if hudImportData['foldToStreet3CBDone'][player]: row[49]+=1 + if hudImportData['foldToStreet4CBChance'][player]: row[50]+=1 + if hudImportData['foldToStreet4CBDone'][player]: row[51]+=1 + + #print "player=", player + #print "len(totalProfit)=", len(hudImportData['totalProfit']) + if hudImportData['totalProfit'][player]: + row[52]+=hudImportData['totalProfit'][player] + + if hudImportData['street1CheckCallRaiseChance'][player]: row[53]+=1 + if hudImportData['street1CheckCallRaiseDone'][player]: row[54]+=1 + if hudImportData['street2CheckCallRaiseChance'][player]: row[55]+=1 + if hudImportData['street2CheckCallRaiseDone'][player]: row[56]+=1 + if hudImportData['street3CheckCallRaiseChance'][player]: row[57]+=1 + if hudImportData['street3CheckCallRaiseDone'][player]: row[58]+=1 + if hudImportData['street4CheckCallRaiseChance'][player]: row[59]+=1 + if hudImportData['street4CheckCallRaiseDone'][player]: row[60]+=1 + + # Try to do the update first: + num = cursor.execute("""UPDATE HudCache +SET HDs=HDs+%s, street0VPI=street0VPI+%s, street0Aggr=street0Aggr+%s, + street0_3B4BChance=%s, street0_3B4BDone=%s, + street1Seen=street1Seen+%s, street2Seen=street2Seen+%s, street3Seen=street3Seen+%s, + street4Seen=street4Seen+%s, sawShowdown=sawShowdown+%s, + street1Aggr=street1Aggr+%s, street2Aggr=street2Aggr+%s, street3Aggr=street3Aggr+%s, + street4Aggr=street4Aggr+%s, otherRaisedStreet1=otherRaisedStreet1+%s, + otherRaisedStreet2=otherRaisedStreet2+%s, otherRaisedStreet3=otherRaisedStreet3+%s, + otherRaisedStreet4=otherRaisedStreet4+%s, + foldToOtherRaisedStreet1=foldToOtherRaisedStreet1+%s, foldToOtherRaisedStreet2=foldToOtherRaisedStreet2+%s, + foldToOtherRaisedStreet3=foldToOtherRaisedStreet3+%s, foldToOtherRaisedStreet4=foldToOtherRaisedStreet4+%s, + wonWhenSeenStreet1=wonWhenSeenStreet1+%s, wonAtSD=wonAtSD+%s, stealAttemptChance=stealAttemptChance+%s, + stealAttempted=stealAttempted+%s, foldBbToStealChance=foldBbToStealChance+%s, + foldedBbToSteal=foldedBbToSteal+%s, + foldSbToStealChance=foldSbToStealChance+%s, foldedSbToSteal=foldedSbToSteal+%s, + street1CBChance=street1CBChance+%s, street1CBDone=street1CBDone+%s, street2CBChance=street2CBChance+%s, + street2CBDone=street2CBDone+%s, street3CBChance=street3CBChance+%s, + street3CBDone=street3CBDone+%s, street4CBChance=street4CBChance+%s, street4CBDone=street4CBDone+%s, + foldToStreet1CBChance=foldToStreet1CBChance+%s, foldToStreet1CBDone=foldToStreet1CBDone+%s, + foldToStreet2CBChance=foldToStreet2CBChance+%s, foldToStreet2CBDone=foldToStreet2CBDone+%s, + foldToStreet3CBChance=foldToStreet3CBChance+%s, + foldToStreet3CBDone=foldToStreet3CBDone+%s, foldToStreet4CBChance=foldToStreet4CBChance+%s, + foldToStreet4CBDone=foldToStreet4CBDone+%s, totalProfit=totalProfit+%s, + street1CheckCallRaiseChance=street1CheckCallRaiseChance+%s, + street1CheckCallRaiseDone=street1CheckCallRaiseDone+%s, street2CheckCallRaiseChance=street2CheckCallRaiseChance+%s, + street2CheckCallRaiseDone=street2CheckCallRaiseDone+%s, street3CheckCallRaiseChance=street3CheckCallRaiseChance+%s, + street3CheckCallRaiseDone=street3CheckCallRaiseDone+%s, street4CheckCallRaiseChance=street4CheckCallRaiseChance+%s, + street4CheckCallRaiseDone=street4CheckCallRaiseDone+%s +WHERE gametypeId+0=%s +AND playerId=%s +AND activeSeats=%s +AND position=%s +AND tourneyTypeId+0=%s""", (row[6], row[7], row[8], row[9], row[10], + row[11], row[12], row[13], row[14], row[15], + row[16], row[17], row[18], row[19], row[20], + row[21], row[22], row[23], row[24], row[25], + row[26], row[27], row[28], row[29], row[30], + row[31], row[32], row[33], row[34], row[35], + row[36], row[37], row[38], row[39], row[40], + row[41], row[42], row[43], row[44], row[45], + row[46], row[47], row[48], row[49], row[50], + row[51], row[52], row[53], row[54], row[55], + row[56], row[57], row[58], row[59], row[60], + row[1], row[2], row[3], str(row[4]), row[5])) + # Test statusmessage to see if update worked, do insert if not + #print "storehud2, upd num =", num + if ( (backend == PGSQL and cursor.statusmessage != "UPDATE 1") + or (backend == MYSQL_INNODB and num == 0) ): + #print "playerid before insert:",row[2]," num = ", num + cursor.execute("""INSERT INTO HudCache +(gametypeId, playerId, activeSeats, position, tourneyTypeId, +HDs, street0VPI, street0Aggr, street0_3B4BChance, street0_3B4BDone, +street1Seen, street2Seen, street3Seen, street4Seen, sawShowdown, +street1Aggr, street2Aggr, street3Aggr, street4Aggr, otherRaisedStreet1, +otherRaisedStreet2, otherRaisedStreet3, otherRaisedStreet4, foldToOtherRaisedStreet1, foldToOtherRaisedStreet2, +foldToOtherRaisedStreet3, foldToOtherRaisedStreet4, wonWhenSeenStreet1, wonAtSD, stealAttemptChance, +stealAttempted, foldBbToStealChance, foldedBbToSteal, foldSbToStealChance, foldedSbToSteal, +street1CBChance, street1CBDone, street2CBChance, street2CBDone, street3CBChance, +street3CBDone, street4CBChance, street4CBDone, foldToStreet1CBChance, foldToStreet1CBDone, +foldToStreet2CBChance, foldToStreet2CBDone, foldToStreet3CBChance, foldToStreet3CBDone, foldToStreet4CBChance, +foldToStreet4CBDone, totalProfit, street1CheckCallRaiseChance, street1CheckCallRaiseDone, street2CheckCallRaiseChance, +street2CheckCallRaiseDone, street3CheckCallRaiseChance, street3CheckCallRaiseDone, street4CheckCallRaiseChance, street4CheckCallRaiseDone) +VALUES (%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s, +%s, %s, %s, %s, %s)""", (row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[11], row[12], row[13], row[14], row[15], row[16], row[17], row[18], row[19], row[20], row[21], row[22], row[23], row[24], row[25], row[26], row[27], row[28], row[29], row[30], row[31], row[32], row[33], row[34], row[35], row[36], row[37], row[38], row[39], row[40], row[41], row[42], row[43], row[44], row[45], row[46], row[47], row[48], row[49], row[50], row[51], row[52], row[53], row[54], row[55], row[56], row[57], row[58], row[59], row[60])) + #print "hopefully inserted hud data line: ", cursor.statusmessage + # message seems to be "INSERT 0 1" + else: + #print "updated(2) hud data line" + pass +# else: +# print "todo: implement storeHudCache for stud base" +#end def storeHudCache2 + def store_tourneys(cursor, tourneyTypeId, siteTourneyNo, entries, prizepool, startTime): cursor.execute("SELECT id FROM Tourneys WHERE siteTourneyNo=%s AND tourneyTypeId+0=%s", (siteTourneyNo, tourneyTypeId)) tmp=cursor.fetchone() From d14dddca909417ebc75609b157d68379c66db1a9 Mon Sep 17 00:00:00 2001 From: Matt Turnbull Date: Tue, 6 Jan 2009 23:13:12 +0000 Subject: [PATCH 5/5] Break from thread loop instead of continuing. Fixes a bug where we'd be looking up a new_hand_id of "" --- pyfpdb/HUD_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyfpdb/HUD_main.py b/pyfpdb/HUD_main.py index 12417a23..b67504ad 100755 --- a/pyfpdb/HUD_main.py +++ b/pyfpdb/HUD_main.py @@ -101,6 +101,7 @@ def read_stdin(): # This is the thread function new_hand_id = string.rstrip(new_hand_id) if new_hand_id == "": # blank line means quit destroy() + break # this thread is not always killed immediately with gtk.main_quit() # delete hud_dict entries for any HUD destroyed since last iteration for h in hud_dict.keys():