From 838c626bba1ffcf8da1d31e2d1735ed9e90ae327 Mon Sep 17 00:00:00 2001 From: "chaz@pokeit.co" Date: Fri, 31 Dec 2010 15:30:29 -0500 Subject: [PATCH 01/17] Changed the HandsActions table join index from 'handsPlayersId' to 'handId' and 'playerId'. This allows us to use executemany(), which is faster, in storeHandsPlayers when storing actions. --- pyfpdb/Database.py | 31 +++++++++++++------------------ pyfpdb/Hand.py | 8 ++++---- pyfpdb/SQL.py | 30 ++++++++++++++++-------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 8ed0e3bc..e1b097fd 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 147 +DB_VERSION = 148 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -125,7 +125,8 @@ class Database: {'tab':'Gametypes', 'col':'siteId', 'drop':0} , {'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09 #, {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped - , {'tab':'HandsActions', 'col':'handsPlayerId', 'drop':0} + , {'tab':'HandsActions', 'col':'handId', 'drop':1} + , {'tab':'HandsActions', 'col':'playerId', 'drop':1} , {'tab':'HandsActions', 'col':'actionId', 'drop':1} , {'tab':'HandsPlayers', 'col':'handId', 'drop':1} , {'tab':'HandsPlayers', 'col':'playerId', 'drop':1} @@ -150,7 +151,8 @@ class Database: , {'tab':'HandsPlayers', 'col':'handId', 'drop':0} , {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0} - , {'tab':'HandsActions', 'col':'handsPlayerId', 'drop':0} + , {'tab':'HandsActions', 'col':'handId', 'drop':0} + , {'tab':'HandsActions', 'col':'playerId', 'drop':0} , {'tab':'HandsActions', 'col':'actionId', 'drop':1} , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'playerId', 'drop':0} @@ -174,7 +176,8 @@ class Database: , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1} - , {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsActions', 'fkcol':'actionId', 'rtab':'Actions', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} @@ -184,7 +187,8 @@ class Database: {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} - , {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} + , {'fktab':'HandsActions', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsActions', 'fkcol':'actionId', 'rtab':'Actions', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} @@ -1755,7 +1759,6 @@ class Database: pp.pprint(pdata) inserts = [] - hpid = {} for p in pdata: inserts.append( (hid, pids[p], @@ -1860,17 +1863,9 @@ class Database: #print "DEBUG: inserts: %s" %inserts #print "DEBUG: q: %s" % q c = self.get_cursor() + c.executemany(q, inserts) - if self.import_options['saveActions']: - for r in inserts: - c.execute(q, r) - hpid[(r[0], r[1])] = self.get_last_insert_id(c) - else: - c.executemany(q, inserts) - - return hpid - - def storeHandsActions(self, hid, pids, hpid, adata, printdata = False): + def storeHandsActions(self, hid, pids, adata, printdata = False): #print "DEBUG: %s %s %s" %(hid, pids, adata) # This can be used to generate test data. Currently unused @@ -1881,8 +1876,8 @@ class Database: inserts = [] for a in adata: - inserts.append( (hpid[(hid, pids[adata[a]['player']])], - #self.getHandsPlayerId(self.hid, pids[adata[a]['player']]), + inserts.append( (hid, + pids[adata[a]['player']], adata[a]['street'], adata[a]['actionNo'], adata[a]['streetActionNo'], diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 405a56e5..a3a1059f 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -266,11 +266,11 @@ db: a connected Database object""" hh['seats'] = len(self.dbid_pids) self.dbid_hands = db.storeHand(hh, printdata = printtest) - self.dbid_hpid = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, - self.stats.getHandsPlayers(), printdata = printtest) + db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.stats.getHandsPlayers(), + printdata = printtest) if self.saveActions: - db.storeHandsActions(self.dbid_hands, self.dbid_pids, self.dbid_hpid, - self.stats.getHandsActions(), printdata = printtest) + db.storeHandsActions(self.dbid_hands, self.dbid_pids, self.stats.getHandsActions(), + printdata = printtest) else: log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) self.is_duplicate = True # i.e. don't update hudcache diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 6a87b433..d396b43c 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -1007,7 +1007,8 @@ class Sql: if db_server == 'mysql': self.query['createHandsActionsTable'] = """CREATE TABLE HandsActions ( id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), - handsPlayerId BIGINT UNSIGNED NOT NULL, FOREIGN KEY (handsPlayerId) REFERENCES HandsPlayers(id), + handId BIGINT UNSIGNED NOT NULL, FOREIGN KEY (handId) REFERENCES Hands(id), + playerId INT UNSIGNED NOT NULL, FOREIGN KEY (playerId) REFERENCES Players(id), street SMALLINT NOT NULL, actionNo SMALLINT NOT NULL, streetActionNo SMALLINT NOT NULL, @@ -1022,7 +1023,8 @@ class Sql: elif db_server == 'postgresql': self.query['createHandsActionsTable'] = """CREATE TABLE HandsActions ( id BIGSERIAL, PRIMARY KEY (id), - handsPlayerId BIGINT, FOREIGN KEY (handsPlayerId) REFERENCES HandsPlayers(id), + handId BIGINT NOT NULL, FOREIGN KEY (handId) REFERENCES Hands(id), + playerId INT NOT NULL, FOREIGN KEY (playerId) REFERENCES Players(id), street SMALLINT, actionNo SMALLINT, streetActionNo SMALLINT, @@ -1036,7 +1038,8 @@ class Sql: elif db_server == 'sqlite': self.query['createHandsActionsTable'] = """CREATE TABLE HandsActions ( id INTEGER PRIMARY KEY, - handsPlayerId BIGINT, + handId INT NOT NULL, + playerId INT NOT NULL, street SMALLINT, actionNo SMALLINT, streetActionNo SMALLINT, @@ -1046,9 +1049,7 @@ class Sql: amountCalled INT, numDiscarded SMALLINT, cardsDiscarded TEXT, - allIn BOOLEAN, - FOREIGN KEY (handsPlayerId) REFERENCES HandsPlayers(id), - FOREIGN KEY (actionId) REFERENCES Actions(id) ON DELETE CASCADE + allIn BOOLEAN )""" @@ -4323,7 +4324,7 @@ class Sql: self.query['handsPlayersTTypeId_joiner'] = " OR TourneysPlayersId+0=" self.query['handsPlayersTTypeId_joiner_id'] = " OR id=" - self.query['store_hand'] = """INSERT INTO Hands ( + self.query['store_hand'] = """insert into Hands ( tablename, gametypeid, sitehandno, @@ -4355,13 +4356,13 @@ class Sql: street4Pot, showdownPot ) - VALUES + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" - self.query['store_hands_players'] = """INSERT INTO HandsPlayers ( + self.query['store_hands_players'] = """insert into HandsPlayers ( handId, playerId, startCash, @@ -4458,7 +4459,7 @@ class Sql: street3Raises, street4Raises ) - VALUES ( + values ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, @@ -4480,8 +4481,9 @@ class Sql: %s, %s, %s, %s, %s )""" - self.query['store_hands_actions'] = """INSERT INTO HandsActions ( - handsPlayerId, + self.query['store_hands_actions'] = """insert into HandsActions ( + handId, + playerId, street, actionNo, streetActionNo, @@ -4493,10 +4495,10 @@ class Sql: cardsDiscarded, allIn ) - VALUES ( + values ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s + %s, %s )""" ################################ From 990e2262547142fa937dea96d43a89183b1965ad Mon Sep 17 00:00:00 2001 From: "chaz@pokeit.co" Date: Sat, 1 Jan 2011 03:35:14 -0500 Subject: [PATCH 02/17] storeSessionsCache() now adds a sessionId to the Hands table --- pyfpdb/Database.py | 69 +++++++------ pyfpdb/DerivedStats.py | 1 + pyfpdb/HUD_config.test.xml | 194 +++++++++++++++++++++++++++++++++++-- pyfpdb/Hand.py | 9 +- pyfpdb/SQL.py | 72 ++++++++------ pyfpdb/TestHandsPlayers.py | 4 +- pyfpdb/fpdb_import.py | 8 -- 7 files changed, 279 insertions(+), 78 deletions(-) mode change 100755 => 100644 pyfpdb/TestHandsPlayers.py diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index e1b097fd..9c924ff1 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 148 +DB_VERSION = 149 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -1718,10 +1718,11 @@ class Database: c.execute(q, ( p['tableName'], - p['gametypeId'], p['siteHandNo'], p['tourneyId'], - p['startTime'], + p['gametypeId'], + p['sessionId'], + p['startTime'], datetime.utcnow(), #importtime p['seats'], p['maxSeats'], @@ -2035,10 +2036,9 @@ class Database: pass def storeSessionsCache(self, pids, startTime, game, pdata): - """Update cached sessions. If update fails because no record exists, do an insert""" + """Update cached sessions. If no record exists, do an insert""" THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) - bigBet = int(Decimal(game['bb'])*200) select_sessionscache = self.sql.query['select_sessionscache'] select_sessionscache = select_sessionscache.replace('%s', self.sql.query['placeholder']) @@ -2061,6 +2061,9 @@ class Database: delete_sessions = self.sql.query['delete_sessions'] delete_sessions = delete_sessions.replace('%s', self.sql.query['placeholder']) + update_hands_sessionid = self.sql.query['update_hands_sessionid'] + update_hands_sessionid = update_hands_sessionid.replace('%s', self.sql.query['placeholder']) + #Grab playerIds using hero names in HUD_Config.xml try: # derive list of program owner's player ids @@ -2090,30 +2093,35 @@ class Database: if (game['type']=='ring'): line[0] = 1 # count ring hands if (game['type']=='tour'): line[1] = 1 # count tour hands - if (game['type']=='ring'): line[2] = pdata[p]['totalProfit'] #sum of profit - if (game['type']=='ring'): line[3] = 0 #float(Decimal(pdata[p]['totalProfit'])/Decimal(bigBet)) #sum of big bets won + if (game['type']=='ring' and game['currency']=='USD'): line[2] = pdata[p]['totalProfit'] #sum of ring profit in USD + if (game['type']=='ring' and game['currency']=='EUR'): line[3] = pdata[p]['totalProfit'] #sum of ring profit in EUR line[4] = startTime inserts.append(line) cursor = self.get_cursor() + id = None for row in inserts: threshold = [] + session_records = [] threshold.append(row[-1]-THRESHOLD) threshold.append(row[-1]+THRESHOLD) cursor.execute(select_sessionscache, threshold) - num = cursor.rowcount + for r in cursor: + if r: session_records.append(r[0]) + num = len(session_records) if (num == 1): + id = session_records[0] #grab the sessionId # Try to do the update first: #print "DEBUG: found 1 record to update" update_mid = row + row[-1:] cursor.execute(select_sessionscache_mid, update_mid[-2:]) - mid = cursor.rowcount - if (mid == 0): + mid = cursor.fetchone() + if mid: update_startend = row[-1:] + row + threshold cursor.execute(select_sessionscache_start, update_startend[-3:]) - start = cursor.rowcount - if (start == 0): + start = cursor.fetchone() + if start: #print "DEBUG:", start, " start record found. Update stats and start time" cursor.execute(update_sessionscache_end, update_startend) else: @@ -2123,37 +2131,36 @@ class Database: #print "DEBUG: update stats mid-session" cursor.execute(update_sessionscache_mid, update_mid) elif (num > 1): + session_ids = [session_records[0][0], session_records[1][0]] + session_ids.sort() # Multiple matches found - merge them into one session and update: - #print "DEBUG:", num, "matches found" - cursor.execute(merge_sessionscache, threshold) + # - Obtain the session start and end times for the new combined session + cursor.execute(merge_sessionscache, session_ids) merge = cursor.fetchone() - cursor.execute(delete_sessions, threshold) + # - Delete the old records + for id in session_ids: + cursor.execute(delete_sessions, id) + # - Insert the new updated record cursor.execute(insert_sessionscache, merge) + # - Obtain the new sessionId and write over the old ids in Hands + id = self.get_last_insert_id(cursor) #grab the sessionId + update_hands = [id] + session_ids + cursor.execute(update_hands_sessionid, update_hands) + # - Update the newly combined record in SessionsCache with data from this hand update_mid = row + row[-1:] - cursor.execute(select_sessionscache_mid, update_mid[-2:]) - mid = cursor.rowcount - if (mid == 0): - update_startend = row[-1:] + row + threshold - cursor.execute(select_sessionscache_start, update_startend[-3:]) - start = cursor.rowcount - if (start == 0): - #print "DEBUG:", start, " start record found. Update stats and start time" - cursor.execute(update_sessionscache_end, update_startend) - else: - #print "DEBUG: 1 end record found. Update stats and end time time" - cursor.execute(update_sessionscache_start, update_startend) - else: - #print "DEBUG: update stats mid-session" - cursor.execute(update_sessionscache_mid, update_mid) + cursor.execute(update_sessionscache_mid, update_mid) elif (num == 0): # No matches found, insert new session: insert = row + row[-1:] insert = insert[-2:] + insert[:-2] #print "DEBUG: No matches found. Insert record", insert cursor.execute(insert_sessionscache, insert) + id = self.get_last_insert_id(cursor) #grab the sessionId else: # Something bad happened - pass + pass + + return id def isDuplicate(self, gametypeID, siteHandNo): dup = False diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 003f5270..e069db7f 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -102,6 +102,7 @@ class DerivedStats(): self.hands['tableName'] = hand.tablename self.hands['siteHandNo'] = hand.handid self.hands['gametypeId'] = None # Leave None, handled later after checking db + self.hands['sessionId'] = None # Leave None, added later if caching sessions self.hands['startTime'] = hand.startTime # format this! self.hands['importTime'] = None self.hands['seats'] = self.countPlayers(hand) diff --git a/pyfpdb/HUD_config.test.xml b/pyfpdb/HUD_config.test.xml index 7ce84651..8352b468 100644 --- a/pyfpdb/HUD_config.test.xml +++ b/pyfpdb/HUD_config.test.xml @@ -2,7 +2,7 @@ - +