From 66e1cc3704aa98ad5beb435fb8d65dd63ebc00ba Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 19:16:22 +0000 Subject: [PATCH 01/12] This commit includes a set of updates which: * implement the new SessionsCache table - The SessionsCache table can be used to track overall or game sepecific sessions - The totalProfit field is summed by gameTypeId for cash games allowing for multiple currencies - Tournament profit (cashes - buy-ins) are also recorded in totalProfit and its grouped by tourneyId * change the sequence and methodology surrounding the import of hands - fpdb_import.py implements a unique Hand.py method for each table - Hands SessionCache and HudCache records themselves are 'cached' to allow for 'bulk insert' at EOF - import is reorganized to allow for efficient locking in multiple connection environments * changes the name of the index created by addTPlayersIndex (so that it is unique) to accommodate a bug in MySQL 5.5 TODO * A 'rebuild_sessionsCache' method is still required * Further commits are expected to fix bugs created during the porting of this code --- pyfpdb/Database.py | 641 +++++++++++++++++++++------------ pyfpdb/DerivedStats.py | 21 +- pyfpdb/FullTiltPokerSummary.py | 1 - pyfpdb/GuiTourneyImport.py | 7 +- pyfpdb/Hand.py | 123 +++---- pyfpdb/SQL.py | 251 +++++++------ pyfpdb/TourneySummary.py | 9 + pyfpdb/fpdb_import.py | 102 +++--- 8 files changed, 696 insertions(+), 459 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 3898c65d..29047a06 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 150 +DB_VERSION = 151 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -134,6 +134,9 @@ class Database: , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1} + , {'tab':'SessionsCache', 'col':'playerId', 'drop':0} + , {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'Players', 'col':'siteId', 'drop':1} #, {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} @@ -157,6 +160,9 @@ class Database: , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1} + , {'tab':'SessionsCache', 'col':'playerId', 'drop':0} + , {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'Players', 'col':'siteId', 'drop':1} , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} , {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0} @@ -182,6 +188,9 @@ class Database: , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} ] , [ # foreign keys for postgres (index 3) {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} @@ -193,6 +202,9 @@ class Database: , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} ] , [ # no foreign keys in sqlite (index 4) ] @@ -1252,6 +1264,12 @@ class Database: c.execute(self.sql.query['createBackingsTable']) c.execute(self.sql.query['createRawHands']) c.execute(self.sql.query['createRawTourneys']) + + # Create sessionscache indexes + log.debug("Creating SessionsCache indexes") + c.execute(self.sql.query['addSessionIdIndex']) + c.execute(self.sql.query['addHandsSessionIdIndex']) + c.execute(self.sql.query['addHandsGameSessionIdIndex']) # Create unique indexes: log.debug("Creating unique indexes") @@ -1713,70 +1731,64 @@ class Database: # NEWIMPORT CODE ########################### - def storeHand(self, p, printdata = False): - if printdata: - print _("######## Hands ##########") - import pprint - pp = pprint.PrettyPrinter(indent=4) - pp.pprint(p) - print _("###### End Hands ########") - - # Tablename can have odd charachers - p['tableName'] = Charset.to_db_utf8(p['tableName']) - + def storeHand(self, hdata, hbulk, doinsert = False): #stores into table hands: - q = self.sql.query['store_hand'] + # Tablename can have odd charachers + hdata['tableName'] = Charset.to_db_utf8(hdata['tableName']) + + hbulk.append( [ hdata['tableName'], + hdata['siteHandNo'], + hdata['tourneyId'], + hdata['gameTypeId'], + hdata['sessionId'], + hdata['gameSessionId'], + hdata['startTime'], + datetime.utcnow(), #importtime + hdata['seats'], + hdata['maxSeats'], + hdata['texture'], + hdata['playersVpi'], + hdata['boardcard1'], + hdata['boardcard2'], + hdata['boardcard3'], + hdata['boardcard4'], + hdata['boardcard5'], + hdata['playersAtStreet1'], + hdata['playersAtStreet2'], + hdata['playersAtStreet3'], + hdata['playersAtStreet4'], + hdata['playersAtShowdown'], + hdata['street0Raises'], + hdata['street1Raises'], + hdata['street2Raises'], + hdata['street3Raises'], + hdata['street4Raises'], + hdata['street1Pot'], + hdata['street2Pot'], + hdata['street3Pot'], + hdata['street4Pot'], + hdata['showdownPot'], + hdata['id'] + ]) - q = q.replace('%s', self.sql.query['placeholder']) + if doinsert: + for h in hbulk: + id = h.pop() + if hdata['sc'] and hdata['gsc']: + h[4] = hdata['sc'][id]['id'] + h[5] = hdata['gsc'][id]['id'] + q = self.sql.query['store_hand'] + q = q.replace('%s', self.sql.query['placeholder']) + c = self.get_cursor() + c.executemany(q, hbulk) + self.commit() + return hbulk - c = self.get_cursor() - - c.execute(q, ( - p['tableName'], - p['siteHandNo'], - p['tourneyId'], - p['gametypeId'], - p['sessionId'], - p['startTime'], - datetime.utcnow(), #importtime - p['seats'], - p['maxSeats'], - p['texture'], - p['playersVpi'], - p['boardcard1'], - p['boardcard2'], - p['boardcard3'], - p['boardcard4'], - p['boardcard5'], - p['playersAtStreet1'], - p['playersAtStreet2'], - p['playersAtStreet3'], - p['playersAtStreet4'], - p['playersAtShowdown'], - p['street0Raises'], - p['street1Raises'], - p['street2Raises'], - p['street3Raises'], - p['street4Raises'], - p['street1Pot'], - p['street2Pot'], - p['street3Pot'], - p['street4Pot'], - p['showdownPot'] - )) - return self.get_last_insert_id(c) - # def storeHand - - def storeHandsPlayers(self, hid, pids, pdata, hp_bulk = None, insert = False, printdata = False): + def storeHandsPlayers(self, hid, pids, pdata, hpbulk, doinsert = False): #print "DEBUG: %s %s %s" %(hid, pids, pdata) - if printdata: - import pprint - pp = pprint.PrettyPrinter(indent=4) - pp.pprint(pdata) - inserts = [] for p in pdata: - inserts.append( (hid, + hpbulk.append( ( hid, pids[p], pdata[p]['startCash'], pdata[p]['seatNo'], @@ -1882,27 +1894,18 @@ class Database: pdata[p]['street4Raises'] ) ) - if insert: - hp_bulk += inserts + if doinsert: q = self.sql.query['store_hands_players'] q = q.replace('%s', self.sql.query['placeholder']) c = self.get_cursor() - c.executemany(q, hp_bulk) - - return inserts + c.executemany(q, hpbulk) + return hpbulk - def storeHandsActions(self, hid, pids, adata, ha_bulk = None, insert = False, printdata = False): + def storeHandsActions(self, hid, pids, adata, habulk, doinsert = False): #print "DEBUG: %s %s %s" %(hid, pids, adata) - - # This can be used to generate test data. Currently unused - #if printdata: - # import pprint - # pp = pprint.PrettyPrinter(indent=4) - # pp.pprint(adata) - - inserts = [] + for a in adata: - inserts.append( (hid, + habulk.append( (hid, pids[adata[a]['player']], adata[a]['street'], adata[a]['actionNo'], @@ -1915,17 +1918,15 @@ class Database: adata[a]['cardsDiscarded'], adata[a]['allIn'] ) ) - - if insert: - ha_bulk += inserts + + if doinsert: q = self.sql.query['store_hands_actions'] q = q.replace('%s', self.sql.query['placeholder']) c = self.get_cursor() - c.executemany(q, ha_bulk) + c.executemany(q, habulk) + return habulk - return inserts - - def storeHudCache(self, gid, pids, starttime, pdata): + def storeHudCache(self, gid, pids, starttime, pdata, hcbulk, doinsert = False): """Update cached statistics. If update fails because no record exists, do an insert.""" tz = datetime.utcnow() - datetime.today() @@ -2042,169 +2043,359 @@ class Database: line.append(pdata[p]['street3Raises']) line.append(pdata[p]['street4Raises']) - line.append(gid) # gametypeId - line.append(pids[p]) # playerId - line.append(len(pids)) # activeSeats + hc['gametypeId'] = gid + hc['playerId'] = pids[p] + hc['activeSeats'] = len(pids) pos = {'B':'B', 'S':'S', 0:'D', 1:'C', 2:'M', 3:'M', 4:'M', 5:'E', 6:'E', 7:'E', 8:'E', 9:'E' } - line.append(pos[pdata[p]['position']]) - line.append(pdata[p]['tourneyTypeId']) - line.append(styleKey) # styleKey - inserts.append(line) - - - cursor = self.get_cursor() - - for row in inserts: - #convert all True/False values to numeric 0/1 - # needed because columns in hudcache are not BOOL they are INT - # and are being summed if an existing hudcache entry exists - # psycopg2 module does not automatically convert these to numeric. - # mantis bug #93 - for ind in range(len(row)): - if row[ind] == True: row[ind] = 1 - if row[ind] == False: row[ind] = 0 - # Try to do the update first: - num = cursor.execute(update_hudcache, row) - #print "DEBUG: values: %s" % row[-6:] - # Test statusmessage to see if update worked, do insert if not - # num is a cursor in sqlite - if ((self.backend == self.PGSQL and cursor.statusmessage != "UPDATE 1") - or (self.backend == self.MYSQL_INNODB and num == 0) - or (self.backend == self.SQLITE and num.rowcount == 0)): - #move the last 6 items in WHERE clause of row from the end of the array - # to the beginning for the INSERT statement - #print "DEBUG: using INSERT: %s" % num - row = row[-6:] + row[:-6] - num = cursor.execute(insert_hudcache, row) - #print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num - else: - #print "DEBUG: Successfully updated HudCacho using UPDATE" - pass + hc['position'] = pos[pdata[p]['position']] + hc['tourneyTypeId'] = pdata[p]['tourneyTypeId'] + hc['styleKey'] = styleKey + hc['line'] = line + hc['game'] = [hc['gametypeId'] + ,hc['playerId'] + ,hc['activeSeats'] + ,hc['position'] + ,hc['tourneyTypeId'] + ,hc['styleKey']] + hcs.append(hc) - def storeSessionsCache(self, pids, startTime, game, pdata): - """Update cached sessions. If no record exists, do an insert""" + for h in hcs: + match = False + for b in hcbulk: + if h['game']==b['game']: + b['line'] = [sum(l) for l in zip(b['line'], h['line'])] + match = True + if not match: hcbulk.append(h) + if doinsert: + inserts = [] + exists = [] + updates = [] + for hc in hcbulk: + row = hc['line'] + hc['game'] + if hc['game'] in exists: + updates.append(row) + continue + c = self.get_cursor() + num = c.execute(update_hudcache, row) + # Try to do the update first. Do insert it did not work + if ((self.backend == self.PGSQL and c.statusmessage != "UPDATE 1") + or (self.backend == self.MYSQL_INNODB and num == 0) + or (self.backend == self.SQLITE and num.rowcount == 0)): + inserts.append(hc['game'] + hc['line']) + #row = hc['game'] + hc['line'] + #num = c.execute(insert_hudcache, row) + #print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num + else: + exists.append(hc['game']) + #print "DEBUG: Successfully updated HudCacho using UPDATE" + if inserts: c.executemany(insert_hudcache, inserts) + if updates: c.executemany(update_hudcache, updates) + + return hcbulk + + def prepSessionsCache(self, hid, pids, startTime, sc, heros, doinsert = False): + """Update cached sessions. If no record exists, do an insert""" THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) - select_sessionscache = self.sql.query['select_sessionscache'] - select_sessionscache = select_sessionscache.replace('%s', self.sql.query['placeholder']) - select_sessionscache_mid = self.sql.query['select_sessionscache_mid'] - select_sessionscache_mid = select_sessionscache_mid.replace('%s', self.sql.query['placeholder']) - select_sessionscache_start = self.sql.query['select_sessionscache_start'] - select_sessionscache_start = select_sessionscache_start.replace('%s', self.sql.query['placeholder']) + select_prepSC = self.sql.query['select_prepSC'].replace('%s', self.sql.query['placeholder']) + update_Hands_sid = self.sql.query['update_Hands_sid'].replace('%s', self.sql.query['placeholder']) + update_SC_sid = self.sql.query['update_SC_sid'].replace('%s', self.sql.query['placeholder']) + update_prepSC = self.sql.query['update_prepSC'].replace('%s', self.sql.query['placeholder']) + + #print "DEBUG: %s %s %s" %(hid, pids, pdata) + hand = {} + for p, id in pids.iteritems(): + if id in heros: + hand['startTime'] = startTime.replace(tzinfo=None) + hand['ids'] = [] + + if hand: + id = [] + lower = hand['startTime']-THRESHOLD + upper = hand['startTime']+THRESHOLD + for i in range(len(sc['bk'])): + if ((lower <= sc['bk'][i]['sessionEnd']) + and (upper >= sc['bk'][i]['sessionStart'])): + if ((hand['startTime'] <= sc['bk'][i]['sessionEnd']) + and (hand['startTime'] >= sc['bk'][i]['sessionStart'])): + id.append(i) + elif hand['startTime'] < sc['bk'][i]['sessionStart']: + sc['bk'][i]['sessionStart'] = hand['startTime'] + id.append(i) + elif hand['startTime'] > sc['bk'][i]['sessionEnd']: + sc['bk'][i]['sessionEnd'] = hand['startTime'] + id.append(i) + if len(id) == 1: + id = id[0] + sc['bk'][id]['ids'].append(hid) + elif len(id) == 2: + if sc['bk'][id[0]]['startTime'] < sc['bk'][id[1]]['startTime']: + sc['bk'][id[0]]['endTime'] = sc['bk'][id[1]]['endTime'] + else: + sc['bk'][id[0]]['startTime'] = sc['bk'][id[1]]['startTime'] + sc['bk'].pop[id[1]] + id = id[0] + sc['bk'][id]['ids'].append(hid) + elif len(id) == 0: + hand['id'] = None + hand['sessionStart'] = hand['startTime'] + hand['sessionEnd'] = hand['startTime'] + id = len(sc['bk']) + hand['ids'].append(hid) + sc['bk'].append(hand) + + if doinsert: + c = self.get_cursor() + c.execute("SELECT max(sessionId) FROM SessionsCache") + id = c.fetchone()[0] + if id: sid = id + else: sid = 0 + for i in range(len(sc['bk'])): + lower = sc['bk'][i]['sessionStart'] - THRESHOLD + upper = sc['bk'][i]['sessionEnd'] + THRESHOLD + c.execute(select_prepSC, (lower, upper)) + r = self.fetchallDict(c) + num = len(r) + if (num == 1): + start, end, update = r[0]['sessionStart'], r[0]['sessionEnd'], False + if sc['bk'][i]['sessionStart'] < start: + start, update = sc['bk'][i]['sessionStart'], True + if sc['bk'][i]['sessionEnd'] > end: + end, update = sc['bk'][i]['sessionEnd'], True + if update: + c.execute(update_prepSC, [start, end, r[0]['id']]) + for h in sc['bk'][i]['ids']: + sc[h] = {'id': r[0]['id'], 'data': [start, end]} + elif (num > 1): + start, end, merge, merge_h, merge_sc = None, None, [], [], [] + sid += 1 + r.append(sc['bk'][i]) + for n in r: + if start: + if start > n['sessionStart']: + start = n['sessionStart'] + else: start = n['sessionStart'] + if end: + if end < n['sessionEnd']: + end = n['sessionEnd'] + else: end = n['sessionEnd'] + for n in r: + if n['id']: + if n['id'] in merge: continue + merge.append(n['id']) + merge_h.append([sid, n['id']]) + merge_sc.append([start, end, sid, n['id']]) + c.executemany(update_Hands_sid, merge_h) + c.executemany(update_SC_sid, merge_sc) + for k, v in sc.iteritems(): + if k!='bk' and v['id'] in merge: + sc[k]['id'] = sid + for h in sc['bk'][i]['ids']: + sc[h] = {'id': sid, 'data': [start, end]} + elif (num == 0): + sid += 1 + start = sc['bk'][i]['sessionStart'] + end = sc['bk'][i]['sessionEnd'] + for h in sc['bk'][i]['ids']: + sc[h] = {'id': sid, 'data': [start, end]} + return sc - update_sessionscache_mid = self.sql.query['update_sessionscache_mid'] - update_sessionscache_mid = update_sessionscache_mid.replace('%s', self.sql.query['placeholder']) - update_sessionscache_start = self.sql.query['update_sessionscache_start'] - update_sessionscache_start = update_sessionscache_start.replace('%s', self.sql.query['placeholder']) - update_sessionscache_end = self.sql.query['update_sessionscache_end'] - update_sessionscache_end = update_sessionscache_end.replace('%s', self.sql.query['placeholder']) + def storeSessionsCache(self, hid, pids, startTime, game, gid, pdata, sc, gsc, tz, heros, doinsert = False): + """Update cached sessions. If no record exists, do an insert""" + THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) + local = startTime + timedelta(hours=int(tz)) + date = "d%02d%02d%02d" % (local.year - 2000, local.month, local.day) - insert_sessionscache = self.sql.query['insert_sessionscache'] - insert_sessionscache = insert_sessionscache.replace('%s', self.sql.query['placeholder']) - merge_sessionscache = self.sql.query['merge_sessionscache'] - merge_sessionscache = merge_sessionscache.replace('%s', self.sql.query['placeholder']) - delete_sessions = self.sql.query['delete_sessions'] - delete_sessions = delete_sessions.replace('%s', self.sql.query['placeholder']) + select_SC = self.sql.query['select_SC'].replace('%s', self.sql.query['placeholder']) + update_SC = self.sql.query['update_SC'].replace('%s', self.sql.query['placeholder']) + insert_SC = self.sql.query['insert_SC'].replace('%s', self.sql.query['placeholder']) + delete_SC = self.sql.query['delete_SC'].replace('%s', self.sql.query['placeholder']) + update_Hands_gsid = self.sql.query['update_Hands_gsid'].replace('%s', self.sql.query['placeholder']) + + #print "DEBUG: %s %s %s" %(hid, pids, pdata) + hand = {} + for p, id in pids.iteritems(): + if id in heros: + hand['hands'] = 0 + hand['totalProfit'] = 0 + hand['playerId'] = id + hand['gametypeId'] = None + hand['date'] = date + hand['startTime'] = startTime.replace(tzinfo=None) + hand['hid'] = hid + hand['tourneys'] = 0 + hand['tourneyTypeId'] = None + hand['ids'] = [] + if (game['type']=='summary'): + hand['type'] = 'tour' + hand['tourneys'] = 1 + hand['tourneyTypeId'] = pdata.tourneyTypeId + hand['totalProfit'] = pdata.winnings[p] - (pdata.buyin + pdata.fee) + elif (game['type']=='ring'): + hand['type'] = game['type'] + hand['hands'] = 1 + hand['gametypeId'] = gid + hand['totalProfit'] = pdata[p]['totalProfit'] + elif (game['type']=='tour'): + hand['type'] = game['type'] + hand['hands'] = 1 + hand['tourneyTypeId'] = pdata[p]['tourneyTypeId'] - update_hands_sessionid = self.sql.query['update_hands_sessionid'] - update_hands_sessionid = update_hands_sessionid.replace('%s', self.sql.query['placeholder']) + if hand: + id = [] + lower = hand['startTime']-THRESHOLD + upper = hand['startTime']+THRESHOLD + for i in range(len(gsc['bk'])): + if ((hand['date'] == gsc['bk'][i]['date']) + and (hand['gametypeId'] == gsc['bk'][i]['gametypeId']) + and (hand['playerId'] == gsc['bk'][i]['playerId']) + and (hand['tourneyTypeId'] == gsc['bk'][i]['tourneyTypeId'])): + if ((lower <= gsc['bk'][i]['gameEnd']) + and (upper >= gsc['bk'][i]['gameStart'])): + if ((hand['startTime'] <= gsc['bk'][i]['gameEnd']) + and (hand['startTime'] >= gsc['bk'][i]['gameStart'])): + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + elif hand['startTime'] < gsc['bk'][i]['gameStart']: + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + gsc['bk'][i]['gameStart'] = hand['startTime'] + elif hand['startTime'] > gsc['bk'][i]['gameEnd']: + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + gsc['bk'][i]['gameEnd'] = hand['startTime'] + id.append(i) + if len(id) == 1: + gsc['bk'][id[0]]['ids'].append(hid) + elif len(id) == 2: + if gsc['bk'][id[0]]['gameStart'] < gsc['bk'][id[1]]['gameStart']: + gsc['bk'][id[0]]['gameEnd'] = gsc['bk'][id[1]]['gameEnd'] + else: gsc['bk'][id[0]]['gameStart'] = gsc['bk'][id[1]]['gameStart'] + gsc['bk'][id[0]]['hands'] += hand['hands'] + gsc['bk'][id[0]]['tourneys'] += hand['tourneys'] + gsc['bk'][id[0]]['totalProfit'] += hand['totalProfit'] + gsc['bk'].pop[id[1]] + gsc['bk'][id[0]]['ids'].append(hid) + elif len(id) == 0: + hand['gameStart'] = hand['startTime'] + hand['gameEnd'] = hand['startTime'] + id = len(gsc['bk']) + hand['ids'].append(hid) + gsc['bk'].append(hand) + if doinsert: + c = self.get_cursor() + for i in range(len(gsc['bk'])): + hid = gsc['bk'][i]['hid'] + sid, start, end = sc[hid]['id'], sc[hid]['data'][0], sc[hid]['data'][1] + lower = gsc['bk'][i]['gameStart'] - THRESHOLD + upper = gsc['bk'][i]['gameEnd'] + THRESHOLD + game = [gsc['bk'][i]['date'] + ,gsc['bk'][i]['type'] + ,gsc['bk'][i]['gametypeId'] + ,gsc['bk'][i]['tourneyTypeId'] + ,gsc['bk'][i]['playerId']] + row = [lower, upper] + game + c.execute(select_SC, row) + r = self.fetchallDict(c) + num = len(r) + if (num == 1): + gstart, gend = r[0]['gameStart'], r[0]['gameEnd'] + if gsc['bk'][i]['gameStart'] < gstart: + gstart = gsc['bk'][i]['gameStart'] + if gsc['bk'][i]['gameEnd'] > gend: + gend = gsc['bk'][i]['gameEnd'] + row = [start, end, gstart, gend + ,gsc['bk'][i]['hands'] + ,gsc['bk'][i]['tourneys'] + ,gsc['bk'][i]['totalProfit'] + ,r[0]['id']] + c.execute(update_SC, row) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': r[0]['id']} + elif (num > 1): + gstart, gend, hands, tourneys, totalProfit, delete, merge = None, None, 0, 0, 0, [], [] + for n in r: delete.append(n['id']) + delete.sort() + for d in delete: c.execute(delete_SC, d) + r.append(gsc['bk'][i]) + for n in r: + if gstart: + if gstart > n['gameStart']: + gstart = n['gameStart'] + else: gstart = n['gameStart'] + if gend: + if gend < n['gameEnd']: + gend = n['gameEnd'] + else: gend = n['gameEnd'] + hands += n['hands'] + tourneys += n['tourneys'] + totalProfit += n['totalProfit'] + row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit] + c.execute(insert_SC, row) + gsid = self.get_last_insert_id(c) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid} + for m in delete: merge.append([gsid, m]) + c.executemany(update_Hands_gsid, merge) + elif (num == 0): + gstart = gsc['bk'][i]['gameStart'] + gend = gsc['bk'][i]['gameEnd'] + hands = gsc['bk'][i]['hands'] + tourneys = gsc['bk'][i]['tourneys'] + totalProfit = gsc['bk'][i]['totalProfit'] + row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit] + c.execute(insert_SC, row) + gsid = self.get_last_insert_id(c) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid} + else: + # Something bad happened + pass + self.commit() + + return gsc + + def getHeroIds(self, pids, sitename): #Grab playerIds using hero names in HUD_Config.xml try: # derive list of program owner's player ids - self.hero = {} # name of program owner indexed by site id - self.hero_ids = [] + hero = {} # name of program owner indexed by site id + hero_ids = [] # make sure at least two values in list # so that tuple generation creates doesn't use # () or (1,) style for site in self.config.get_supported_sites(): - result = self.get_site_id(site) - if result: - site_id = result[0][0] - self.hero[site_id] = self.config.supported_sites[site].screen_name - p_id = self.get_player_id(self.config, site, self.hero[site_id]) - if p_id: - self.hero_ids.append(int(p_id)) - + hero = self.config.supported_sites[site].screen_name + for n, v in pids.iteritems(): + if n == hero and sitename == site: + hero_ids.append(v) + except: err = traceback.extract_tb(sys.exc_info()[2])[-1] - print _("Error aquiring hero ids:"), str(sys.exc_value) - print err - - inserts = [] - for p in pdata: - if pids[p] in self.hero_ids: - line = [0]*5 + #print _("Error aquiring hero ids:"), str(sys.exc_value) + return hero_ids - if (game['type']=='ring'): line[0] = 1 # count ring hands - if (game['type']=='tour'): line[1] = 1 # count tour hands - if (game['type']=='ring' and game['currency']=='USD'): line[2] = pdata[p]['totalProfit'] #sum of ring profit in USD - if (game['type']=='ring' and game['currency']=='EUR'): line[3] = pdata[p]['totalProfit'] #sum of ring profit in EUR - line[4] = startTime - inserts.append(line) - - cursor = self.get_cursor() - id = None - - for row in inserts: - threshold = [] - threshold.append(row[-1]-THRESHOLD) - threshold.append(row[-1]+THRESHOLD) - cursor.execute(select_sessionscache, threshold) - session_records = cursor.fetchall() - num = len(session_records) - if (num == 1): - id = session_records[0][0] #grab the sessionId - # Try to do the update first: - #print "DEBUG: found 1 record to update" - update_mid = row + row[-1:] - cursor.execute(select_sessionscache_mid, update_mid[-2:]) - mid = len(cursor.fetchall()) - if (mid == 0): - update_startend = row[-1:] + row + threshold - cursor.execute(select_sessionscache_start, update_startend[-3:]) - start = len(cursor.fetchall()) - if (start == 0): - #print "DEBUG:", start, " start record found. Update stats and start time" - cursor.execute(update_sessionscache_end, update_startend) - else: - #print "DEBUG: 1 end record found. Update stats and end time time" - cursor.execute(update_sessionscache_start, update_startend) - else: - #print "DEBUG: update stats mid-session" - cursor.execute(update_sessionscache_mid, update_mid) - elif (num > 1): - session_ids = [session_records[0][0], session_records[1][0]] - session_ids.sort() - # Multiple matches found - merge them into one session and update: - # - Obtain the session start and end times for the new combined session - cursor.execute(merge_sessionscache, session_ids) - merge = cursor.fetchone() - # - Delete the old records - for id in session_ids: - cursor.execute(delete_sessions, id) - # - Insert the new updated record - cursor.execute(insert_sessionscache, merge) - # - Obtain the new sessionId and write over the old ids in Hands - id = self.get_last_insert_id(cursor) #grab the sessionId - update_hands = [id] + session_ids - cursor.execute(update_hands_sessionid, update_hands) - # - Update the newly combined record in SessionsCache with data from this hand - update_mid = row + row[-1:] - cursor.execute(update_sessionscache_mid, update_mid) - elif (num == 0): - # No matches found, insert new session: - insert = row + row[-1:] - insert = insert[-2:] + insert[:-2] - #print "DEBUG: No matches found. Insert record", insert - cursor.execute(insert_sessionscache, insert) - id = self.get_last_insert_id(cursor) #grab the sessionId - else: - # Something bad happened - pass - - return id + def fetchallDict(self, cursor): + data = cursor.fetchall() + if not data: return [] + desc = cursor.description + results = [0]*len(data) + for i in range(len(data)): + results[i] = {} + for n in range(len(desc)): + name = desc[n][0] + results[i][name] = data[i][n] + return results + + def nextHandId(self): + c = self.get_cursor() + c.execute("SELECT max(id) FROM Hands") + id = c.fetchone()[0] + if not id: id = 0 + id += 1 + return id def isDuplicate(self, gametypeID, siteHandNo): dup = False diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 262a3fee..3b955929 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -119,16 +119,17 @@ class DerivedStats(): return self.handsactions def assembleHands(self, hand): - self.hands['tableName'] = hand.tablename - self.hands['siteHandNo'] = hand.handid - self.hands['gametypeId'] = None # Leave None, handled later after checking db - self.hands['sessionId'] = None # Leave None, added later if caching sessions - self.hands['startTime'] = hand.startTime # format this! - self.hands['importTime'] = None - self.hands['seats'] = self.countPlayers(hand) - self.hands['maxSeats'] = hand.maxseats - self.hands['texture'] = None # No calculation done for this yet. - self.hands['tourneyId'] = hand.tourneyId + self.hands['tableName'] = hand.tablename + self.hands['siteHandNo'] = hand.handid + self.hands['gametypeId'] = None # Leave None, handled later after checking db + self.hands['sessionId'] = None # Leave None, added later if caching sessions + self.hands['gameSessionId'] = None # Leave None, added later if caching sessions + self.hands['startTime'] = hand.startTime # format this! + self.hands['importTime'] = None + self.hands['seats'] = self.countPlayers(hand) + self.hands['maxSeats'] = hand.maxseats + self.hands['texture'] = None # No calculation done for this yet. + self.hands['tourneyId'] = hand.tourneyId # This (i think...) is correct for both stud and flop games, as hand.board['street'] disappears, and # those values remain default in stud. diff --git a/pyfpdb/FullTiltPokerSummary.py b/pyfpdb/FullTiltPokerSummary.py index 98169393..c06042d0 100644 --- a/pyfpdb/FullTiltPokerSummary.py +++ b/pyfpdb/FullTiltPokerSummary.py @@ -25,7 +25,6 @@ import datetime from Exceptions import FpdbParseError from HandHistoryConverter import * -import PokerStarsToFpdb from TourneySummary import * class FullTiltPokerSummary(TourneySummary): diff --git a/pyfpdb/GuiTourneyImport.py b/pyfpdb/GuiTourneyImport.py index 0bc71f3d..c6f0bb48 100755 --- a/pyfpdb/GuiTourneyImport.py +++ b/pyfpdb/GuiTourneyImport.py @@ -223,13 +223,18 @@ class SummaryImporter: print "Found %s summaries" %(len(summaryTexts)) errors = 0 imported = 0 + ####Lock Placeholder#### for j, summaryText in enumerate(summaryTexts, start=1): + sc, gsc = {'bk': []}, {'bk': []} + doinsert = len(summaryTexts)==j try: - conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") + conv = obj(db=self.database, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") + sc, gsc = conv.updateSessionsCache(sc, gsc, self.tz, doinsert) except FpdbParseError, e: errors += 1 print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts)) imported = j + ####Lock Placeholder#### return (imported - errors, errors) def clearFileList(self): diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 01b5c70e..f86e5df9 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -57,6 +57,7 @@ class Hand(object): #log.debug( _("Hand.init(): handText is ") + str(handText) ) self.config = config self.saveActions = self.config.get_import_parameters().get('saveActions') + self.callHud = self.config.get_import_parameters().get("callFpdbHud") self.cacheSessions = self.config.get_import_parameters().get("cacheSessions") #log = Configuration.get_logger("logging.conf", "db", log_dir=self.config.dir_log) self.sitename = sitename @@ -227,83 +228,77 @@ dealt whether they were seen in a 'dealt to' line self.holecards[street][player] = [open, closed] - def prepInsert(self, db, printtest = False): + def prepInsert(self, db): ##### # Players, Gametypes, TourneyTypes are all shared functions that are needed for additional tables # These functions are intended for prep insert eventually ##### - # Players - base playerid and siteid tuple self.dbid_pids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId) - - #Gametypes - hilo = "h" - if self.gametype['category'] in ['studhilo', 'omahahilo']: - hilo = "s" - elif self.gametype['category'] in ['razz','27_3draw','badugi', '27_1draw']: - hilo = "l" - - self.gametyperow = (self.siteId, self.gametype['currency'], self.gametype['type'], self.gametype['base'], - self.gametype['category'], self.gametype['limitType'], hilo, - int(Decimal(self.gametype['sb'])*100), int(Decimal(self.gametype['bb'])*100), - int(Decimal(self.gametype['bb'])*100), int(Decimal(self.gametype['bb'])*200)) - # Note: the above data is calculated in db.getGameTypeId - # Only being calculated above so we can grab the testdata - self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype, printdata = printtest) - + self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype) + if self.tourNo!=None: self.tourneyTypeId = db.createTourneyType(self) - db.commit() self.tourneyId = db.createOrUpdateTourney(self, "HHC") - db.commit() self.tourneysPlayersIds = db.createOrUpdateTourneysPlayers(self, "HHC") - db.commit() - #end def prepInsert - - def insert(self, db, hp_data = None, ha_data = None, insert_data=False, printtest = False): - """ Function to insert Hand into database -Should not commit, and do minimal selects. Callers may want to cache commits -db: a connected Database object""" - - + #db.commit() #commit these transactions' + + def assembleHand(self): self.stats.getStats(self) - - ##### - # End prep functions - ##### - hh = self.stats.getHands() - hp_inserts, ha_inserts = [], [] - - if not db.isDuplicate(self.dbid_gt, hh['siteHandNo']): - # Hands - Summary information of hand indexed by handId - gameinfo - hh['gametypeId'] = self.dbid_gt - # seats TINYINT NOT NULL, - hh['seats'] = len(self.dbid_pids) - - hp = self.stats.getHandsPlayers() - - if self.cacheSessions: - hh['sessionId'] = db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, hp) - - self.dbid_hands = db.storeHand(hh, printdata = printtest) - - hp_inserts = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, hp, - insert=insert_data, hp_bulk = hp_data, printdata = printtest) - - if self.saveActions: - ha_inserts = db.storeHandsActions(self.dbid_hands, self.dbid_pids, self.stats.getHandsActions(), - insert=insert_data, ha_bulk = ha_data, printdata = printtest) - else: - log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) + self.hands = self.stats.getHands() + self.handsplayers = self.stats.getHandsPlayers() + + def getHandId(self, db, id): + if db.isDuplicate(self.dbid_gt, self.hands['siteHandNo']): + #log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) self.is_duplicate = True # i.e. don't update hudcache - raise FpdbHandDuplicate(hh['siteHandNo']) - - return hp_inserts, ha_inserts + next = id + raise FpdbHandDuplicate(self.hands['siteHandNo']) + else: + self.dbid_hands = id + self.hands['id'] = self.dbid_hands + next = id +1 + return next - def updateHudCache(self, db): - db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.stats.getHandsPlayers()) + def insertHands(self, db, hbulk, doinsert = False): + """ Function to insert Hand into database + Should not commit, and do minimal selects. Callers may want to cache commits + db: a connected Database object""" + self.hands['gameTypeId'] = self.dbid_gt + self.hands['seats'] = len(self.dbid_pids) + hbulk = db.storeHand(self.hands, hbulk, doinsert) + return hbulk + + def insertHandsPlayers(self, db, hpbulk, doinsert = False): + """ Function to inserts HandsPlayers into database""" + hpbulk = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.handsplayers, hpbulk, doinsert) + return hpbulk + + def insertHandsActions(self, db, habulk, doinsert = False): + """ Function to inserts HandsActions into database""" + handsactions = self.stats.getHandsActions() + habulk = db.storeHandsActions(self.dbid_hands, self.dbid_pids, handsactions, habulk, doinsert) + return habulk + + def updateHudCache(self, db, hcbulk, doinsert = False): + """ Function to update the HudCache""" + if self.callHud: + hcbulk = db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.handsplayers, hcbulk, doinsert) + return hcbulk - def updateSessionsCache(self, db): - db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, self.stats.getHandsPlayers()) + def updateSessionsCache(self, db, sc, gsc, tz, doinsert = False): + """ Function to update the SessionsCache""" + if self.cacheSessions: + self.heros = db.getHeroIds(self.dbid_pids, self.sitename) + sc = db.prepSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, sc, self.heros, doinsert) + gsc = db.storeSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, self.gametype + ,self.dbid_gt, self.handsplayers, sc, gsc, tz, self.heros, doinsert) + if doinsert: + self.hands['sc'] = sc + self.hands['gsc'] = gsc + else: + self.hands['sc'] = None + self.hands['gsc'] = None + return sc, gsc def select(self, db, handId): """ Function to create Hand object from database """ diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 6ca016f3..f7f84a98 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -346,7 +346,8 @@ class Sql: siteHandNo BIGINT NOT NULL, tourneyId INT UNSIGNED, gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), - sessionId INT UNSIGNED, + sessionId INT UNSIGNED, + gameSessionId INT UNSIGNED, startTime DATETIME NOT NULL, importTime DATETIME NOT NULL, seats TINYINT NOT NULL, @@ -385,6 +386,7 @@ class Sql: tourneyId INT, gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), sessionId INT, + gameSessionId INT, startTime timestamp without time zone NOT NULL, importTime timestamp without time zone NOT NULL, seats SMALLINT NOT NULL, @@ -422,6 +424,7 @@ class Sql: tourneyId INT, gametypeId INT NOT NULL, sessionId INT, + gameSessionId INT, startTime REAL NOT NULL, importTime REAL NOT NULL, seats INT NOT NULL, @@ -1430,32 +1433,61 @@ class Sql: id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), sessionStart DATETIME NOT NULL, sessionEnd DATETIME NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) - - ENGINE=INNODB""" + gameStart DATETIME NOT NULL, + gameEnd DATETIME NOT NULL, + sessionId BIGINT, + date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type char(7) NOT NULL, + gametypeId SMALLINT UNSIGNED, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), + tourneyTypeId SMALLINT UNSIGNED, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), + playerId INT UNSIGNED NOT NULL, FOREIGN KEY (playerId) REFERENCES Players(id), + hands INT NOT NULL, + tourneys INT NOT NULL, + totalProfit INT) + ENGINE=INNODB + """ + elif db_server == 'postgresql': self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( id BIGSERIAL, PRIMARY KEY (id), sessionStart REAL NOT NULL, sessionEnd REAL NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) + gameStart REAL NOT NULL, + gameEnd REAL NOT NULL, + sessionId INT, + date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type char(7), + gametypeId INT, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), + tourneyTypeId INT, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), + playerId INT, FOREIGN KEY (playerId) REFERENCES Players(id), + hands INT, + tourneys INT, + totalProfit INT) """ + elif db_server == 'sqlite': self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( id INTEGER PRIMARY KEY, sessionStart REAL NOT NULL, sessionEnd REAL NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) + gameStart REAL NOT NULL, + gameEnd REAL NOT NULL, + sessionId INT, + date TEXT NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type TEXT, + gametypeId INT, + tourneyTypeId INT, + playerId INT, + hands INT, + tourneys INT, + totalProfit INT) """ + + self.query['addSessionIdIndex'] = """CREATE INDEX index_SessionId ON SessionsCache (sessionId)""" + + self.query['addHandsSessionIdIndex'] = """CREATE INDEX index_handsSessionId ON Hands (sessionId)""" + + self.query['addHandsGameSessionIdIndex'] = """CREATE INDEX index_handsGameSessionId ON Hands (gameSessionId)""" if db_server == 'mysql': self.query['addTourneyIndex'] = """ALTER TABLE Tourneys ADD UNIQUE INDEX siteTourneyNo(siteTourneyNo, tourneyTypeId)""" @@ -1479,7 +1511,7 @@ class Sql: self.query['addPlayersIndex'] = """CREATE UNIQUE INDEX name ON Players (name, siteId)""" if db_server == 'mysql': - self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX tourneyId(tourneyId, playerId)""" + self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX _tourneyId(tourneyId, playerId)""" elif db_server == 'postgresql': self.query['addTPlayersIndex'] = """CREATE UNIQUE INDEX tourneyId ON TourneysPlayers (tourneyId, playerId)""" elif db_server == 'sqlite': @@ -4142,7 +4174,7 @@ class Sql: """ self.query['insert_hudcache'] = """ - INSERT INTO HudCache ( + insert into HudCache ( gametypeId, playerId, activeSeats, @@ -4237,7 +4269,7 @@ class Sql: street2Raises, street3Raises, street4Raises) - VALUES (%s, %s, %s, %s, %s, + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, @@ -4364,95 +4396,97 @@ class Sql: # Queries to rebuild/modify sessionscache #################################### - self.query['select_sessionscache'] = """ - SELECT id, - sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['select_sessionscache_mid'] = """ - SELECT sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['select_sessionscache_start'] = """ - SELECT sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionStart>%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['update_sessionscache_mid'] = """ - UPDATE SessionsCache SET - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionStart<=%s - AND sessionEnd>=%s""" - - self.query['update_sessionscache_start'] = """ - UPDATE SessionsCache SET - sessionStart=%s, - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionStart>%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['update_sessionscache_end'] = """ - UPDATE SessionsCache SET - sessionEnd=%s, - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionEnd<%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['insert_sessionscache'] = """ - INSERT INTO SessionsCache ( - sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR) - VALUES (%s, %s, %s, %s, %s, %s)""" - - self.query['merge_sessionscache'] = """ - SELECT min(sessionStart), max(sessionEnd), sum(ringHDs), sum(tourHDs), sum(ringProfitUSD), sum(ringProfitEUR) - FROM SessionsCache - WHERE (case when id=%s or id=%s then 1 else 0 end)=1""" - - self.query['delete_sessions'] = """ - DELETE FROM SessionsCache - WHERE id=%s""" - - self.query['update_hands_sessionid'] = """ - UPDATE Hands SET - sessionId=%s - WHERE (case when sessionId=%s or sessionId=%s then 1 else 0 end)=1""" + self.query['select_prepSC'] = """ + SELECT sessionId as id, + sessionStart, + sessionEnd, + count(sessionId) as count + FROM SessionsCache + WHERE sessionEnd>=%s + AND sessionStart<=%s + GROUP BY sessionId, sessionStart, sessionEnd""" + + self.query['update_prepSC'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s + WHERE sessionId=%s""" + + self.query['update_SC'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s, + gameStart=%s, + gameEnd=%s, + hands=hands+%s, + tourneys=tourneys+%s, + totalProfit=totalProfit+%s + WHERE id=%s""" + + self.query['select_SC'] = """ + SELECT id, + sessionStart, + sessionEnd, + gameStart, + gameEnd, + sessionId, + date, + type, + gametypeId, + tourneyTypeId, + playerId, + hands, + tourneys, + totalProfit + FROM SessionsCache + WHERE gameEnd>=%s + AND gameStart<=%s + AND date=%s + AND type=%s + AND (case when gametypeId is NULL then 1 else + (case when gametypeId=%s then 1 else 0 end) end)=1 + AND (case when tourneyTypeId is NULL then 1 else + (case when tourneyTypeId=%s then 1 else 0 end) end)=1 + AND playerId=%s""" + + self.query['insert_SC'] = """ + insert into SessionsCache ( + sessionStart, + sessionEnd, + gameStart, + gameEnd, + sessionId, + date, + type, + gametypeId, + tourneyTypeId, + playerId, + hands, + tourneys, + totalProfit) + values (%s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s)""" + + self.query['update_Hands_gsid'] = """ + UPDATE Hands SET + gameSessionId=%s + WHERE gameSessionId=%s""" + + self.query['update_Hands_sid'] = """ + UPDATE Hands SET + sessionId=%s + WHERE sessionId=%s""" + + self.query['update_SC_sid'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s, + sessionId=%s + WHERE sessionId=%s""" + + self.query['delete_SC'] = """ + DELETE FROM SessionsCache + WHERE id=%s""" #################################### # Database management queries @@ -4652,6 +4686,7 @@ class Sql: tourneyId, gametypeid, sessionId, + gameSessionId, startTime, importtime, seats, @@ -4682,7 +4717,7 @@ class Sql: values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, %s, %s)""" + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" self.query['store_hands_players'] = """insert into HandsPlayers ( diff --git a/pyfpdb/TourneySummary.py b/pyfpdb/TourneySummary.py index 94055b92..0483755f 100644 --- a/pyfpdb/TourneySummary.py +++ b/pyfpdb/TourneySummary.py @@ -287,6 +287,15 @@ winnings (int) the money the player ended the tourney with (can be 0, or -1 i if player not in [p[1] for p in self.players]: print "checkPlayerExists", player, "fail" raise FpdbParseError + + def updateSessionsCache(self, sc, gsc, tz, doinsert): + self.heros = self.db.getHeroIds(self.dbid_pids, self.siteName) + sc = self.db.prepSessionsCache(self.tourNo, self.dbid_pids, self.startTime, sc , self.heros, doinsert) + + + gsc = self.db.storeSessionsCache(self.tourNo, self.dbid_pids, self.startTime, {'type': 'summary'} + , None, self, sc, gsc, tz, self.heros, doinsert) + return sc, gsc def writeSummary(self, fh=sys.__stdout__): print >>fh, "Override me" diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index df45bc63..c59e9bf8 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -242,7 +242,7 @@ class Importer: #print "dropInd =", self.settings['dropIndexes'], " dropHudCache =", self.settings['dropHudCache'] if self.settings['threads'] <= 0: - (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, None) + (totstored, totdups, totpartial, toterrors) = self.importFiles(None) else: # create queue (will probably change to deque at some point): self.writeq = Queue.Queue( self.settings['writeQSize'] ) @@ -254,7 +254,7 @@ class Importer: t.setDaemon(True) t.start() # read hands and write to q: - (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, self.writeq) + (totstored, totdups, totpartial, toterrors) = self.importFiles(self.writeq) if self.writeq.empty(): print _("writers finished already") @@ -286,7 +286,7 @@ class Importer: return (totstored, totdups, totpartial, toterrors, endtime-starttime) # end def runImport - def importFiles(self, db, q): + def importFiles(self, q): """"Read filenames in self.filelist and pass to import_file_dict(). Uses a separate database connection if created as a thread (caller passes None or no param as db).""" @@ -304,7 +304,7 @@ class Importer: ProgressDialog.progress_update() - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(db, file + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file ,self.filelist[file][0], self.filelist[file][1], q) totstored += stored totdups += duplicates @@ -395,7 +395,7 @@ class Importer: self.caller.addText("\n"+os.path.basename(file)) except KeyError: # TODO: What error happens here? pass - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1], None) try: if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime)) @@ -426,68 +426,70 @@ class Importer: #rulog.close() # This is now an internal function that should not be called directly. - def import_file_dict(self, db, file, site, filter, q=None): - #print "import_file_dict" + def import_file_dict(self, file, site, filter, q=None): if os.path.isdir(file): self.addToDirList[file] = [site] + [filter] return (0,0,0,0,0) - conv = None (stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, time()) - # sc: is there any need to decode this? maybe easier to skip it than guess at the encoding? - #file = file.decode("utf-8") #(Configuration.LOCALE_ENCODING) - # Load filter, process file, pass returned filename to import_fpdb_file if self.settings['threads'] > 0 and self.writeq is not None: - log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")") - else: - log.info(_("Converting %s") % file) - + log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")") + else: log.info(_("Converting %s") % file) + filter_name = filter.replace("ToFpdb", "") - mod = __import__(filter) obj = getattr(mod, filter_name, None) if callable(obj): - idx = 0 - if file in self.pos_in_file: - idx = self.pos_in_file[file] - else: - self.pos_in_file[file] = 0 - hhc = obj( self.config, in_path = file, index = idx, starsArchive = self.settings['starsArchive'], ftpArchive = self.settings['ftpArchive'], sitename = site ) + + if file in self.pos_in_file: idx = self.pos_in_file[file] + else: self.pos_in_file[file], idx = 0, 0 + + hhc = obj( self.config, in_path = file, index = idx + ,starsArchive = self.settings['starsArchive'] + ,ftpArchive = self.settings['ftpArchive'] + ,sitename = site ) + if hhc.getStatus(): handlist = hhc.getProcessedHands() self.pos_in_file[file] = hhc.getLastCharacterRead() - to_hud = [] - hp_bulk = [] - ha_bulk = [] - i = 0 - + (hbulk, hpbulk, habulk, hcbulk, phands, ihands) = ([], [], [], [], [], []) + sc, gsc = {'bk': []}, {'bk': []} + + ####Lock Placeholder#### for hand in handlist: - i += 1 - if hand is not None: - hand.prepInsert(self.database, printtest = self.settings['testData']) - try: - hp_inserts, ha_inserts = hand.insert(self.database, hp_data = hp_bulk, - ha_data = ha_bulk, insert_data = len(handlist)==i, - printtest = self.settings['testData']) - hp_bulk += hp_inserts - ha_bulk += ha_inserts - except Exceptions.FpdbHandDuplicate: - duplicates += 1 - else: - if self.callHud and hand.dbid_hands != 0: - to_hud.append(hand.dbid_hands) - else: # TODO: Treat empty as an error, or just ignore? - log.error(_("Hand processed but empty")) - - # Call hudcache update if not in bulk import mode - # FIXME: Need to test for bulk import that isn't rebuilding the cache - if self.callHud: - for hand in handlist: - if hand is not None and not hand.is_duplicate: - hand.updateHudCache(self.database) + hand.prepInsert(self.database) + self.database.commit() + phands.append(hand) + ####Lock Placeholder#### + + for hand in phands: + hand.assembleHand() + + ####Lock Placeholder#### + id = self.database.nextHandId() + for i in range(len(phands)): + doinsert = len(phands)==i+1 + hand = phands[i] + try: + id = hand.getHandId(self.database, id) + sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, self.tz, doinsert) + hbulk = hand.insertHands(self.database, hbulk, doinsert) + hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) + ihands.append(hand) + to_hud.append(id) + except Exceptions.FpdbHandDuplicate: + duplicates += 1 + self.database.commit() + ####Lock Placeholder#### + + for i in range(len(ihands)): + doinsert = len(ihands)==i+1 + hand = ihands[i] + hpbulk = hand.insertHandsPlayers(self.database, hpbulk, doinsert) + habulk = hand.insertHandsActions(self.database, habulk, doinsert) self.database.commit() #pipe the Hands.id out to the HUD From d0ae2a155d0a5a4c0b7b122679b7cf1db69e233a Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 20:43:09 +0000 Subject: [PATCH 02/12] changed all of the currency substitutions from byte stream (i.e. \xe2\x82\xac for EURO) to unicode (u'\u20AC') as it was throwing unicode errors when attempting to format the string in the various regex statements. I am aware that I may be the only one to encounter this problem. --- pyfpdb/EverestToFpdb.py | 2 +- pyfpdb/FullTiltPokerSummary.py | 2 +- pyfpdb/FulltiltToFpdb.py | 2 +- pyfpdb/OnGameToFpdb.py | 6 +++--- pyfpdb/PacificPokerToFpdb.py | 4 ++-- pyfpdb/PartyPokerToFpdb.py | 4 ++-- pyfpdb/PokerStarsSummary.py | 2 +- pyfpdb/PokerStarsToFpdb.py | 4 ++-- pyfpdb/WinamaxSummary.py | 2 +- pyfpdb/WinamaxToFpdb.py | 4 ++-- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pyfpdb/EverestToFpdb.py b/pyfpdb/EverestToFpdb.py index 662d2889..befec65f 100644 --- a/pyfpdb/EverestToFpdb.py +++ b/pyfpdb/EverestToFpdb.py @@ -35,7 +35,7 @@ class Everest(HandHistoryConverter): siteID = 15 substitutions = { - 'LS' : u"\$|\xe2\x82\xac|\u20ac|", + 'LS' : u"\$|\u20ac|", 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename } diff --git a/pyfpdb/FullTiltPokerSummary.py b/pyfpdb/FullTiltPokerSummary.py index c06042d0..d1bfd917 100644 --- a/pyfpdb/FullTiltPokerSummary.py +++ b/pyfpdb/FullTiltPokerSummary.py @@ -45,7 +45,7 @@ class FullTiltPokerSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|", # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|", # legal currency symbols - Euro(cp1252, utf-8) 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename 'NUM' : u".,\d", # legal characters in number format } diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index b3d5a172..956e362a 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -36,7 +36,7 @@ class Fulltilt(HandHistoryConverter): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\u20AC|\xe2\x82\xac|", # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|", # legal currency symbols - Euro(cp1252, utf-8) 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename 'NUM' : u".,\d", # legal characters in number format } diff --git a/pyfpdb/OnGameToFpdb.py b/pyfpdb/OnGameToFpdb.py index fd5efe31..32874d41 100755 --- a/pyfpdb/OnGameToFpdb.py +++ b/pyfpdb/OnGameToFpdb.py @@ -42,12 +42,12 @@ class OnGame(HandHistoryConverter): siteId = 5 # Needs to match id entry in Sites database mixes = { } # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20ac" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\\u20AC" # legal currency symbols - Euro(cp1252, utf-8) } - currencies = { u'\u20ac':'EUR', u'\xe2\x82\xac':'EUR', '$':'USD', '':'T$' } + currencies = { u'\u20AC':'EUR', '$':'USD', '':'T$' } limits = { 'NO_LIMIT':'nl', 'POT_LIMIT':'pl', 'LIMIT':'fl'} diff --git a/pyfpdb/PacificPokerToFpdb.py b/pyfpdb/PacificPokerToFpdb.py index a46ea739..d0321492 100644 --- a/pyfpdb/PacificPokerToFpdb.py +++ b/pyfpdb/PacificPokerToFpdb.py @@ -39,10 +39,10 @@ class PacificPoker(HandHistoryConverter): siteId = 13 # Needs to match id entry in Sites database mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } # translations from captured groups to fpdb info strings diff --git a/pyfpdb/PartyPokerToFpdb.py b/pyfpdb/PartyPokerToFpdb.py index a2f708a1..e4e3fc95 100755 --- a/pyfpdb/PartyPokerToFpdb.py +++ b/pyfpdb/PartyPokerToFpdb.py @@ -46,10 +46,10 @@ class PartyPoker(HandHistoryConverter): siteId = 9 filetype = "text" sym = {'USD': "\$", 'EUR': u"\u20ac", 'T$': ""} - currencies = {"\$": "USD", "$": "USD", u"\xe2\x82\xac": "EUR", u"\u20ac": "EUR", '': "T$"} + currencies = {"\$": "USD", "$": "USD", u"\u20ac": "EUR", '': "T$"} substitutions = { 'LEGAL_ISO' : "USD|EUR", # legal ISO currency codes - 'LS' : u"\$|\u20ac|\xe2\x82\xac|", # Currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20ac|", # Currency symbols - Euro(cp1252, utf-8) 'NUM' : u".,\d", } diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index 06358598..09891233 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -45,7 +45,7 @@ class PokerStarsSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } re_SplitTourneys = re.compile("PokerStars Tournament ") diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 4d89bff1..f1364ab1 100644 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -39,10 +39,10 @@ class PokerStars(HandHistoryConverter): siteId = 2 # Needs to match id entry in Sites database mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } # translations from captured groups to fpdb info strings diff --git a/pyfpdb/WinamaxSummary.py b/pyfpdb/WinamaxSummary.py index e1d5b0ce..54e81058 100644 --- a/pyfpdb/WinamaxSummary.py +++ b/pyfpdb/WinamaxSummary.py @@ -45,7 +45,7 @@ class WinamaxSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols + 'LS' : u"\$|\u20AC|" # legal currency symbols } re_GameType = re.compile("""

((?PNo Limit|Pot Limit) (?PHold\'em))

""") diff --git a/pyfpdb/WinamaxToFpdb.py b/pyfpdb/WinamaxToFpdb.py index b01df4ec..be86ae64 100644 --- a/pyfpdb/WinamaxToFpdb.py +++ b/pyfpdb/WinamaxToFpdb.py @@ -52,10 +52,10 @@ class Winamax(HandHistoryConverter): siteId = 14 # Needs to match id entry in Sites database mixes = { } # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } limits = { 'no limit':'nl', 'pot limit' : 'pl','LIMIT':'fl'} From 09f1616e876d3e023c41c442965b06d8c7952383 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 20:49:27 +0000 Subject: [PATCH 03/12] fixed a small bug in storeHudCache --- pyfpdb/Database.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 29047a06..5fe97be5 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1949,11 +1949,9 @@ class Database: insert_hudcache = insert_hudcache.replace('%s', self.sql.query['placeholder']) #print "DEBUG: %s %s %s" %(hid, pids, pdata) - inserts = [] for p in pdata: #NOTE: Insert new stats at right place because SQL needs strict order line = [] - line.append(1) # HDs line.append(pdata[p]['street0VPI']) line.append(pdata[p]['street0Aggr']) @@ -2042,7 +2040,8 @@ class Database: line.append(pdata[p]['street2Raises']) line.append(pdata[p]['street3Raises']) line.append(pdata[p]['street4Raises']) - + + hc, hcs = {}, [] hc['gametypeId'] = gid hc['playerId'] = pids[p] hc['activeSeats'] = len(pids) From 91d791a3ac5d86a1c2e521b6bfc2b9c06bd2367c Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 20:56:14 +0000 Subject: [PATCH 04/12] Commented out the truly absurd amount of printing that goes on whenever you load a Config file --- pyfpdb/Configuration.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index da22a363..adf582b2 100644 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -123,7 +123,7 @@ def get_config(file_name, fallback = True): # OK, fall back to the .example file, should be in the start dir elif os.path.exists(file_name + ".example"): try: - print "" + #print "" example_path = file_name + ".example" check_dir(default_dir) if not config_found and fallback: @@ -173,7 +173,7 @@ def get_logger(file_name, config = "config", fallback = False, log_dir=None, log log = logging.getLogger() # but it looks like default is no output :-( maybe because all the calls name a module? log.debug(_("Default logger initialised for %s") % file) - print(_("Default logger initialised for %s") % file) + #print(_("Default logger initialised for %s") % file) return log def check_dir(path, create = True): @@ -314,7 +314,7 @@ class Site: self.layout = {} self.emails = {} - print _("Loading site"), self.site_name + #print _("Loading site"), self.site_name for layout_node in node.getElementsByTagName('layout'): lo = Layout(layout_node) @@ -631,7 +631,7 @@ class RawHands: if node==None: self.save="error" self.compression="none" - print _("missing config section raw_hands") + #print _("missing config section raw_hands") else: save=node.getAttribute("save") if save in ("none", "error", "all"): @@ -657,7 +657,7 @@ class RawTourneys: if node==None: self.save="error" self.compression="none" - print _("missing config section raw_tourneys") + #print _("missing config section raw_tourneys") else: save=node.getAttribute("save") if save in ("none", "error", "all"): @@ -720,7 +720,7 @@ class Config: while added > 0 and n < 2: n = n + 1 log.info(_("Reading configuration file %s") % file) - print (("\n"+_("Reading configuration file %s")+"\n") % file) + #print (("\n"+_("Reading configuration file %s")+"\n") % file) try: doc = xml.dom.minidom.parse(file) self.doc = doc @@ -829,7 +829,7 @@ class Config: for raw_tourneys_node in doc.getElementsByTagName('raw_tourneys'): self.raw_tourneys = RawTourneys(raw_tourneys_node) - print "" + #print "" #end def __init__ def add_missing_elements(self, doc, example_file): From 3c2fdaf53e6eda4da10ecafda0cf59097449f7ed Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 21:31:27 +0000 Subject: [PATCH 05/12] Fixed some data-type bugs in the Import class -- essentialy the same commit as 530a21cb9f65f83f8b09ef08f6415c8bd256fccc in the chaz branch of my repo --- pyfpdb/Configuration.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index adf582b2..5186c2df 100644 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -488,16 +488,16 @@ class Popup: class Import: def __init__(self, node): - self.node = node - self.interval = node.getAttribute("interval") - self.callFpdbHud = node.getAttribute("callFpdbHud") - self.ResultsDirectory = node.getAttribute("ResultsDirectory") - self.hhBulkPath = node.getAttribute("hhBulkPath") - self.saveActions = string_to_bool(node.getAttribute("saveActions"), default=False) - self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions"), default=False) - self.sessionTimeout = string_to_bool(node.getAttribute("sessionTimeout"), default=30) - self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) - self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH"), default=False) + self.node = node + self.interval = node.getAttribute("interval") + self.sessionTimeout = node.getAttribute("sessionTimeout") + self.ResultsDirectory = node.getAttribute("ResultsDirectory") + self.hhBulkPath = node.getAttribute("hhBulkPath") + self.saveActions = string_to_bool(node.getAttribute("saveActions") , default=False) + self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions") , default=False) + self.callFpdbHud = string_to_bool(node.getAttribute("callFpdbHud") , default=False) + self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) + self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH") , default=False) def __str__(self): return " interval = %s\n callFpdbHud = %s\n saveActions = %s\n fastStoreHudCache = %s\nResultsDirectory = %s" \ From dd6ce464875185d33c6a4ef0e7ac2634648ad05e Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Tue, 22 Mar 2011 21:45:09 +0000 Subject: [PATCH 06/12] Added methods to create utilize a lock table for managing access to the database during a multi-threaded import. Currently, only MySQL is supported --- pyfpdb/Database.py | 29 +++++++++++++++++++++++++++++ pyfpdb/SQL.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 5fe97be5..b739f159 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -273,6 +273,7 @@ class Database: self.db_path = '' gen = c.get_general_params() self.day_start = 0 + self._has_lock = False if 'day_start' in gen: self.day_start = float(gen['day_start']) @@ -1719,6 +1720,34 @@ class Database: # however the calling prog requires. Main aims: # - existing static routines from fpdb_simple just modified + def setThreadId(self, threadid): + self.threadId = threadid + + def acquireLock(self, wait=True, retry_time=.01): + while not self._has_lock: + cursor = self.get_cursor() + cursor.execute(self.sql.query['selectLock']) + record = cursor.fetchall() + self.commit() + if not len(record): + cursor.execute(self.sql.query['switchLock'], (True, self.threadId)) + self.commit() + self._has_lock = True + return True + else: + cursor.execute(self.sql.query['missedLock'], (1, self.threadId)) + self.commit() + if not wait: + return False + sleep(retry_time) + + def releaseLock(self): + if self._has_lock: + cursor = self.get_cursor() + num = cursor.execute(self.sql.query['switchLock'], (False, self.threadId)) + self.commit() + self._has_lock = False + def lock_for_insert(self): """Lock tables in MySQL to try to speed inserts up""" try: diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index f7f84a98..ec3c03df 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -107,6 +107,15 @@ class Sql: elif db_server == 'sqlite': self.query['createSettingsTable'] = """CREATE TABLE Settings (version INTEGER NOT NULL) """ + + ################################ + # Create InsertLock + ################################ + if db_server == 'mysql': + self.query['createLockTable'] = """CREATE TABLE InsertLock ( + id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), + locked BOOLEAN NOT NULL DEFAULT FALSE) + ENGINE=INNODB""" ################################ # Create RawHands (this table is all but identical with RawTourneys) @@ -4501,6 +4510,25 @@ class Sql: self.query['analyze'] = "analyze" elif db_server == 'sqlite': self.query['analyze'] = "analyze" + + if db_server == 'mysql': + self.query['selectLock'] = """ + SELECT locked + FROM InsertLock + WHERE locked=True + LOCK IN SHARE MODE""" + + if db_server == 'mysql': + self.query['switchLock'] = """ + UPDATE InsertLock SET + locked=%s + WHERE id=%s""" + + if db_server == 'mysql': + self.query['missedLock'] = """ + UPDATE InsertLock SET + missed=missed+%s + WHERE id=%s""" if db_server == 'mysql': self.query['lockForInsert'] = """ From 3ac088b74842205d6eedc00945b8a6d91c37a3a7 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Wed, 23 Mar 2011 01:22:18 +0000 Subject: [PATCH 07/12] Fixed a timezone issue in storeSessionsCache --- pyfpdb/Database.py | 4 ++++ pyfpdb/fpdb_import.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index b739f159..fbb29655 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -2231,6 +2231,10 @@ class Database: def storeSessionsCache(self, hid, pids, startTime, game, gid, pdata, sc, gsc, tz, heros, doinsert = False): """Update cached sessions. If no record exists, do an insert""" + if not tz: + tz_dt = datetime.utcnow() - datetime.today() + tz = tz_dt.seconds/3600 + THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) local = startTime + timedelta(hours=int(tz)) date = "d%02d%02d%02d" % (local.year - 2000, local.month, local.day) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index c59e9bf8..93e57554 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -475,7 +475,7 @@ class Importer: hand = phands[i] try: id = hand.getHandId(self.database, id) - sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, self.tz, doinsert) + sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert) hbulk = hand.insertHands(self.database, hbulk, doinsert) hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) ihands.append(hand) From 8c4d5d42b42fc6884415cf56e51f040287b6ec79 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Wed, 23 Mar 2011 04:59:44 +0000 Subject: [PATCH 08/12] Added printtest back in and fixed a couple of SessionsCache bugs --- pyfpdb/Database.py | 30 ++++++++++++++++++++++++------ pyfpdb/GuiTourneyImport.py | 2 +- pyfpdb/Hand.py | 12 ++++++------ pyfpdb/TourneySummary.py | 13 +++++++++++-- pyfpdb/fpdb_import.py | 6 +++--- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index fbb29655..d684e60a 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1760,8 +1760,14 @@ class Database: # NEWIMPORT CODE ########################### - def storeHand(self, hdata, hbulk, doinsert = False): - #stores into table hands: + def storeHand(self, hdata, hbulk, doinsert = False, printdata = False): + if printdata: + print _("######## Hands ##########") + import pprint + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(hdata) + print _("###### End Hands ########") + # Tablename can have odd charachers hdata['tableName'] = Charset.to_db_utf8(hdata['tableName']) @@ -1813,8 +1819,12 @@ class Database: self.commit() return hbulk - def storeHandsPlayers(self, hid, pids, pdata, hpbulk, doinsert = False): + def storeHandsPlayers(self, hid, pids, pdata, hpbulk, doinsert = False, printdata = False): #print "DEBUG: %s %s %s" %(hid, pids, pdata) + if printdata: + import pprint + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(pdata) for p in pdata: hpbulk.append( ( hid, @@ -1930,8 +1940,14 @@ class Database: c.executemany(q, hpbulk) return hpbulk - def storeHandsActions(self, hid, pids, adata, habulk, doinsert = False): + def storeHandsActions(self, hid, pids, adata, habulk, doinsert = False, printdata = False): #print "DEBUG: %s %s %s" %(hid, pids, adata) + + # This can be used to generate test data. Currently unused + #if printdata: + # import pprint + # pp = pprint.PrettyPrinter(indent=4) + # pp.pprint(adata) for a in adata: habulk.append( (hid, @@ -2262,8 +2278,10 @@ class Database: if (game['type']=='summary'): hand['type'] = 'tour' hand['tourneys'] = 1 - hand['tourneyTypeId'] = pdata.tourneyTypeId - hand['totalProfit'] = pdata.winnings[p] - (pdata.buyin + pdata.fee) + hand['tourneyTypeId'] = pdata['tourneyTypeId'] + hand['totalProfit'] = pdata['winnings'][p] + if pdata['buyinCurrency'] == pdata['winningsCurrency'][p]: + hand['totalProfit'] - (pdata['buyin'] + pdata['fee']) elif (game['type']=='ring'): hand['type'] = game['type'] hand['hands'] = 1 diff --git a/pyfpdb/GuiTourneyImport.py b/pyfpdb/GuiTourneyImport.py index c6f0bb48..0ed33b18 100755 --- a/pyfpdb/GuiTourneyImport.py +++ b/pyfpdb/GuiTourneyImport.py @@ -229,7 +229,7 @@ class SummaryImporter: doinsert = len(summaryTexts)==j try: conv = obj(db=self.database, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") - sc, gsc = conv.updateSessionsCache(sc, gsc, self.tz, doinsert) + sc, gsc = conv.updateSessionsCache(sc, gsc, None, doinsert) except FpdbParseError, e: errors += 1 print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts)) diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index f86e5df9..cb3bb405 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -259,24 +259,24 @@ dealt whether they were seen in a 'dealt to' line next = id +1 return next - def insertHands(self, db, hbulk, doinsert = False): + def insertHands(self, db, hbulk, doinsert = False, printtest = False): """ Function to insert Hand into database Should not commit, and do minimal selects. Callers may want to cache commits db: a connected Database object""" self.hands['gameTypeId'] = self.dbid_gt self.hands['seats'] = len(self.dbid_pids) - hbulk = db.storeHand(self.hands, hbulk, doinsert) + hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest) return hbulk - def insertHandsPlayers(self, db, hpbulk, doinsert = False): + def insertHandsPlayers(self, db, hpbulk, doinsert = False, printtest = False): """ Function to inserts HandsPlayers into database""" - hpbulk = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.handsplayers, hpbulk, doinsert) + hpbulk = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.handsplayers, hpbulk, doinsert, printtest) return hpbulk - def insertHandsActions(self, db, habulk, doinsert = False): + def insertHandsActions(self, db, habulk, doinsert = False, printtest = False): """ Function to inserts HandsActions into database""" handsactions = self.stats.getHandsActions() - habulk = db.storeHandsActions(self.dbid_hands, self.dbid_pids, handsactions, habulk, doinsert) + habulk = db.storeHandsActions(self.dbid_hands, self.dbid_pids, handsactions, habulk, doinsert, printtest) return habulk def updateHudCache(self, db, hcbulk, doinsert = False): diff --git a/pyfpdb/TourneySummary.py b/pyfpdb/TourneySummary.py index 0483755f..2e655f6b 100644 --- a/pyfpdb/TourneySummary.py +++ b/pyfpdb/TourneySummary.py @@ -291,11 +291,20 @@ winnings (int) the money the player ended the tourney with (can be 0, or -1 i def updateSessionsCache(self, sc, gsc, tz, doinsert): self.heros = self.db.getHeroIds(self.dbid_pids, self.siteName) sc = self.db.prepSessionsCache(self.tourNo, self.dbid_pids, self.startTime, sc , self.heros, doinsert) - gsc = self.db.storeSessionsCache(self.tourNo, self.dbid_pids, self.startTime, {'type': 'summary'} - , None, self, sc, gsc, tz, self.heros, doinsert) + ,None, self.assembleInfo(), sc, gsc, tz, self.heros, doinsert) return sc, gsc + + def assembleInfo(self): + info = {} + info['tourneyTypeId'] = self.tourneyTypeId + info['winnings'] = self.winnings + info['winningsCurrency'] = self.winningsCurrency + info['buyinCurrency'] = self.buyinCurrency + info['buyin'] = self.buyin + info['fee'] = self.fee + return info def writeSummary(self, fh=sys.__stdout__): print >>fh, "Override me" diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 93e57554..7aab478c 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -476,7 +476,7 @@ class Importer: try: id = hand.getHandId(self.database, id) sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert) - hbulk = hand.insertHands(self.database, hbulk, doinsert) + hbulk = hand.insertHands(self.database, hbulk, doinsert, self.settings['testData']) hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) ihands.append(hand) to_hud.append(id) @@ -488,8 +488,8 @@ class Importer: for i in range(len(ihands)): doinsert = len(ihands)==i+1 hand = ihands[i] - hpbulk = hand.insertHandsPlayers(self.database, hpbulk, doinsert) - habulk = hand.insertHandsActions(self.database, habulk, doinsert) + hpbulk = hand.insertHandsPlayers(self.database, hpbulk, doinsert, self.settings['testData']) + habulk = hand.insertHandsActions(self.database, habulk, doinsert, self.settings['testData']) self.database.commit() #pipe the Hands.id out to the HUD From 3e9dacf900c49136a4df1a970c11883b573679c0 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Wed, 23 Mar 2011 05:44:03 +0000 Subject: [PATCH 09/12] Added a rebuild_sessionscache method for creating the SC table from hands in the database --- pyfpdb/Database.py | 88 ++++++++++++++++++++++++++++++++++++++++++---- pyfpdb/SQL.py | 38 ++++++++++++++++++++ 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index d684e60a..9e7895be 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1637,10 +1637,86 @@ class Database: print err #end def rebuild_hudcache - def rebuild_sessionscache(self, h_start=None, v_start=None): - """clears sessionscache and rebuilds from the individual handsplayers records""" - #Will get to this soon - pass + def rebuild_sessionscache(self): + """clears sessionscache and rebuilds from the individual records""" + heros = [] + for site in self.config.get_supported_sites(): + result = self.get_site_id(site) + if result: + site_id = result[0][0] + hero = self.config.supported_sites[site].screen_name + p_id = self.get_player_id(self.config, site, hero) + if p_id: + heros.append(int(p_id)) + + rebuildSessionsCache = self.sql.query['rebuildSessionsCache'] + rebuildSessionsCacheSum = self.sql.query['rebuildSessionsCacheSum'] + + if len(heros) == 0: + where = '0' + where_summary = '0' + elif len(heros) > 0: + where = str(heros[0]) + where_summary = str(heros[0]) + if len(heros) > 1: + for i in heros: + if i != heros[0]: + where = where + ' OR HandsPlayers.playerId = %s' % str(i) + where_summary = where_summary + ' OR TourneysPlayers.playerId = %s' % str(i) + rebuildSessionsCache = rebuildSessionsCache.replace('', where) + rebuildSessionsCacheSum = rebuildSessionsCacheSum.replace('', where_summary) + + c = self.get_cursor() + c.execute(self.sql.query['clearSessionsCache']) + self.commit() + + sc, gsc = {'bk': []}, {'bk': []} + c.execute(rebuildSessionsCache) + tmp = c.fetchone() + while True: + pids, game, pdata = {}, {}, {} + pdata['pname'] = {} + id = tmp[0] + startTime = tmp[1] + pids['pname'] = tmp[2] + gid = tmp[3] + game['type'] = tmp[4] + pdata['pname']['totalProfit'] = tmp[5] + pdata['pname']['tourneyTypeId'] = tmp[6] + tmp = c.fetchone() + sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None) + gsc = self.storeSessionsCache(id, pids, startTime, game, gid, pdata, sc, gsc, None, heros, tmp == None) + if tmp == None: + for i, id in sc.iteritems(): + if i!='bk': + sid = id['id'] + gid = gsc[i]['id'] + c.execute("UPDATE Hands SET sessionId = %s, gameSessionId = %s WHERE id = %s", (sid, gid, i)) + break + self.commit() + + sc, gsc = {'bk': []}, {'bk': []} + c.execute(rebuildSessionsCacheSum) + tmp = c.fetchone() + while True: + pids, game, info = {}, {}, {} + id = tmp[0] + startTime = tmp[1] + pids['pname'] = tmp[2] + game['type'] = 'summary' + info['tourneyTypeId'] = tmp[3] + info['winnings'] = {} + info['winnings']['pname'] = tmp[4] + info['winningsCurrency'] = {} + info['winningsCurrency']['pname'] = tmp[5] + info['buyinCurrency'] = tmp[6] + info['buyin'] = tmp[7] + info['fee'] = tmp[8] + tmp = c.fetchone() + sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None) + gsc = self.storeSessionsCache(id, pids, startTime, game, None, info, sc, gsc, None, heros, tmp == None) + if tmp == None: + break def get_hero_hudcache_start(self): """fetches earliest stylekey from hudcache for one of hero's player ids""" @@ -2279,9 +2355,9 @@ class Database: hand['type'] = 'tour' hand['tourneys'] = 1 hand['tourneyTypeId'] = pdata['tourneyTypeId'] - hand['totalProfit'] = pdata['winnings'][p] if pdata['buyinCurrency'] == pdata['winningsCurrency'][p]: - hand['totalProfit'] - (pdata['buyin'] + pdata['fee']) + hand['totalProfit'] = pdata['winnings'][p] - (pdata['buyin'] + pdata['fee']) + else: hand['totalProfit'] = pdata['winnings'][p] elif (game['type']=='ring'): hand['type'] = game['type'] hand['hands'] = 1 diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index ec3c03df..e389feea 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -4404,6 +4404,44 @@ class Sql: #################################### # Queries to rebuild/modify sessionscache #################################### + + self.query['clearSessionsCache'] = """DELETE FROM SessionsCache""" + + self.query['rebuildSessionsCache'] = """ + SELECT Hands.id as id, + Hands.startTime as startTime, + HandsPlayers.playerId as playerId, + Hands.gametypeId as gametypeId, + Gametypes.type as game, + HandsPlayers.totalProfit as totalProfit, + Tourneys.tourneyTypeId as tourneyTypeId + FROM Gametypes, HandsPlayers, Hands + LEFT JOIN Tourneys ON Hands.tourneyId = Tourneys.tourneyTypeId + WHERE HandsPlayers.handId = Hands.id + AND Hands.gametypeId = Gametypes.id + AND (case when HandsPlayers.playerId = then 1 else 0 end) = 1 + ORDER BY Hands.startTime ASC""" + + self.query['rebuildSessionsCacheSum'] = """ + SELECT Tourneys.id as id, + Tourneys.startTime as startTime, + TourneysPlayers.playerId, + TourneyTypes.id as tourneyTypeId, + TourneysPlayers.winnings as winnings, + TourneysPlayers.winningsCurrency as winningsCurrency, + TourneyTypes.currency as buyinCurrency, + TourneyTypes.buyIn as buyIn, + TourneyTypes.fee as fee, + case when TourneyTypes.rebuy then TourneyTypes.rebuyCost else 0 end as rebuyCost, + case when TourneyTypes.rebuy then TourneyTypes.rebuyFee else 0 end as rebuyFee, + case when TourneyTypes.addOn then TourneyTypes.addOnCost else 0 end as addOnCost, + case when TourneyTypes.addOn then TourneyTypes.addOnFee else 0 end as addOnFee, + case when TourneyTypes.knockout then TourneyTypes.koBounty else 0 end as koBounty + FROM Tourneys, TourneyTypes, TourneysPlayers + WHERE Tourneys.tourneyTypeId = TourneyTypes.id + AND Tourneys.id = TourneysPlayers.tourneyId + AND (case when TourneysPlayers.playerId = then 1 else 0 end) = 1 + ORDER BY Tourneys.startTime ASC""" self.query['select_prepSC'] = """ SELECT sessionId as id, From 7c31a190770cf4ed3170d12dcbbfdadd8a15c494 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Wed, 23 Mar 2011 19:34:15 +0000 Subject: [PATCH 10/12] Modified HHC so gtk & pygtk are imported to execute a callback only if import is run from the gui --- pyfpdb/HandHistoryConverter.py | 13 ++++++------- pyfpdb/fpdb_import.py | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index b41f9342..6d25d6aa 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -50,9 +50,6 @@ import Hand from Exceptions import FpdbParseError import Configuration -import pygtk -import gtk - class HandHistoryConverter(): READ_CHUNK_SIZE = 10000 # bytes to read at a time from file in tail mode @@ -128,9 +125,6 @@ If in follow mode, wait for more data to turn up. Otherwise, finish at EOF. """ - while gtk.events_pending(): - gtk.main_iteration(False) - starttime = time.time() if not self.sanityCheck(): log.warning(_("Failed sanity check")) @@ -182,7 +176,12 @@ Otherwise, finish at EOF. finally: if self.out_fh != sys.stdout: self.out_fh.close() - + + def progressNotify(self): + "A callback to the interface while events are pending" + import gtk, pygtk + while gtk.events_pending(): + gtk.main_iteration(False) def tailHands(self): """Generator of handTexts from a tailed file: diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 7aab478c..a104d5ce 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -453,6 +453,7 @@ class Importer: ,sitename = site ) if hhc.getStatus(): + if self.caller: hhc.progressNotify() handlist = hhc.getProcessedHands() self.pos_in_file[file] = hhc.getLastCharacterRead() (hbulk, hpbulk, habulk, hcbulk, phands, ihands) = ([], [], [], [], [], []) From 5eeafc4503acc84498f72154b8f0117431203590 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Thu, 24 Mar 2011 04:08:08 +0000 Subject: [PATCH 11/12] Added regex & methods for picking up the 'discards' and 'stands pat' actions in FTP draw hands --- pyfpdb/FulltiltToFpdb.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 956e362a..c6775e92 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -172,7 +172,10 @@ class Fulltilt(HandHistoryConverter): self.re_BringIn = re.compile(r"^%(PLAYERS)s brings in for [%(LS)s]?(?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_PostBoth = re.compile(r"^%(PLAYERS)s posts small \& big blinds \[[%(LS)s]? (?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_HeroCards = re.compile(r"^Dealt to %s(?: \[(?P.+?)\])?( \[(?P.+?)\])" % player_re, re.MULTILINE) - self.re_Action = re.compile(r"^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds)( [%(LS)s]?(?P[%(NUM)s]+))?" % self.substitutions, re.MULTILINE) + self.re_Action = re.compile(r""" + ^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds| discards| stands pat) + ( [%(LS)s]?(?P[%(NUM)s]+))? + (\scards?(\s\[(?P.+?)\])?)?""" % self.substitutions, re.MULTILINE) self.re_ShowdownAction = re.compile(r"^%s shows \[(?P.*)\]" % player_re, re.MULTILINE) self.re_CollectPot = re.compile(r"^Seat (?P[0-9]+): %(PLAYERS)s (\(button\) |\(small blind\) |\(big blind\) )?(collected|showed \[.*\] and won) \([%(LS)s]?(?P[%(NUM)s]+)\)(, mucked| with.*)?" % self.substitutions, re.MULTILINE) self.re_SitsOut = re.compile(r"^%s sits out" % player_re, re.MULTILINE) @@ -483,6 +486,10 @@ class Fulltilt(HandHistoryConverter): hand.addFold( street, action.group('PNAME')) elif action.group('ATYPE') == ' checks': hand.addCheck( street, action.group('PNAME')) + elif action.group('ATYPE') == ' discards': + hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('DISCARDED')) + elif action.group('ATYPE') == ' stands pat': + hand.addStandsPat( street, action.group('PNAME')) else: print _("FullTilt: DEBUG: unimplemented readAction: '%s' '%s'") %(action.group('PNAME'),action.group('ATYPE'),) From a87f43f9336a15f8fff47b2b30d5b1dfc9211a42 Mon Sep 17 00:00:00 2001 From: Chaz Littlejohn Date: Thu, 24 Mar 2011 06:00:10 +0000 Subject: [PATCH 12/12] Updated the import process for Draw hands so that cards, along with additional draws can now be stored. HandsPlayers now includes 20 card fields to accomodate up to 4 five card hands for each player. The regex for the 'stands pat' action was also improved so that hero cards from those streets could be taken from that line of text --- pyfpdb/Database.py | 15 +++++++++- pyfpdb/DerivedStats.py | 6 ++-- pyfpdb/FulltiltToFpdb.py | 9 ++---- pyfpdb/Hand.py | 14 ++++++++-- pyfpdb/PokerStarsToFpdb.py | 7 +++-- pyfpdb/SQL.py | 57 +++++++++++++++++++++++++++++++++++++- 6 files changed, 92 insertions(+), 16 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 9e7895be..38eb802e 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 151 +DB_VERSION = 152 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -1915,6 +1915,19 @@ class Database: pdata[p]['card5'], pdata[p]['card6'], pdata[p]['card7'], + pdata[p]['card8'], + pdata[p]['card9'], + pdata[p]['card10'], + pdata[p]['card11'], + pdata[p]['card12'], + pdata[p]['card13'], + pdata[p]['card14'], + pdata[p]['card15'], + pdata[p]['card16'], + pdata[p]['card17'], + pdata[p]['card18'], + pdata[p]['card19'], + pdata[p]['card20'], pdata[p]['winnings'], pdata[p]['rake'], pdata[p]['totalProfit'], diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 3b955929..d29a33ad 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -214,10 +214,10 @@ class DerivedStats(): for player in hand.players: hcs = hand.join_holecards(player[1], asList=True) - hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x'] - #for i, card in enumerate(hcs[:7], 1): #Python 2.6 syntax + hcs = hcs + [u'0x']*18 + #for i, card in enumerate(hcs[:20, 1): #Python 2.6 syntax # self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card) - for i, card in enumerate(hcs[:7]): + for i, card in enumerate(hcs[:20]): self.handsplayers[player[1]]['card%s' % (i+1)] = Card.encodeCard(card) self.handsplayers[player[1]]['startCards'] = Card.calcStartCards(hand, player[1]) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index c6775e92..d26c60eb 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -172,10 +172,7 @@ class Fulltilt(HandHistoryConverter): self.re_BringIn = re.compile(r"^%(PLAYERS)s brings in for [%(LS)s]?(?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_PostBoth = re.compile(r"^%(PLAYERS)s posts small \& big blinds \[[%(LS)s]? (?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_HeroCards = re.compile(r"^Dealt to %s(?: \[(?P.+?)\])?( \[(?P.+?)\])" % player_re, re.MULTILINE) - self.re_Action = re.compile(r""" - ^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds| discards| stands pat) - ( [%(LS)s]?(?P[%(NUM)s]+))? - (\scards?(\s\[(?P.+?)\])?)?""" % self.substitutions, re.MULTILINE) + self.re_Action = re.compile(r"^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds| discards| stands pat)( [%(LS)s]?(?P[%(NUM)s]+))?(\son|\scards?)?(\s\[(?P.+?)\])?" % self.substitutions, re.MULTILINE) self.re_ShowdownAction = re.compile(r"^%s shows \[(?P.*)\]" % player_re, re.MULTILINE) self.re_CollectPot = re.compile(r"^Seat (?P[0-9]+): %(PLAYERS)s (\(button\) |\(small blind\) |\(big blind\) )?(collected|showed \[.*\] and won) \([%(LS)s]?(?P[%(NUM)s]+)\)(, mucked| with.*)?" % self.substitutions, re.MULTILINE) self.re_SitsOut = re.compile(r"^%s sits out" % player_re, re.MULTILINE) @@ -487,9 +484,9 @@ class Fulltilt(HandHistoryConverter): elif action.group('ATYPE') == ' checks': hand.addCheck( street, action.group('PNAME')) elif action.group('ATYPE') == ' discards': - hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('DISCARDED')) + hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS')) elif action.group('ATYPE') == ' stands pat': - hand.addStandsPat( street, action.group('PNAME')) + hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS')) else: print _("FullTilt: DEBUG: unimplemented readAction: '%s' '%s'") %(action.group('PNAME'),action.group('ATYPE'),) diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index cb3bb405..7f89c050 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -661,10 +661,13 @@ Add a raise on [street] by [player] to [amountTo] self.pot.addMoney(player, amount) - def addStandsPat(self, street, player): + def addStandsPat(self, street, player, cards): self.checkPlayerExists(player) act = (player, 'stands pat') self.actions[street].append(act) + if cards: + cards = cards.split(' ') + self.addHoleCards(street, player, open=[], closed=cards) def addFold(self, street, player): @@ -1224,7 +1227,14 @@ class DrawHand(Hand): def join_holecards(self, player, asList=False): """With asList = True it returns the set cards for a player including down cards if they aren't know""" # FIXME: This should actually return - holecards = [u'0x', u'0x', u'0x', u'0x', u'0x'] + holecards = [u'0x']*20 + + for i, street in enumerate(self.holeStreets): + if player in self.holecards[street].keys(): + allhole = self.holecards[street][player][0] + self.holecards[street][player][1] + for c in range(len(allhole)): + idx = c + (i*5) + holecards[idx] = allhole[c] if asList == False: return " ".join(holecards) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index f1364ab1..d6abca84 100644 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -145,7 +145,8 @@ class PokerStars(HandHistoryConverter): (\s(%(CUR)s)?(?P[.\d]+))?(\sto\s%(CUR)s(?P[.\d]+))? # the number discarded goes in \s*(and\sis\sall.in)? (and\shas\sreached\sthe\s[%(CUR)s\d\.]+\scap)? - (\scards?(\s\[(?P.+?)\])?)?\s*$""" + (\son|\scards?)? + (\s\[(?P.+?)\])?\s*$""" % short_subst, re.MULTILINE|re.VERBOSE) re_ShowdownAction = re.compile(r"^%s: shows \[(?P.*)\]" % short_subst['PLYR'], re.MULTILINE) re_sitsOut = re.compile("^%s sits out" % short_subst['PLYR'], re.MULTILINE) @@ -432,9 +433,9 @@ class PokerStars(HandHistoryConverter): elif action.group('ATYPE') == ' checks': hand.addCheck( street, action.group('PNAME')) elif action.group('ATYPE') == ' discards': - hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('DISCARDED')) + hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS')) elif action.group('ATYPE') == ' stands pat': - hand.addStandsPat( street, action.group('PNAME')) + hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS')) else: print (_("DEBUG: ") + _("Unimplemented readAction: '%s' '%s'") % (action.group('PNAME'),action.group('ATYPE'))) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index e389feea..e5b4c3d8 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -633,6 +633,19 @@ class Sql: card5 smallint, card6 smallint, card7 smallint, + card8 smallint, /* cards 8-20 for draw hands */ + card9 smallint, + card10 smallint, + card11 smallint, + card12 smallint, + card13 smallint, + card14 smallint, + card15 smallint, + card16 smallint, + card17 smallint, + card18 smallint, + card19 smallint, + card20 smallint, startCards smallint, ante INT, @@ -760,6 +773,19 @@ class Sql: card5 smallint, card6 smallint, card7 smallint, + card8 smallint, /* cards 8-20 for draw hands */ + card9 smallint, + card10 smallint, + card11 smallint, + card12 smallint, + card13 smallint, + card14 smallint, + card15 smallint, + card16 smallint, + card17 smallint, + card18 smallint, + card19 smallint, + card20 smallint, startCards smallint, ante INT, @@ -886,6 +912,19 @@ class Sql: card5 INT, card6 INT, card7 INT, + card8 INT, /* cards 8-20 for draw hands */ + card9 INT, + card10 INT, + card11 INT, + card12 INT, + card13 INT, + card14 INT, + card15 INT, + card16 INT, + card17 INT, + card18 INT, + card19 INT, + card20 INT, startCards INT, ante INT, @@ -4799,6 +4838,19 @@ class Sql: card5, card6, card7, + card8, + card9, + card10, + card11, + card12, + card13, + card14, + card15, + card16, + card17, + card18, + card19, + card20, winnings, rake, totalProfit, @@ -4913,7 +4965,10 @@ class Sql: %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, %s )""" self.query['store_hands_actions'] = """insert into HandsActions (