diff --git a/pyfpdb/Configuration.py b/pyfpdb/Configuration.py index da22a363..5186c2df 100644 --- a/pyfpdb/Configuration.py +++ b/pyfpdb/Configuration.py @@ -123,7 +123,7 @@ def get_config(file_name, fallback = True): # OK, fall back to the .example file, should be in the start dir elif os.path.exists(file_name + ".example"): try: - print "" + #print "" example_path = file_name + ".example" check_dir(default_dir) if not config_found and fallback: @@ -173,7 +173,7 @@ def get_logger(file_name, config = "config", fallback = False, log_dir=None, log log = logging.getLogger() # but it looks like default is no output :-( maybe because all the calls name a module? log.debug(_("Default logger initialised for %s") % file) - print(_("Default logger initialised for %s") % file) + #print(_("Default logger initialised for %s") % file) return log def check_dir(path, create = True): @@ -314,7 +314,7 @@ class Site: self.layout = {} self.emails = {} - print _("Loading site"), self.site_name + #print _("Loading site"), self.site_name for layout_node in node.getElementsByTagName('layout'): lo = Layout(layout_node) @@ -488,16 +488,16 @@ class Popup: class Import: def __init__(self, node): - self.node = node - self.interval = node.getAttribute("interval") - self.callFpdbHud = node.getAttribute("callFpdbHud") - self.ResultsDirectory = node.getAttribute("ResultsDirectory") - self.hhBulkPath = node.getAttribute("hhBulkPath") - self.saveActions = string_to_bool(node.getAttribute("saveActions"), default=False) - self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions"), default=False) - self.sessionTimeout = string_to_bool(node.getAttribute("sessionTimeout"), default=30) - self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) - self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH"), default=False) + self.node = node + self.interval = node.getAttribute("interval") + self.sessionTimeout = node.getAttribute("sessionTimeout") + self.ResultsDirectory = node.getAttribute("ResultsDirectory") + self.hhBulkPath = node.getAttribute("hhBulkPath") + self.saveActions = string_to_bool(node.getAttribute("saveActions") , default=False) + self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions") , default=False) + self.callFpdbHud = string_to_bool(node.getAttribute("callFpdbHud") , default=False) + self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) + self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH") , default=False) def __str__(self): return " interval = %s\n callFpdbHud = %s\n saveActions = %s\n fastStoreHudCache = %s\nResultsDirectory = %s" \ @@ -631,7 +631,7 @@ class RawHands: if node==None: self.save="error" self.compression="none" - print _("missing config section raw_hands") + #print _("missing config section raw_hands") else: save=node.getAttribute("save") if save in ("none", "error", "all"): @@ -657,7 +657,7 @@ class RawTourneys: if node==None: self.save="error" self.compression="none" - print _("missing config section raw_tourneys") + #print _("missing config section raw_tourneys") else: save=node.getAttribute("save") if save in ("none", "error", "all"): @@ -720,7 +720,7 @@ class Config: while added > 0 and n < 2: n = n + 1 log.info(_("Reading configuration file %s") % file) - print (("\n"+_("Reading configuration file %s")+"\n") % file) + #print (("\n"+_("Reading configuration file %s")+"\n") % file) try: doc = xml.dom.minidom.parse(file) self.doc = doc @@ -829,7 +829,7 @@ class Config: for raw_tourneys_node in doc.getElementsByTagName('raw_tourneys'): self.raw_tourneys = RawTourneys(raw_tourneys_node) - print "" + #print "" #end def __init__ def add_missing_elements(self, doc, example_file): diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 3898c65d..38eb802e 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 150 +DB_VERSION = 152 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -134,6 +134,9 @@ class Database: , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1} + , {'tab':'SessionsCache', 'col':'playerId', 'drop':0} + , {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'Players', 'col':'siteId', 'drop':1} #, {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} @@ -157,6 +160,9 @@ class Database: , {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} + , {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1} + , {'tab':'SessionsCache', 'col':'playerId', 'drop':0} + , {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'Players', 'col':'siteId', 'drop':1} , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} , {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0} @@ -182,6 +188,9 @@ class Database: , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} ] , [ # foreign keys for postgres (index 3) {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} @@ -193,6 +202,9 @@ class Database: , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} + , {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} ] , [ # no foreign keys in sqlite (index 4) ] @@ -261,6 +273,7 @@ class Database: self.db_path = '' gen = c.get_general_params() self.day_start = 0 + self._has_lock = False if 'day_start' in gen: self.day_start = float(gen['day_start']) @@ -1252,6 +1265,12 @@ class Database: c.execute(self.sql.query['createBackingsTable']) c.execute(self.sql.query['createRawHands']) c.execute(self.sql.query['createRawTourneys']) + + # Create sessionscache indexes + log.debug("Creating SessionsCache indexes") + c.execute(self.sql.query['addSessionIdIndex']) + c.execute(self.sql.query['addHandsSessionIdIndex']) + c.execute(self.sql.query['addHandsGameSessionIdIndex']) # Create unique indexes: log.debug("Creating unique indexes") @@ -1618,10 +1637,86 @@ class Database: print err #end def rebuild_hudcache - def rebuild_sessionscache(self, h_start=None, v_start=None): - """clears sessionscache and rebuilds from the individual handsplayers records""" - #Will get to this soon - pass + def rebuild_sessionscache(self): + """clears sessionscache and rebuilds from the individual records""" + heros = [] + for site in self.config.get_supported_sites(): + result = self.get_site_id(site) + if result: + site_id = result[0][0] + hero = self.config.supported_sites[site].screen_name + p_id = self.get_player_id(self.config, site, hero) + if p_id: + heros.append(int(p_id)) + + rebuildSessionsCache = self.sql.query['rebuildSessionsCache'] + rebuildSessionsCacheSum = self.sql.query['rebuildSessionsCacheSum'] + + if len(heros) == 0: + where = '0' + where_summary = '0' + elif len(heros) > 0: + where = str(heros[0]) + where_summary = str(heros[0]) + if len(heros) > 1: + for i in heros: + if i != heros[0]: + where = where + ' OR HandsPlayers.playerId = %s' % str(i) + where_summary = where_summary + ' OR TourneysPlayers.playerId = %s' % str(i) + rebuildSessionsCache = rebuildSessionsCache.replace('', where) + rebuildSessionsCacheSum = rebuildSessionsCacheSum.replace('', where_summary) + + c = self.get_cursor() + c.execute(self.sql.query['clearSessionsCache']) + self.commit() + + sc, gsc = {'bk': []}, {'bk': []} + c.execute(rebuildSessionsCache) + tmp = c.fetchone() + while True: + pids, game, pdata = {}, {}, {} + pdata['pname'] = {} + id = tmp[0] + startTime = tmp[1] + pids['pname'] = tmp[2] + gid = tmp[3] + game['type'] = tmp[4] + pdata['pname']['totalProfit'] = tmp[5] + pdata['pname']['tourneyTypeId'] = tmp[6] + tmp = c.fetchone() + sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None) + gsc = self.storeSessionsCache(id, pids, startTime, game, gid, pdata, sc, gsc, None, heros, tmp == None) + if tmp == None: + for i, id in sc.iteritems(): + if i!='bk': + sid = id['id'] + gid = gsc[i]['id'] + c.execute("UPDATE Hands SET sessionId = %s, gameSessionId = %s WHERE id = %s", (sid, gid, i)) + break + self.commit() + + sc, gsc = {'bk': []}, {'bk': []} + c.execute(rebuildSessionsCacheSum) + tmp = c.fetchone() + while True: + pids, game, info = {}, {}, {} + id = tmp[0] + startTime = tmp[1] + pids['pname'] = tmp[2] + game['type'] = 'summary' + info['tourneyTypeId'] = tmp[3] + info['winnings'] = {} + info['winnings']['pname'] = tmp[4] + info['winningsCurrency'] = {} + info['winningsCurrency']['pname'] = tmp[5] + info['buyinCurrency'] = tmp[6] + info['buyin'] = tmp[7] + info['fee'] = tmp[8] + tmp = c.fetchone() + sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None) + gsc = self.storeSessionsCache(id, pids, startTime, game, None, info, sc, gsc, None, heros, tmp == None) + if tmp == None: + break def get_hero_hudcache_start(self): """fetches earliest stylekey from hudcache for one of hero's player ids""" @@ -1701,6 +1796,34 @@ class Database: # however the calling prog requires. Main aims: # - existing static routines from fpdb_simple just modified + def setThreadId(self, threadid): + self.threadId = threadid + + def acquireLock(self, wait=True, retry_time=.01): + while not self._has_lock: + cursor = self.get_cursor() + cursor.execute(self.sql.query['selectLock']) + record = cursor.fetchall() + self.commit() + if not len(record): + cursor.execute(self.sql.query['switchLock'], (True, self.threadId)) + self.commit() + self._has_lock = True + return True + else: + cursor.execute(self.sql.query['missedLock'], (1, self.threadId)) + self.commit() + if not wait: + return False + sleep(retry_time) + + def releaseLock(self): + if self._has_lock: + cursor = self.get_cursor() + num = cursor.execute(self.sql.query['switchLock'], (False, self.threadId)) + self.commit() + self._has_lock = False + def lock_for_insert(self): """Lock tables in MySQL to try to speed inserts up""" try: @@ -1713,70 +1836,74 @@ class Database: # NEWIMPORT CODE ########################### - def storeHand(self, p, printdata = False): + def storeHand(self, hdata, hbulk, doinsert = False, printdata = False): if printdata: print _("######## Hands ##########") import pprint pp = pprint.PrettyPrinter(indent=4) - pp.pprint(p) + pp.pprint(hdata) print _("###### End Hands ########") - + # Tablename can have odd charachers - p['tableName'] = Charset.to_db_utf8(p['tableName']) + hdata['tableName'] = Charset.to_db_utf8(hdata['tableName']) + + hbulk.append( [ hdata['tableName'], + hdata['siteHandNo'], + hdata['tourneyId'], + hdata['gameTypeId'], + hdata['sessionId'], + hdata['gameSessionId'], + hdata['startTime'], + datetime.utcnow(), #importtime + hdata['seats'], + hdata['maxSeats'], + hdata['texture'], + hdata['playersVpi'], + hdata['boardcard1'], + hdata['boardcard2'], + hdata['boardcard3'], + hdata['boardcard4'], + hdata['boardcard5'], + hdata['playersAtStreet1'], + hdata['playersAtStreet2'], + hdata['playersAtStreet3'], + hdata['playersAtStreet4'], + hdata['playersAtShowdown'], + hdata['street0Raises'], + hdata['street1Raises'], + hdata['street2Raises'], + hdata['street3Raises'], + hdata['street4Raises'], + hdata['street1Pot'], + hdata['street2Pot'], + hdata['street3Pot'], + hdata['street4Pot'], + hdata['showdownPot'], + hdata['id'] + ]) - #stores into table hands: - q = self.sql.query['store_hand'] + if doinsert: + for h in hbulk: + id = h.pop() + if hdata['sc'] and hdata['gsc']: + h[4] = hdata['sc'][id]['id'] + h[5] = hdata['gsc'][id]['id'] + q = self.sql.query['store_hand'] + q = q.replace('%s', self.sql.query['placeholder']) + c = self.get_cursor() + c.executemany(q, hbulk) + self.commit() + return hbulk - q = q.replace('%s', self.sql.query['placeholder']) - - c = self.get_cursor() - - c.execute(q, ( - p['tableName'], - p['siteHandNo'], - p['tourneyId'], - p['gametypeId'], - p['sessionId'], - p['startTime'], - datetime.utcnow(), #importtime - p['seats'], - p['maxSeats'], - p['texture'], - p['playersVpi'], - p['boardcard1'], - p['boardcard2'], - p['boardcard3'], - p['boardcard4'], - p['boardcard5'], - p['playersAtStreet1'], - p['playersAtStreet2'], - p['playersAtStreet3'], - p['playersAtStreet4'], - p['playersAtShowdown'], - p['street0Raises'], - p['street1Raises'], - p['street2Raises'], - p['street3Raises'], - p['street4Raises'], - p['street1Pot'], - p['street2Pot'], - p['street3Pot'], - p['street4Pot'], - p['showdownPot'] - )) - return self.get_last_insert_id(c) - # def storeHand - - def storeHandsPlayers(self, hid, pids, pdata, hp_bulk = None, insert = False, printdata = False): + def storeHandsPlayers(self, hid, pids, pdata, hpbulk, doinsert = False, printdata = False): #print "DEBUG: %s %s %s" %(hid, pids, pdata) if printdata: import pprint pp = pprint.PrettyPrinter(indent=4) pp.pprint(pdata) - inserts = [] for p in pdata: - inserts.append( (hid, + hpbulk.append( ( hid, pids[p], pdata[p]['startCash'], pdata[p]['seatNo'], @@ -1788,6 +1915,19 @@ class Database: pdata[p]['card5'], pdata[p]['card6'], pdata[p]['card7'], + pdata[p]['card8'], + pdata[p]['card9'], + pdata[p]['card10'], + pdata[p]['card11'], + pdata[p]['card12'], + pdata[p]['card13'], + pdata[p]['card14'], + pdata[p]['card15'], + pdata[p]['card16'], + pdata[p]['card17'], + pdata[p]['card18'], + pdata[p]['card19'], + pdata[p]['card20'], pdata[p]['winnings'], pdata[p]['rake'], pdata[p]['totalProfit'], @@ -1882,16 +2022,14 @@ class Database: pdata[p]['street4Raises'] ) ) - if insert: - hp_bulk += inserts + if doinsert: q = self.sql.query['store_hands_players'] q = q.replace('%s', self.sql.query['placeholder']) c = self.get_cursor() - c.executemany(q, hp_bulk) - - return inserts + c.executemany(q, hpbulk) + return hpbulk - def storeHandsActions(self, hid, pids, adata, ha_bulk = None, insert = False, printdata = False): + def storeHandsActions(self, hid, pids, adata, habulk, doinsert = False, printdata = False): #print "DEBUG: %s %s %s" %(hid, pids, adata) # This can be used to generate test data. Currently unused @@ -1899,10 +2037,9 @@ class Database: # import pprint # pp = pprint.PrettyPrinter(indent=4) # pp.pprint(adata) - - inserts = [] + for a in adata: - inserts.append( (hid, + habulk.append( (hid, pids[adata[a]['player']], adata[a]['street'], adata[a]['actionNo'], @@ -1915,17 +2052,15 @@ class Database: adata[a]['cardsDiscarded'], adata[a]['allIn'] ) ) - - if insert: - ha_bulk += inserts + + if doinsert: q = self.sql.query['store_hands_actions'] q = q.replace('%s', self.sql.query['placeholder']) c = self.get_cursor() - c.executemany(q, ha_bulk) + c.executemany(q, habulk) + return habulk - return inserts - - def storeHudCache(self, gid, pids, starttime, pdata): + def storeHudCache(self, gid, pids, starttime, pdata, hcbulk, doinsert = False): """Update cached statistics. If update fails because no record exists, do an insert.""" tz = datetime.utcnow() - datetime.today() @@ -1948,11 +2083,9 @@ class Database: insert_hudcache = insert_hudcache.replace('%s', self.sql.query['placeholder']) #print "DEBUG: %s %s %s" %(hid, pids, pdata) - inserts = [] for p in pdata: #NOTE: Insert new stats at right place because SQL needs strict order line = [] - line.append(1) # HDs line.append(pdata[p]['street0VPI']) line.append(pdata[p]['street0Aggr']) @@ -2041,170 +2174,367 @@ class Database: line.append(pdata[p]['street2Raises']) line.append(pdata[p]['street3Raises']) line.append(pdata[p]['street4Raises']) - - line.append(gid) # gametypeId - line.append(pids[p]) # playerId - line.append(len(pids)) # activeSeats - pos = {'B':'B', 'S':'S', 0:'D', 1:'C', 2:'M', 3:'M', 4:'M', 5:'E', 6:'E', 7:'E', 8:'E', 9:'E' } - line.append(pos[pdata[p]['position']]) - line.append(pdata[p]['tourneyTypeId']) - line.append(styleKey) # styleKey - inserts.append(line) - - - cursor = self.get_cursor() - - for row in inserts: - #convert all True/False values to numeric 0/1 - # needed because columns in hudcache are not BOOL they are INT - # and are being summed if an existing hudcache entry exists - # psycopg2 module does not automatically convert these to numeric. - # mantis bug #93 - for ind in range(len(row)): - if row[ind] == True: row[ind] = 1 - if row[ind] == False: row[ind] = 0 - # Try to do the update first: - num = cursor.execute(update_hudcache, row) - #print "DEBUG: values: %s" % row[-6:] - # Test statusmessage to see if update worked, do insert if not - # num is a cursor in sqlite - if ((self.backend == self.PGSQL and cursor.statusmessage != "UPDATE 1") - or (self.backend == self.MYSQL_INNODB and num == 0) - or (self.backend == self.SQLITE and num.rowcount == 0)): - #move the last 6 items in WHERE clause of row from the end of the array - # to the beginning for the INSERT statement - #print "DEBUG: using INSERT: %s" % num - row = row[-6:] + row[:-6] - num = cursor.execute(insert_hudcache, row) - #print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num - else: - #print "DEBUG: Successfully updated HudCacho using UPDATE" - pass - def storeSessionsCache(self, pids, startTime, game, pdata): - """Update cached sessions. If no record exists, do an insert""" + hc, hcs = {}, [] + hc['gametypeId'] = gid + hc['playerId'] = pids[p] + hc['activeSeats'] = len(pids) + pos = {'B':'B', 'S':'S', 0:'D', 1:'C', 2:'M', 3:'M', 4:'M', 5:'E', 6:'E', 7:'E', 8:'E', 9:'E' } + hc['position'] = pos[pdata[p]['position']] + hc['tourneyTypeId'] = pdata[p]['tourneyTypeId'] + hc['styleKey'] = styleKey + hc['line'] = line + hc['game'] = [hc['gametypeId'] + ,hc['playerId'] + ,hc['activeSeats'] + ,hc['position'] + ,hc['tourneyTypeId'] + ,hc['styleKey']] + hcs.append(hc) + + for h in hcs: + match = False + for b in hcbulk: + if h['game']==b['game']: + b['line'] = [sum(l) for l in zip(b['line'], h['line'])] + match = True + if not match: hcbulk.append(h) + if doinsert: + inserts = [] + exists = [] + updates = [] + for hc in hcbulk: + row = hc['line'] + hc['game'] + if hc['game'] in exists: + updates.append(row) + continue + c = self.get_cursor() + num = c.execute(update_hudcache, row) + # Try to do the update first. Do insert it did not work + if ((self.backend == self.PGSQL and c.statusmessage != "UPDATE 1") + or (self.backend == self.MYSQL_INNODB and num == 0) + or (self.backend == self.SQLITE and num.rowcount == 0)): + inserts.append(hc['game'] + hc['line']) + #row = hc['game'] + hc['line'] + #num = c.execute(insert_hudcache, row) + #print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num + else: + exists.append(hc['game']) + #print "DEBUG: Successfully updated HudCacho using UPDATE" + if inserts: c.executemany(insert_hudcache, inserts) + if updates: c.executemany(update_hudcache, updates) + + return hcbulk + + def prepSessionsCache(self, hid, pids, startTime, sc, heros, doinsert = False): + """Update cached sessions. If no record exists, do an insert""" THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) - select_sessionscache = self.sql.query['select_sessionscache'] - select_sessionscache = select_sessionscache.replace('%s', self.sql.query['placeholder']) - select_sessionscache_mid = self.sql.query['select_sessionscache_mid'] - select_sessionscache_mid = select_sessionscache_mid.replace('%s', self.sql.query['placeholder']) - select_sessionscache_start = self.sql.query['select_sessionscache_start'] - select_sessionscache_start = select_sessionscache_start.replace('%s', self.sql.query['placeholder']) + select_prepSC = self.sql.query['select_prepSC'].replace('%s', self.sql.query['placeholder']) + update_Hands_sid = self.sql.query['update_Hands_sid'].replace('%s', self.sql.query['placeholder']) + update_SC_sid = self.sql.query['update_SC_sid'].replace('%s', self.sql.query['placeholder']) + update_prepSC = self.sql.query['update_prepSC'].replace('%s', self.sql.query['placeholder']) + + #print "DEBUG: %s %s %s" %(hid, pids, pdata) + hand = {} + for p, id in pids.iteritems(): + if id in heros: + hand['startTime'] = startTime.replace(tzinfo=None) + hand['ids'] = [] + + if hand: + id = [] + lower = hand['startTime']-THRESHOLD + upper = hand['startTime']+THRESHOLD + for i in range(len(sc['bk'])): + if ((lower <= sc['bk'][i]['sessionEnd']) + and (upper >= sc['bk'][i]['sessionStart'])): + if ((hand['startTime'] <= sc['bk'][i]['sessionEnd']) + and (hand['startTime'] >= sc['bk'][i]['sessionStart'])): + id.append(i) + elif hand['startTime'] < sc['bk'][i]['sessionStart']: + sc['bk'][i]['sessionStart'] = hand['startTime'] + id.append(i) + elif hand['startTime'] > sc['bk'][i]['sessionEnd']: + sc['bk'][i]['sessionEnd'] = hand['startTime'] + id.append(i) + if len(id) == 1: + id = id[0] + sc['bk'][id]['ids'].append(hid) + elif len(id) == 2: + if sc['bk'][id[0]]['startTime'] < sc['bk'][id[1]]['startTime']: + sc['bk'][id[0]]['endTime'] = sc['bk'][id[1]]['endTime'] + else: + sc['bk'][id[0]]['startTime'] = sc['bk'][id[1]]['startTime'] + sc['bk'].pop[id[1]] + id = id[0] + sc['bk'][id]['ids'].append(hid) + elif len(id) == 0: + hand['id'] = None + hand['sessionStart'] = hand['startTime'] + hand['sessionEnd'] = hand['startTime'] + id = len(sc['bk']) + hand['ids'].append(hid) + sc['bk'].append(hand) + + if doinsert: + c = self.get_cursor() + c.execute("SELECT max(sessionId) FROM SessionsCache") + id = c.fetchone()[0] + if id: sid = id + else: sid = 0 + for i in range(len(sc['bk'])): + lower = sc['bk'][i]['sessionStart'] - THRESHOLD + upper = sc['bk'][i]['sessionEnd'] + THRESHOLD + c.execute(select_prepSC, (lower, upper)) + r = self.fetchallDict(c) + num = len(r) + if (num == 1): + start, end, update = r[0]['sessionStart'], r[0]['sessionEnd'], False + if sc['bk'][i]['sessionStart'] < start: + start, update = sc['bk'][i]['sessionStart'], True + if sc['bk'][i]['sessionEnd'] > end: + end, update = sc['bk'][i]['sessionEnd'], True + if update: + c.execute(update_prepSC, [start, end, r[0]['id']]) + for h in sc['bk'][i]['ids']: + sc[h] = {'id': r[0]['id'], 'data': [start, end]} + elif (num > 1): + start, end, merge, merge_h, merge_sc = None, None, [], [], [] + sid += 1 + r.append(sc['bk'][i]) + for n in r: + if start: + if start > n['sessionStart']: + start = n['sessionStart'] + else: start = n['sessionStart'] + if end: + if end < n['sessionEnd']: + end = n['sessionEnd'] + else: end = n['sessionEnd'] + for n in r: + if n['id']: + if n['id'] in merge: continue + merge.append(n['id']) + merge_h.append([sid, n['id']]) + merge_sc.append([start, end, sid, n['id']]) + c.executemany(update_Hands_sid, merge_h) + c.executemany(update_SC_sid, merge_sc) + for k, v in sc.iteritems(): + if k!='bk' and v['id'] in merge: + sc[k]['id'] = sid + for h in sc['bk'][i]['ids']: + sc[h] = {'id': sid, 'data': [start, end]} + elif (num == 0): + sid += 1 + start = sc['bk'][i]['sessionStart'] + end = sc['bk'][i]['sessionEnd'] + for h in sc['bk'][i]['ids']: + sc[h] = {'id': sid, 'data': [start, end]} + return sc - update_sessionscache_mid = self.sql.query['update_sessionscache_mid'] - update_sessionscache_mid = update_sessionscache_mid.replace('%s', self.sql.query['placeholder']) - update_sessionscache_start = self.sql.query['update_sessionscache_start'] - update_sessionscache_start = update_sessionscache_start.replace('%s', self.sql.query['placeholder']) - update_sessionscache_end = self.sql.query['update_sessionscache_end'] - update_sessionscache_end = update_sessionscache_end.replace('%s', self.sql.query['placeholder']) + def storeSessionsCache(self, hid, pids, startTime, game, gid, pdata, sc, gsc, tz, heros, doinsert = False): + """Update cached sessions. If no record exists, do an insert""" + if not tz: + tz_dt = datetime.utcnow() - datetime.today() + tz = tz_dt.seconds/3600 + + THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) + local = startTime + timedelta(hours=int(tz)) + date = "d%02d%02d%02d" % (local.year - 2000, local.month, local.day) - insert_sessionscache = self.sql.query['insert_sessionscache'] - insert_sessionscache = insert_sessionscache.replace('%s', self.sql.query['placeholder']) - merge_sessionscache = self.sql.query['merge_sessionscache'] - merge_sessionscache = merge_sessionscache.replace('%s', self.sql.query['placeholder']) - delete_sessions = self.sql.query['delete_sessions'] - delete_sessions = delete_sessions.replace('%s', self.sql.query['placeholder']) + select_SC = self.sql.query['select_SC'].replace('%s', self.sql.query['placeholder']) + update_SC = self.sql.query['update_SC'].replace('%s', self.sql.query['placeholder']) + insert_SC = self.sql.query['insert_SC'].replace('%s', self.sql.query['placeholder']) + delete_SC = self.sql.query['delete_SC'].replace('%s', self.sql.query['placeholder']) + update_Hands_gsid = self.sql.query['update_Hands_gsid'].replace('%s', self.sql.query['placeholder']) + + #print "DEBUG: %s %s %s" %(hid, pids, pdata) + hand = {} + for p, id in pids.iteritems(): + if id in heros: + hand['hands'] = 0 + hand['totalProfit'] = 0 + hand['playerId'] = id + hand['gametypeId'] = None + hand['date'] = date + hand['startTime'] = startTime.replace(tzinfo=None) + hand['hid'] = hid + hand['tourneys'] = 0 + hand['tourneyTypeId'] = None + hand['ids'] = [] + if (game['type']=='summary'): + hand['type'] = 'tour' + hand['tourneys'] = 1 + hand['tourneyTypeId'] = pdata['tourneyTypeId'] + if pdata['buyinCurrency'] == pdata['winningsCurrency'][p]: + hand['totalProfit'] = pdata['winnings'][p] - (pdata['buyin'] + pdata['fee']) + else: hand['totalProfit'] = pdata['winnings'][p] + elif (game['type']=='ring'): + hand['type'] = game['type'] + hand['hands'] = 1 + hand['gametypeId'] = gid + hand['totalProfit'] = pdata[p]['totalProfit'] + elif (game['type']=='tour'): + hand['type'] = game['type'] + hand['hands'] = 1 + hand['tourneyTypeId'] = pdata[p]['tourneyTypeId'] - update_hands_sessionid = self.sql.query['update_hands_sessionid'] - update_hands_sessionid = update_hands_sessionid.replace('%s', self.sql.query['placeholder']) + if hand: + id = [] + lower = hand['startTime']-THRESHOLD + upper = hand['startTime']+THRESHOLD + for i in range(len(gsc['bk'])): + if ((hand['date'] == gsc['bk'][i]['date']) + and (hand['gametypeId'] == gsc['bk'][i]['gametypeId']) + and (hand['playerId'] == gsc['bk'][i]['playerId']) + and (hand['tourneyTypeId'] == gsc['bk'][i]['tourneyTypeId'])): + if ((lower <= gsc['bk'][i]['gameEnd']) + and (upper >= gsc['bk'][i]['gameStart'])): + if ((hand['startTime'] <= gsc['bk'][i]['gameEnd']) + and (hand['startTime'] >= gsc['bk'][i]['gameStart'])): + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + elif hand['startTime'] < gsc['bk'][i]['gameStart']: + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + gsc['bk'][i]['gameStart'] = hand['startTime'] + elif hand['startTime'] > gsc['bk'][i]['gameEnd']: + gsc['bk'][i]['hands'] += hand['hands'] + gsc['bk'][i]['tourneys'] += hand['tourneys'] + gsc['bk'][i]['totalProfit'] += hand['totalProfit'] + gsc['bk'][i]['gameEnd'] = hand['startTime'] + id.append(i) + if len(id) == 1: + gsc['bk'][id[0]]['ids'].append(hid) + elif len(id) == 2: + if gsc['bk'][id[0]]['gameStart'] < gsc['bk'][id[1]]['gameStart']: + gsc['bk'][id[0]]['gameEnd'] = gsc['bk'][id[1]]['gameEnd'] + else: gsc['bk'][id[0]]['gameStart'] = gsc['bk'][id[1]]['gameStart'] + gsc['bk'][id[0]]['hands'] += hand['hands'] + gsc['bk'][id[0]]['tourneys'] += hand['tourneys'] + gsc['bk'][id[0]]['totalProfit'] += hand['totalProfit'] + gsc['bk'].pop[id[1]] + gsc['bk'][id[0]]['ids'].append(hid) + elif len(id) == 0: + hand['gameStart'] = hand['startTime'] + hand['gameEnd'] = hand['startTime'] + id = len(gsc['bk']) + hand['ids'].append(hid) + gsc['bk'].append(hand) + if doinsert: + c = self.get_cursor() + for i in range(len(gsc['bk'])): + hid = gsc['bk'][i]['hid'] + sid, start, end = sc[hid]['id'], sc[hid]['data'][0], sc[hid]['data'][1] + lower = gsc['bk'][i]['gameStart'] - THRESHOLD + upper = gsc['bk'][i]['gameEnd'] + THRESHOLD + game = [gsc['bk'][i]['date'] + ,gsc['bk'][i]['type'] + ,gsc['bk'][i]['gametypeId'] + ,gsc['bk'][i]['tourneyTypeId'] + ,gsc['bk'][i]['playerId']] + row = [lower, upper] + game + c.execute(select_SC, row) + r = self.fetchallDict(c) + num = len(r) + if (num == 1): + gstart, gend = r[0]['gameStart'], r[0]['gameEnd'] + if gsc['bk'][i]['gameStart'] < gstart: + gstart = gsc['bk'][i]['gameStart'] + if gsc['bk'][i]['gameEnd'] > gend: + gend = gsc['bk'][i]['gameEnd'] + row = [start, end, gstart, gend + ,gsc['bk'][i]['hands'] + ,gsc['bk'][i]['tourneys'] + ,gsc['bk'][i]['totalProfit'] + ,r[0]['id']] + c.execute(update_SC, row) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': r[0]['id']} + elif (num > 1): + gstart, gend, hands, tourneys, totalProfit, delete, merge = None, None, 0, 0, 0, [], [] + for n in r: delete.append(n['id']) + delete.sort() + for d in delete: c.execute(delete_SC, d) + r.append(gsc['bk'][i]) + for n in r: + if gstart: + if gstart > n['gameStart']: + gstart = n['gameStart'] + else: gstart = n['gameStart'] + if gend: + if gend < n['gameEnd']: + gend = n['gameEnd'] + else: gend = n['gameEnd'] + hands += n['hands'] + tourneys += n['tourneys'] + totalProfit += n['totalProfit'] + row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit] + c.execute(insert_SC, row) + gsid = self.get_last_insert_id(c) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid} + for m in delete: merge.append([gsid, m]) + c.executemany(update_Hands_gsid, merge) + elif (num == 0): + gstart = gsc['bk'][i]['gameStart'] + gend = gsc['bk'][i]['gameEnd'] + hands = gsc['bk'][i]['hands'] + tourneys = gsc['bk'][i]['tourneys'] + totalProfit = gsc['bk'][i]['totalProfit'] + row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit] + c.execute(insert_SC, row) + gsid = self.get_last_insert_id(c) + for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid} + else: + # Something bad happened + pass + self.commit() + + return gsc + + def getHeroIds(self, pids, sitename): #Grab playerIds using hero names in HUD_Config.xml try: # derive list of program owner's player ids - self.hero = {} # name of program owner indexed by site id - self.hero_ids = [] + hero = {} # name of program owner indexed by site id + hero_ids = [] # make sure at least two values in list # so that tuple generation creates doesn't use # () or (1,) style for site in self.config.get_supported_sites(): - result = self.get_site_id(site) - if result: - site_id = result[0][0] - self.hero[site_id] = self.config.supported_sites[site].screen_name - p_id = self.get_player_id(self.config, site, self.hero[site_id]) - if p_id: - self.hero_ids.append(int(p_id)) - + hero = self.config.supported_sites[site].screen_name + for n, v in pids.iteritems(): + if n == hero and sitename == site: + hero_ids.append(v) + except: err = traceback.extract_tb(sys.exc_info()[2])[-1] - print _("Error aquiring hero ids:"), str(sys.exc_value) - print err - - inserts = [] - for p in pdata: - if pids[p] in self.hero_ids: - line = [0]*5 + #print _("Error aquiring hero ids:"), str(sys.exc_value) + return hero_ids - if (game['type']=='ring'): line[0] = 1 # count ring hands - if (game['type']=='tour'): line[1] = 1 # count tour hands - if (game['type']=='ring' and game['currency']=='USD'): line[2] = pdata[p]['totalProfit'] #sum of ring profit in USD - if (game['type']=='ring' and game['currency']=='EUR'): line[3] = pdata[p]['totalProfit'] #sum of ring profit in EUR - line[4] = startTime - inserts.append(line) - - cursor = self.get_cursor() - id = None - - for row in inserts: - threshold = [] - threshold.append(row[-1]-THRESHOLD) - threshold.append(row[-1]+THRESHOLD) - cursor.execute(select_sessionscache, threshold) - session_records = cursor.fetchall() - num = len(session_records) - if (num == 1): - id = session_records[0][0] #grab the sessionId - # Try to do the update first: - #print "DEBUG: found 1 record to update" - update_mid = row + row[-1:] - cursor.execute(select_sessionscache_mid, update_mid[-2:]) - mid = len(cursor.fetchall()) - if (mid == 0): - update_startend = row[-1:] + row + threshold - cursor.execute(select_sessionscache_start, update_startend[-3:]) - start = len(cursor.fetchall()) - if (start == 0): - #print "DEBUG:", start, " start record found. Update stats and start time" - cursor.execute(update_sessionscache_end, update_startend) - else: - #print "DEBUG: 1 end record found. Update stats and end time time" - cursor.execute(update_sessionscache_start, update_startend) - else: - #print "DEBUG: update stats mid-session" - cursor.execute(update_sessionscache_mid, update_mid) - elif (num > 1): - session_ids = [session_records[0][0], session_records[1][0]] - session_ids.sort() - # Multiple matches found - merge them into one session and update: - # - Obtain the session start and end times for the new combined session - cursor.execute(merge_sessionscache, session_ids) - merge = cursor.fetchone() - # - Delete the old records - for id in session_ids: - cursor.execute(delete_sessions, id) - # - Insert the new updated record - cursor.execute(insert_sessionscache, merge) - # - Obtain the new sessionId and write over the old ids in Hands - id = self.get_last_insert_id(cursor) #grab the sessionId - update_hands = [id] + session_ids - cursor.execute(update_hands_sessionid, update_hands) - # - Update the newly combined record in SessionsCache with data from this hand - update_mid = row + row[-1:] - cursor.execute(update_sessionscache_mid, update_mid) - elif (num == 0): - # No matches found, insert new session: - insert = row + row[-1:] - insert = insert[-2:] + insert[:-2] - #print "DEBUG: No matches found. Insert record", insert - cursor.execute(insert_sessionscache, insert) - id = self.get_last_insert_id(cursor) #grab the sessionId - else: - # Something bad happened - pass - - return id + def fetchallDict(self, cursor): + data = cursor.fetchall() + if not data: return [] + desc = cursor.description + results = [0]*len(data) + for i in range(len(data)): + results[i] = {} + for n in range(len(desc)): + name = desc[n][0] + results[i][name] = data[i][n] + return results + + def nextHandId(self): + c = self.get_cursor() + c.execute("SELECT max(id) FROM Hands") + id = c.fetchone()[0] + if not id: id = 0 + id += 1 + return id def isDuplicate(self, gametypeID, siteHandNo): dup = False diff --git a/pyfpdb/DerivedStats.py b/pyfpdb/DerivedStats.py index 262a3fee..d29a33ad 100644 --- a/pyfpdb/DerivedStats.py +++ b/pyfpdb/DerivedStats.py @@ -119,16 +119,17 @@ class DerivedStats(): return self.handsactions def assembleHands(self, hand): - self.hands['tableName'] = hand.tablename - self.hands['siteHandNo'] = hand.handid - self.hands['gametypeId'] = None # Leave None, handled later after checking db - self.hands['sessionId'] = None # Leave None, added later if caching sessions - self.hands['startTime'] = hand.startTime # format this! - self.hands['importTime'] = None - self.hands['seats'] = self.countPlayers(hand) - self.hands['maxSeats'] = hand.maxseats - self.hands['texture'] = None # No calculation done for this yet. - self.hands['tourneyId'] = hand.tourneyId + self.hands['tableName'] = hand.tablename + self.hands['siteHandNo'] = hand.handid + self.hands['gametypeId'] = None # Leave None, handled later after checking db + self.hands['sessionId'] = None # Leave None, added later if caching sessions + self.hands['gameSessionId'] = None # Leave None, added later if caching sessions + self.hands['startTime'] = hand.startTime # format this! + self.hands['importTime'] = None + self.hands['seats'] = self.countPlayers(hand) + self.hands['maxSeats'] = hand.maxseats + self.hands['texture'] = None # No calculation done for this yet. + self.hands['tourneyId'] = hand.tourneyId # This (i think...) is correct for both stud and flop games, as hand.board['street'] disappears, and # those values remain default in stud. @@ -213,10 +214,10 @@ class DerivedStats(): for player in hand.players: hcs = hand.join_holecards(player[1], asList=True) - hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x'] - #for i, card in enumerate(hcs[:7], 1): #Python 2.6 syntax + hcs = hcs + [u'0x']*18 + #for i, card in enumerate(hcs[:20, 1): #Python 2.6 syntax # self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card) - for i, card in enumerate(hcs[:7]): + for i, card in enumerate(hcs[:20]): self.handsplayers[player[1]]['card%s' % (i+1)] = Card.encodeCard(card) self.handsplayers[player[1]]['startCards'] = Card.calcStartCards(hand, player[1]) diff --git a/pyfpdb/EverestToFpdb.py b/pyfpdb/EverestToFpdb.py index 662d2889..befec65f 100644 --- a/pyfpdb/EverestToFpdb.py +++ b/pyfpdb/EverestToFpdb.py @@ -35,7 +35,7 @@ class Everest(HandHistoryConverter): siteID = 15 substitutions = { - 'LS' : u"\$|\xe2\x82\xac|\u20ac|", + 'LS' : u"\$|\u20ac|", 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename } diff --git a/pyfpdb/FullTiltPokerSummary.py b/pyfpdb/FullTiltPokerSummary.py index 98169393..d1bfd917 100644 --- a/pyfpdb/FullTiltPokerSummary.py +++ b/pyfpdb/FullTiltPokerSummary.py @@ -25,7 +25,6 @@ import datetime from Exceptions import FpdbParseError from HandHistoryConverter import * -import PokerStarsToFpdb from TourneySummary import * class FullTiltPokerSummary(TourneySummary): @@ -46,7 +45,7 @@ class FullTiltPokerSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|", # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|", # legal currency symbols - Euro(cp1252, utf-8) 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename 'NUM' : u".,\d", # legal characters in number format } diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index a13cca50..a023a043 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -172,7 +172,7 @@ class Fulltilt(HandHistoryConverter): self.re_BringIn = re.compile(r"^%(PLAYERS)s brings in for [%(LS)s]?(?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_PostBoth = re.compile(r"^%(PLAYERS)s posts small \& big blinds \[[%(LS)s]? (?P[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_HeroCards = re.compile(r"^Dealt to %s(?: \[(?P.+?)\])?( \[(?P.+?)\])" % player_re, re.MULTILINE) - self.re_Action = re.compile(r"^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds)( [%(LS)s]?(?P[%(NUM)s]+))?" % self.substitutions, re.MULTILINE) + self.re_Action = re.compile(r"^%(PLAYERS)s(?P bets| checks| raises to| completes it to| calls| folds| discards| stands pat)( [%(LS)s]?(?P[%(NUM)s]+))?(\son|\scards?)?(\s\[(?P.+?)\])?" % self.substitutions, re.MULTILINE) self.re_ShowdownAction = re.compile(r"^%s shows \[(?P.*)\]" % player_re, re.MULTILINE) self.re_CollectPot = re.compile(r"^Seat (?P[0-9]+): %(PLAYERS)s (\(button\) |\(small blind\) |\(big blind\) )?(collected|showed \[.*\] and won) \([%(LS)s]?(?P[%(NUM)s]+)\)(, mucked| with.*)?" % self.substitutions, re.MULTILINE) self.re_SitsOut = re.compile(r"^%s sits out" % player_re, re.MULTILINE) @@ -483,6 +483,10 @@ class Fulltilt(HandHistoryConverter): hand.addFold( street, action.group('PNAME')) elif action.group('ATYPE') == ' checks': hand.addCheck( street, action.group('PNAME')) + elif action.group('ATYPE') == ' discards': + hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS')) + elif action.group('ATYPE') == ' stands pat': + hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS')) else: print _("FullTilt: DEBUG: unimplemented readAction: '%s' '%s'") %(action.group('PNAME'),action.group('ATYPE'),) diff --git a/pyfpdb/GuiTourneyImport.py b/pyfpdb/GuiTourneyImport.py index 0bc71f3d..0ed33b18 100755 --- a/pyfpdb/GuiTourneyImport.py +++ b/pyfpdb/GuiTourneyImport.py @@ -223,13 +223,18 @@ class SummaryImporter: print "Found %s summaries" %(len(summaryTexts)) errors = 0 imported = 0 + ####Lock Placeholder#### for j, summaryText in enumerate(summaryTexts, start=1): + sc, gsc = {'bk': []}, {'bk': []} + doinsert = len(summaryTexts)==j try: - conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") + conv = obj(db=self.database, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") + sc, gsc = conv.updateSessionsCache(sc, gsc, None, doinsert) except FpdbParseError, e: errors += 1 print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts)) imported = j + ####Lock Placeholder#### return (imported - errors, errors) def clearFileList(self): diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 01b5c70e..7f89c050 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -57,6 +57,7 @@ class Hand(object): #log.debug( _("Hand.init(): handText is ") + str(handText) ) self.config = config self.saveActions = self.config.get_import_parameters().get('saveActions') + self.callHud = self.config.get_import_parameters().get("callFpdbHud") self.cacheSessions = self.config.get_import_parameters().get("cacheSessions") #log = Configuration.get_logger("logging.conf", "db", log_dir=self.config.dir_log) self.sitename = sitename @@ -227,83 +228,77 @@ dealt whether they were seen in a 'dealt to' line self.holecards[street][player] = [open, closed] - def prepInsert(self, db, printtest = False): + def prepInsert(self, db): ##### # Players, Gametypes, TourneyTypes are all shared functions that are needed for additional tables # These functions are intended for prep insert eventually ##### - # Players - base playerid and siteid tuple self.dbid_pids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId) - - #Gametypes - hilo = "h" - if self.gametype['category'] in ['studhilo', 'omahahilo']: - hilo = "s" - elif self.gametype['category'] in ['razz','27_3draw','badugi', '27_1draw']: - hilo = "l" - - self.gametyperow = (self.siteId, self.gametype['currency'], self.gametype['type'], self.gametype['base'], - self.gametype['category'], self.gametype['limitType'], hilo, - int(Decimal(self.gametype['sb'])*100), int(Decimal(self.gametype['bb'])*100), - int(Decimal(self.gametype['bb'])*100), int(Decimal(self.gametype['bb'])*200)) - # Note: the above data is calculated in db.getGameTypeId - # Only being calculated above so we can grab the testdata - self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype, printdata = printtest) - + self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype) + if self.tourNo!=None: self.tourneyTypeId = db.createTourneyType(self) - db.commit() self.tourneyId = db.createOrUpdateTourney(self, "HHC") - db.commit() self.tourneysPlayersIds = db.createOrUpdateTourneysPlayers(self, "HHC") - db.commit() - #end def prepInsert - - def insert(self, db, hp_data = None, ha_data = None, insert_data=False, printtest = False): - """ Function to insert Hand into database -Should not commit, and do minimal selects. Callers may want to cache commits -db: a connected Database object""" - - + #db.commit() #commit these transactions' + + def assembleHand(self): self.stats.getStats(self) - - ##### - # End prep functions - ##### - hh = self.stats.getHands() - hp_inserts, ha_inserts = [], [] - - if not db.isDuplicate(self.dbid_gt, hh['siteHandNo']): - # Hands - Summary information of hand indexed by handId - gameinfo - hh['gametypeId'] = self.dbid_gt - # seats TINYINT NOT NULL, - hh['seats'] = len(self.dbid_pids) - - hp = self.stats.getHandsPlayers() - - if self.cacheSessions: - hh['sessionId'] = db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, hp) - - self.dbid_hands = db.storeHand(hh, printdata = printtest) - - hp_inserts = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, hp, - insert=insert_data, hp_bulk = hp_data, printdata = printtest) - - if self.saveActions: - ha_inserts = db.storeHandsActions(self.dbid_hands, self.dbid_pids, self.stats.getHandsActions(), - insert=insert_data, ha_bulk = ha_data, printdata = printtest) - else: - log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) + self.hands = self.stats.getHands() + self.handsplayers = self.stats.getHandsPlayers() + + def getHandId(self, db, id): + if db.isDuplicate(self.dbid_gt, self.hands['siteHandNo']): + #log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) self.is_duplicate = True # i.e. don't update hudcache - raise FpdbHandDuplicate(hh['siteHandNo']) - - return hp_inserts, ha_inserts + next = id + raise FpdbHandDuplicate(self.hands['siteHandNo']) + else: + self.dbid_hands = id + self.hands['id'] = self.dbid_hands + next = id +1 + return next - def updateHudCache(self, db): - db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.stats.getHandsPlayers()) + def insertHands(self, db, hbulk, doinsert = False, printtest = False): + """ Function to insert Hand into database + Should not commit, and do minimal selects. Callers may want to cache commits + db: a connected Database object""" + self.hands['gameTypeId'] = self.dbid_gt + self.hands['seats'] = len(self.dbid_pids) + hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest) + return hbulk + + def insertHandsPlayers(self, db, hpbulk, doinsert = False, printtest = False): + """ Function to inserts HandsPlayers into database""" + hpbulk = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.handsplayers, hpbulk, doinsert, printtest) + return hpbulk + + def insertHandsActions(self, db, habulk, doinsert = False, printtest = False): + """ Function to inserts HandsActions into database""" + handsactions = self.stats.getHandsActions() + habulk = db.storeHandsActions(self.dbid_hands, self.dbid_pids, handsactions, habulk, doinsert, printtest) + return habulk + + def updateHudCache(self, db, hcbulk, doinsert = False): + """ Function to update the HudCache""" + if self.callHud: + hcbulk = db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.handsplayers, hcbulk, doinsert) + return hcbulk - def updateSessionsCache(self, db): - db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, self.stats.getHandsPlayers()) + def updateSessionsCache(self, db, sc, gsc, tz, doinsert = False): + """ Function to update the SessionsCache""" + if self.cacheSessions: + self.heros = db.getHeroIds(self.dbid_pids, self.sitename) + sc = db.prepSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, sc, self.heros, doinsert) + gsc = db.storeSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, self.gametype + ,self.dbid_gt, self.handsplayers, sc, gsc, tz, self.heros, doinsert) + if doinsert: + self.hands['sc'] = sc + self.hands['gsc'] = gsc + else: + self.hands['sc'] = None + self.hands['gsc'] = None + return sc, gsc def select(self, db, handId): """ Function to create Hand object from database """ @@ -666,10 +661,13 @@ Add a raise on [street] by [player] to [amountTo] self.pot.addMoney(player, amount) - def addStandsPat(self, street, player): + def addStandsPat(self, street, player, cards): self.checkPlayerExists(player) act = (player, 'stands pat') self.actions[street].append(act) + if cards: + cards = cards.split(' ') + self.addHoleCards(street, player, open=[], closed=cards) def addFold(self, street, player): @@ -1229,7 +1227,14 @@ class DrawHand(Hand): def join_holecards(self, player, asList=False): """With asList = True it returns the set cards for a player including down cards if they aren't know""" # FIXME: This should actually return - holecards = [u'0x', u'0x', u'0x', u'0x', u'0x'] + holecards = [u'0x']*20 + + for i, street in enumerate(self.holeStreets): + if player in self.holecards[street].keys(): + allhole = self.holecards[street][player][0] + self.holecards[street][player][1] + for c in range(len(allhole)): + idx = c + (i*5) + holecards[idx] = allhole[c] if asList == False: return " ".join(holecards) diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index b41f9342..6d25d6aa 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -50,9 +50,6 @@ import Hand from Exceptions import FpdbParseError import Configuration -import pygtk -import gtk - class HandHistoryConverter(): READ_CHUNK_SIZE = 10000 # bytes to read at a time from file in tail mode @@ -128,9 +125,6 @@ If in follow mode, wait for more data to turn up. Otherwise, finish at EOF. """ - while gtk.events_pending(): - gtk.main_iteration(False) - starttime = time.time() if not self.sanityCheck(): log.warning(_("Failed sanity check")) @@ -182,7 +176,12 @@ Otherwise, finish at EOF. finally: if self.out_fh != sys.stdout: self.out_fh.close() - + + def progressNotify(self): + "A callback to the interface while events are pending" + import gtk, pygtk + while gtk.events_pending(): + gtk.main_iteration(False) def tailHands(self): """Generator of handTexts from a tailed file: diff --git a/pyfpdb/OnGameToFpdb.py b/pyfpdb/OnGameToFpdb.py index fd5efe31..32874d41 100755 --- a/pyfpdb/OnGameToFpdb.py +++ b/pyfpdb/OnGameToFpdb.py @@ -42,12 +42,12 @@ class OnGame(HandHistoryConverter): siteId = 5 # Needs to match id entry in Sites database mixes = { } # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20ac" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\\u20AC" # legal currency symbols - Euro(cp1252, utf-8) } - currencies = { u'\u20ac':'EUR', u'\xe2\x82\xac':'EUR', '$':'USD', '':'T$' } + currencies = { u'\u20AC':'EUR', '$':'USD', '':'T$' } limits = { 'NO_LIMIT':'nl', 'POT_LIMIT':'pl', 'LIMIT':'fl'} diff --git a/pyfpdb/PacificPokerToFpdb.py b/pyfpdb/PacificPokerToFpdb.py index a46ea739..d0321492 100644 --- a/pyfpdb/PacificPokerToFpdb.py +++ b/pyfpdb/PacificPokerToFpdb.py @@ -39,10 +39,10 @@ class PacificPoker(HandHistoryConverter): siteId = 13 # Needs to match id entry in Sites database mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } # translations from captured groups to fpdb info strings diff --git a/pyfpdb/PartyPokerToFpdb.py b/pyfpdb/PartyPokerToFpdb.py index a2f708a1..e4e3fc95 100755 --- a/pyfpdb/PartyPokerToFpdb.py +++ b/pyfpdb/PartyPokerToFpdb.py @@ -46,10 +46,10 @@ class PartyPoker(HandHistoryConverter): siteId = 9 filetype = "text" sym = {'USD': "\$", 'EUR': u"\u20ac", 'T$': ""} - currencies = {"\$": "USD", "$": "USD", u"\xe2\x82\xac": "EUR", u"\u20ac": "EUR", '': "T$"} + currencies = {"\$": "USD", "$": "USD", u"\u20ac": "EUR", '': "T$"} substitutions = { 'LEGAL_ISO' : "USD|EUR", # legal ISO currency codes - 'LS' : u"\$|\u20ac|\xe2\x82\xac|", # Currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20ac|", # Currency symbols - Euro(cp1252, utf-8) 'NUM' : u".,\d", } diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index b1d31a26..c5e0c0cb 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -45,7 +45,7 @@ class PokerStarsSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } re_SplitTourneys = re.compile("PokerStars Tournament ") diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 4d89bff1..d6abca84 100644 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -39,10 +39,10 @@ class PokerStars(HandHistoryConverter): siteId = 2 # Needs to match id entry in Sites database mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } # translations from captured groups to fpdb info strings @@ -145,7 +145,8 @@ class PokerStars(HandHistoryConverter): (\s(%(CUR)s)?(?P[.\d]+))?(\sto\s%(CUR)s(?P[.\d]+))? # the number discarded goes in \s*(and\sis\sall.in)? (and\shas\sreached\sthe\s[%(CUR)s\d\.]+\scap)? - (\scards?(\s\[(?P.+?)\])?)?\s*$""" + (\son|\scards?)? + (\s\[(?P.+?)\])?\s*$""" % short_subst, re.MULTILINE|re.VERBOSE) re_ShowdownAction = re.compile(r"^%s: shows \[(?P.*)\]" % short_subst['PLYR'], re.MULTILINE) re_sitsOut = re.compile("^%s sits out" % short_subst['PLYR'], re.MULTILINE) @@ -432,9 +433,9 @@ class PokerStars(HandHistoryConverter): elif action.group('ATYPE') == ' checks': hand.addCheck( street, action.group('PNAME')) elif action.group('ATYPE') == ' discards': - hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('DISCARDED')) + hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS')) elif action.group('ATYPE') == ' stands pat': - hand.addStandsPat( street, action.group('PNAME')) + hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS')) else: print (_("DEBUG: ") + _("Unimplemented readAction: '%s' '%s'") % (action.group('PNAME'),action.group('ATYPE'))) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 69f4a0b2..92c89f29 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -107,6 +107,15 @@ class Sql: elif db_server == 'sqlite': self.query['createSettingsTable'] = """CREATE TABLE Settings (version INTEGER NOT NULL) """ + + ################################ + # Create InsertLock + ################################ + if db_server == 'mysql': + self.query['createLockTable'] = """CREATE TABLE InsertLock ( + id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), + locked BOOLEAN NOT NULL DEFAULT FALSE) + ENGINE=INNODB""" ################################ # Create RawHands (this table is all but identical with RawTourneys) @@ -346,7 +355,8 @@ class Sql: siteHandNo BIGINT NOT NULL, tourneyId INT UNSIGNED, gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), - sessionId INT UNSIGNED, + sessionId INT UNSIGNED, + gameSessionId INT UNSIGNED, startTime DATETIME NOT NULL, importTime DATETIME NOT NULL, seats TINYINT NOT NULL, @@ -385,6 +395,7 @@ class Sql: tourneyId INT, gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), sessionId INT, + gameSessionId INT, startTime timestamp without time zone NOT NULL, importTime timestamp without time zone NOT NULL, seats SMALLINT NOT NULL, @@ -422,6 +433,7 @@ class Sql: tourneyId INT, gametypeId INT NOT NULL, sessionId INT, + gameSessionId INT, startTime REAL NOT NULL, importTime REAL NOT NULL, seats INT NOT NULL, @@ -621,6 +633,19 @@ class Sql: card5 smallint, card6 smallint, card7 smallint, + card8 smallint, /* cards 8-20 for draw hands */ + card9 smallint, + card10 smallint, + card11 smallint, + card12 smallint, + card13 smallint, + card14 smallint, + card15 smallint, + card16 smallint, + card17 smallint, + card18 smallint, + card19 smallint, + card20 smallint, startCards smallint, ante INT, @@ -748,6 +773,19 @@ class Sql: card5 smallint, card6 smallint, card7 smallint, + card8 smallint, /* cards 8-20 for draw hands */ + card9 smallint, + card10 smallint, + card11 smallint, + card12 smallint, + card13 smallint, + card14 smallint, + card15 smallint, + card16 smallint, + card17 smallint, + card18 smallint, + card19 smallint, + card20 smallint, startCards smallint, ante INT, @@ -874,6 +912,19 @@ class Sql: card5 INT, card6 INT, card7 INT, + card8 INT, /* cards 8-20 for draw hands */ + card9 INT, + card10 INT, + card11 INT, + card12 INT, + card13 INT, + card14 INT, + card15 INT, + card16 INT, + card17 INT, + card18 INT, + card19 INT, + card20 INT, startCards INT, ante INT, @@ -1430,32 +1481,61 @@ class Sql: id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), sessionStart DATETIME NOT NULL, sessionEnd DATETIME NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) - - ENGINE=INNODB""" + gameStart DATETIME NOT NULL, + gameEnd DATETIME NOT NULL, + sessionId BIGINT, + date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type char(7) NOT NULL, + gametypeId SMALLINT UNSIGNED, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), + tourneyTypeId SMALLINT UNSIGNED, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), + playerId INT UNSIGNED NOT NULL, FOREIGN KEY (playerId) REFERENCES Players(id), + hands INT NOT NULL, + tourneys INT NOT NULL, + totalProfit INT) + ENGINE=INNODB + """ + elif db_server == 'postgresql': self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( id BIGSERIAL, PRIMARY KEY (id), sessionStart REAL NOT NULL, sessionEnd REAL NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) + gameStart REAL NOT NULL, + gameEnd REAL NOT NULL, + sessionId INT, + date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type char(7), + gametypeId INT, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), + tourneyTypeId INT, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), + playerId INT, FOREIGN KEY (playerId) REFERENCES Players(id), + hands INT, + tourneys INT, + totalProfit INT) """ + elif db_server == 'sqlite': self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( id INTEGER PRIMARY KEY, sessionStart REAL NOT NULL, sessionEnd REAL NOT NULL, - ringHDs INT NOT NULL, - tourHDs INT NOT NULL, - ringProfitUSD INT NOT NULL, - ringProfitEUR INT NOT NULL) + gameStart REAL NOT NULL, + gameEnd REAL NOT NULL, + sessionId INT, + date TEXT NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + type TEXT, + gametypeId INT, + tourneyTypeId INT, + playerId INT, + hands INT, + tourneys INT, + totalProfit INT) """ + + self.query['addSessionIdIndex'] = """CREATE INDEX index_SessionId ON SessionsCache (sessionId)""" + + self.query['addHandsSessionIdIndex'] = """CREATE INDEX index_handsSessionId ON Hands (sessionId)""" + + self.query['addHandsGameSessionIdIndex'] = """CREATE INDEX index_handsGameSessionId ON Hands (gameSessionId)""" if db_server == 'mysql': self.query['addTourneyIndex'] = """ALTER TABLE Tourneys ADD UNIQUE INDEX siteTourneyNo(siteTourneyNo, tourneyTypeId)""" @@ -1479,7 +1559,7 @@ class Sql: self.query['addPlayersIndex'] = """CREATE UNIQUE INDEX name ON Players (name, siteId)""" if db_server == 'mysql': - self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX tourneyId(tourneyId, playerId)""" + self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX _tourneyId(tourneyId, playerId)""" elif db_server == 'postgresql': self.query['addTPlayersIndex'] = """CREATE UNIQUE INDEX tourneyId ON TourneysPlayers (tourneyId, playerId)""" elif db_server == 'sqlite': @@ -4142,7 +4222,7 @@ class Sql: """ self.query['insert_hudcache'] = """ - INSERT INTO HudCache ( + insert into HudCache ( gametypeId, playerId, activeSeats, @@ -4237,7 +4317,7 @@ class Sql: street2Raises, street3Raises, street4Raises) - VALUES (%s, %s, %s, %s, %s, + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, @@ -4363,96 +4443,136 @@ class Sql: #################################### # Queries to rebuild/modify sessionscache #################################### + + self.query['clearSessionsCache'] = """DELETE FROM SessionsCache""" + + self.query['rebuildSessionsCache'] = """ + SELECT Hands.id as id, + Hands.startTime as startTime, + HandsPlayers.playerId as playerId, + Hands.gametypeId as gametypeId, + Gametypes.type as game, + HandsPlayers.totalProfit as totalProfit, + Tourneys.tourneyTypeId as tourneyTypeId + FROM Gametypes, HandsPlayers, Hands + LEFT JOIN Tourneys ON Hands.tourneyId = Tourneys.tourneyTypeId + WHERE HandsPlayers.handId = Hands.id + AND Hands.gametypeId = Gametypes.id + AND (case when HandsPlayers.playerId = then 1 else 0 end) = 1 + ORDER BY Hands.startTime ASC""" + + self.query['rebuildSessionsCacheSum'] = """ + SELECT Tourneys.id as id, + Tourneys.startTime as startTime, + TourneysPlayers.playerId, + TourneyTypes.id as tourneyTypeId, + TourneysPlayers.winnings as winnings, + TourneysPlayers.winningsCurrency as winningsCurrency, + TourneyTypes.currency as buyinCurrency, + TourneyTypes.buyIn as buyIn, + TourneyTypes.fee as fee, + case when TourneyTypes.rebuy then TourneyTypes.rebuyCost else 0 end as rebuyCost, + case when TourneyTypes.rebuy then TourneyTypes.rebuyFee else 0 end as rebuyFee, + case when TourneyTypes.addOn then TourneyTypes.addOnCost else 0 end as addOnCost, + case when TourneyTypes.addOn then TourneyTypes.addOnFee else 0 end as addOnFee, + case when TourneyTypes.knockout then TourneyTypes.koBounty else 0 end as koBounty + FROM Tourneys, TourneyTypes, TourneysPlayers + WHERE Tourneys.tourneyTypeId = TourneyTypes.id + AND Tourneys.id = TourneysPlayers.tourneyId + AND (case when TourneysPlayers.playerId = then 1 else 0 end) = 1 + ORDER BY Tourneys.startTime ASC""" - self.query['select_sessionscache'] = """ - SELECT id, - sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['select_sessionscache_mid'] = """ - SELECT sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['select_sessionscache_start'] = """ - SELECT sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR - FROM SessionsCache - WHERE sessionStart>%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['update_sessionscache_mid'] = """ - UPDATE SessionsCache SET - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionStart<=%s - AND sessionEnd>=%s""" - - self.query['update_sessionscache_start'] = """ - UPDATE SessionsCache SET - sessionStart=%s, - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionStart>%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['update_sessionscache_end'] = """ - UPDATE SessionsCache SET - sessionEnd=%s, - ringHDs=ringHDs+%s, - tourHDs=tourHDs+%s, - ringProfitUSD=ringProfitUSD+%s, - ringProfitEUR=ringProfitEUR+%s - WHERE sessionEnd<%s - AND sessionEnd>=%s - AND sessionStart<=%s""" - - self.query['insert_sessionscache'] = """ - INSERT INTO SessionsCache ( - sessionStart, - sessionEnd, - ringHDs, - tourHDs, - ringProfitUSD, - ringProfitEUR) - VALUES (%s, %s, %s, %s, %s, %s)""" - - self.query['merge_sessionscache'] = """ - SELECT min(sessionStart), max(sessionEnd), sum(ringHDs), sum(tourHDs), sum(ringProfitUSD), sum(ringProfitEUR) - FROM SessionsCache - WHERE (case when id=%s or id=%s then 1 else 0 end)=1""" - - self.query['delete_sessions'] = """ - DELETE FROM SessionsCache - WHERE id=%s""" - - self.query['update_hands_sessionid'] = """ - UPDATE Hands SET - sessionId=%s - WHERE (case when sessionId=%s or sessionId=%s then 1 else 0 end)=1""" + self.query['select_prepSC'] = """ + SELECT sessionId as id, + sessionStart, + sessionEnd, + count(sessionId) as count + FROM SessionsCache + WHERE sessionEnd>=%s + AND sessionStart<=%s + GROUP BY sessionId, sessionStart, sessionEnd""" + + self.query['update_prepSC'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s + WHERE sessionId=%s""" + + self.query['update_SC'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s, + gameStart=%s, + gameEnd=%s, + hands=hands+%s, + tourneys=tourneys+%s, + totalProfit=totalProfit+%s + WHERE id=%s""" + + self.query['select_SC'] = """ + SELECT id, + sessionStart, + sessionEnd, + gameStart, + gameEnd, + sessionId, + date, + type, + gametypeId, + tourneyTypeId, + playerId, + hands, + tourneys, + totalProfit + FROM SessionsCache + WHERE gameEnd>=%s + AND gameStart<=%s + AND date=%s + AND type=%s + AND (case when gametypeId is NULL then 1 else + (case when gametypeId=%s then 1 else 0 end) end)=1 + AND (case when tourneyTypeId is NULL then 1 else + (case when tourneyTypeId=%s then 1 else 0 end) end)=1 + AND playerId=%s""" + + self.query['insert_SC'] = """ + insert into SessionsCache ( + sessionStart, + sessionEnd, + gameStart, + gameEnd, + sessionId, + date, + type, + gametypeId, + tourneyTypeId, + playerId, + hands, + tourneys, + totalProfit) + values (%s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s)""" + + self.query['update_Hands_gsid'] = """ + UPDATE Hands SET + gameSessionId=%s + WHERE gameSessionId=%s""" + + self.query['update_Hands_sid'] = """ + UPDATE Hands SET + sessionId=%s + WHERE sessionId=%s""" + + self.query['update_SC_sid'] = """ + UPDATE SessionsCache SET + sessionStart=%s, + sessionEnd=%s, + sessionId=%s + WHERE sessionId=%s""" + + self.query['delete_SC'] = """ + DELETE FROM SessionsCache + WHERE id=%s""" #################################### # Database management queries @@ -4467,6 +4587,25 @@ class Sql: self.query['analyze'] = "analyze" elif db_server == 'sqlite': self.query['analyze'] = "analyze" + + if db_server == 'mysql': + self.query['selectLock'] = """ + SELECT locked + FROM InsertLock + WHERE locked=True + LOCK IN SHARE MODE""" + + if db_server == 'mysql': + self.query['switchLock'] = """ + UPDATE InsertLock SET + locked=%s + WHERE id=%s""" + + if db_server == 'mysql': + self.query['missedLock'] = """ + UPDATE InsertLock SET + missed=missed+%s + WHERE id=%s""" if db_server == 'mysql': self.query['lockForInsert'] = """ @@ -4652,6 +4791,7 @@ class Sql: tourneyId, gametypeid, sessionId, + gameSessionId, startTime, importtime, seats, @@ -4682,7 +4822,7 @@ class Sql: values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, %s, %s)""" + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" self.query['store_hands_players'] = """insert into HandsPlayers ( @@ -4698,6 +4838,19 @@ class Sql: card5, card6, card7, + card8, + card9, + card10, + card11, + card12, + card13, + card14, + card15, + card16, + card17, + card18, + card19, + card20, winnings, rake, totalProfit, @@ -4812,7 +4965,10 @@ class Sql: %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s, %s )""" self.query['store_hands_actions'] = """insert into HandsActions ( diff --git a/pyfpdb/TourneySummary.py b/pyfpdb/TourneySummary.py index 94055b92..2e655f6b 100644 --- a/pyfpdb/TourneySummary.py +++ b/pyfpdb/TourneySummary.py @@ -287,6 +287,24 @@ winnings (int) the money the player ended the tourney with (can be 0, or -1 i if player not in [p[1] for p in self.players]: print "checkPlayerExists", player, "fail" raise FpdbParseError + + def updateSessionsCache(self, sc, gsc, tz, doinsert): + self.heros = self.db.getHeroIds(self.dbid_pids, self.siteName) + sc = self.db.prepSessionsCache(self.tourNo, self.dbid_pids, self.startTime, sc , self.heros, doinsert) + + gsc = self.db.storeSessionsCache(self.tourNo, self.dbid_pids, self.startTime, {'type': 'summary'} + ,None, self.assembleInfo(), sc, gsc, tz, self.heros, doinsert) + return sc, gsc + + def assembleInfo(self): + info = {} + info['tourneyTypeId'] = self.tourneyTypeId + info['winnings'] = self.winnings + info['winningsCurrency'] = self.winningsCurrency + info['buyinCurrency'] = self.buyinCurrency + info['buyin'] = self.buyin + info['fee'] = self.fee + return info def writeSummary(self, fh=sys.__stdout__): print >>fh, "Override me" diff --git a/pyfpdb/WinamaxSummary.py b/pyfpdb/WinamaxSummary.py index e1d5b0ce..54e81058 100644 --- a/pyfpdb/WinamaxSummary.py +++ b/pyfpdb/WinamaxSummary.py @@ -45,7 +45,7 @@ class WinamaxSummary(TourneySummary): substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols + 'LS' : u"\$|\u20AC|" # legal currency symbols } re_GameType = re.compile("""

((?PNo Limit|Pot Limit) (?PHold\'em))

""") diff --git a/pyfpdb/WinamaxToFpdb.py b/pyfpdb/WinamaxToFpdb.py index b01df4ec..be86ae64 100644 --- a/pyfpdb/WinamaxToFpdb.py +++ b/pyfpdb/WinamaxToFpdb.py @@ -52,10 +52,10 @@ class Winamax(HandHistoryConverter): siteId = 14 # Needs to match id entry in Sites database mixes = { } # Legal mixed games - sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE + sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes - 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) } limits = { 'no limit':'nl', 'pot limit' : 'pl','LIMIT':'fl'} diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index df45bc63..a104d5ce 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -242,7 +242,7 @@ class Importer: #print "dropInd =", self.settings['dropIndexes'], " dropHudCache =", self.settings['dropHudCache'] if self.settings['threads'] <= 0: - (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, None) + (totstored, totdups, totpartial, toterrors) = self.importFiles(None) else: # create queue (will probably change to deque at some point): self.writeq = Queue.Queue( self.settings['writeQSize'] ) @@ -254,7 +254,7 @@ class Importer: t.setDaemon(True) t.start() # read hands and write to q: - (totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, self.writeq) + (totstored, totdups, totpartial, toterrors) = self.importFiles(self.writeq) if self.writeq.empty(): print _("writers finished already") @@ -286,7 +286,7 @@ class Importer: return (totstored, totdups, totpartial, toterrors, endtime-starttime) # end def runImport - def importFiles(self, db, q): + def importFiles(self, q): """"Read filenames in self.filelist and pass to import_file_dict(). Uses a separate database connection if created as a thread (caller passes None or no param as db).""" @@ -304,7 +304,7 @@ class Importer: ProgressDialog.progress_update() - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(db, file + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file ,self.filelist[file][0], self.filelist[file][1], q) totstored += stored totdups += duplicates @@ -395,7 +395,7 @@ class Importer: self.caller.addText("\n"+os.path.basename(file)) except KeyError: # TODO: What error happens here? pass - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1], None) try: if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime)) @@ -426,68 +426,71 @@ class Importer: #rulog.close() # This is now an internal function that should not be called directly. - def import_file_dict(self, db, file, site, filter, q=None): - #print "import_file_dict" + def import_file_dict(self, file, site, filter, q=None): if os.path.isdir(file): self.addToDirList[file] = [site] + [filter] return (0,0,0,0,0) - conv = None (stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, time()) - # sc: is there any need to decode this? maybe easier to skip it than guess at the encoding? - #file = file.decode("utf-8") #(Configuration.LOCALE_ENCODING) - # Load filter, process file, pass returned filename to import_fpdb_file if self.settings['threads'] > 0 and self.writeq is not None: - log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")") - else: - log.info(_("Converting %s") % file) - + log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")") + else: log.info(_("Converting %s") % file) + filter_name = filter.replace("ToFpdb", "") - mod = __import__(filter) obj = getattr(mod, filter_name, None) if callable(obj): - idx = 0 - if file in self.pos_in_file: - idx = self.pos_in_file[file] - else: - self.pos_in_file[file] = 0 - hhc = obj( self.config, in_path = file, index = idx, starsArchive = self.settings['starsArchive'], ftpArchive = self.settings['ftpArchive'], sitename = site ) + + if file in self.pos_in_file: idx = self.pos_in_file[file] + else: self.pos_in_file[file], idx = 0, 0 + + hhc = obj( self.config, in_path = file, index = idx + ,starsArchive = self.settings['starsArchive'] + ,ftpArchive = self.settings['ftpArchive'] + ,sitename = site ) + if hhc.getStatus(): + if self.caller: hhc.progressNotify() handlist = hhc.getProcessedHands() self.pos_in_file[file] = hhc.getLastCharacterRead() - to_hud = [] - hp_bulk = [] - ha_bulk = [] - i = 0 - + (hbulk, hpbulk, habulk, hcbulk, phands, ihands) = ([], [], [], [], [], []) + sc, gsc = {'bk': []}, {'bk': []} + + ####Lock Placeholder#### for hand in handlist: - i += 1 - if hand is not None: - hand.prepInsert(self.database, printtest = self.settings['testData']) - try: - hp_inserts, ha_inserts = hand.insert(self.database, hp_data = hp_bulk, - ha_data = ha_bulk, insert_data = len(handlist)==i, - printtest = self.settings['testData']) - hp_bulk += hp_inserts - ha_bulk += ha_inserts - except Exceptions.FpdbHandDuplicate: - duplicates += 1 - else: - if self.callHud and hand.dbid_hands != 0: - to_hud.append(hand.dbid_hands) - else: # TODO: Treat empty as an error, or just ignore? - log.error(_("Hand processed but empty")) - - # Call hudcache update if not in bulk import mode - # FIXME: Need to test for bulk import that isn't rebuilding the cache - if self.callHud: - for hand in handlist: - if hand is not None and not hand.is_duplicate: - hand.updateHudCache(self.database) + hand.prepInsert(self.database) + self.database.commit() + phands.append(hand) + ####Lock Placeholder#### + + for hand in phands: + hand.assembleHand() + + ####Lock Placeholder#### + id = self.database.nextHandId() + for i in range(len(phands)): + doinsert = len(phands)==i+1 + hand = phands[i] + try: + id = hand.getHandId(self.database, id) + sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert) + hbulk = hand.insertHands(self.database, hbulk, doinsert, self.settings['testData']) + hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) + ihands.append(hand) + to_hud.append(id) + except Exceptions.FpdbHandDuplicate: + duplicates += 1 + self.database.commit() + ####Lock Placeholder#### + + for i in range(len(ihands)): + doinsert = len(ihands)==i+1 + hand = ihands[i] + hpbulk = hand.insertHandsPlayers(self.database, hpbulk, doinsert, self.settings['testData']) + habulk = hand.insertHandsActions(self.database, habulk, doinsert, self.settings['testData']) self.database.commit() #pipe the Hands.id out to the HUD