evilny0/fpdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'chazdazzle' of git://github.com/ChazDazzle/fpdb-chaz

Browse Source 
Conflicts:
	pyfpdb/FulltiltToFpdb.py
This commit is contained in:
Worros 2011-03-29 13:52:08 +08:00
commit 59b65ca146
19 changed files with 1022 additions and 501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pyfpdb/Configuration.py
View File

@ -123,7 +123,7 @@ def get_config(file_name, fallback = True):
# OK, fall back to the .example file, should be in the start dir # OK, fall back to the .example file, should be in the start dir
elif os.path.exists(file_name + ".example"): elif os.path.exists(file_name + ".example"):
try: try:
print "" #print ""
example_path = file_name + ".example" example_path = file_name + ".example"
check_dir(default_dir) check_dir(default_dir)
if not config_found and fallback: if not config_found and fallback:
@ -173,7 +173,7 @@ def get_logger(file_name, config = "config", fallback = False, log_dir=None, log
log = logging.getLogger() log = logging.getLogger()
# but it looks like default is no output :-( maybe because all the calls name a module? # but it looks like default is no output :-( maybe because all the calls name a module?
log.debug(_("Default logger initialised for %s") % file) log.debug(_("Default logger initialised for %s") % file)
print(_("Default logger initialised for %s") % file) #print(_("Default logger initialised for %s") % file)
return log return log
def check_dir(path, create = True): def check_dir(path, create = True):
@ -314,7 +314,7 @@ class Site:
self.layout = {} self.layout = {}
self.emails = {} self.emails = {}
print _("Loading site"), self.site_name #print _("Loading site"), self.site_name
for layout_node in node.getElementsByTagName('layout'): for layout_node in node.getElementsByTagName('layout'):
lo = Layout(layout_node) lo = Layout(layout_node)
@ -490,12 +490,12 @@ class Import:
def __init__(self, node): def __init__(self, node):
self.node = node self.node = node
self.interval = node.getAttribute("interval") self.interval = node.getAttribute("interval")
self.callFpdbHud = node.getAttribute("callFpdbHud") self.sessionTimeout = node.getAttribute("sessionTimeout")
self.ResultsDirectory = node.getAttribute("ResultsDirectory") self.ResultsDirectory = node.getAttribute("ResultsDirectory")
self.hhBulkPath = node.getAttribute("hhBulkPath") self.hhBulkPath = node.getAttribute("hhBulkPath")
self.saveActions = string_to_bool(node.getAttribute("saveActions") , default=False) self.saveActions = string_to_bool(node.getAttribute("saveActions") , default=False)
self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions") , default=False) self.cacheSessions = string_to_bool(node.getAttribute("cacheSessions") , default=False)
self.sessionTimeout = string_to_bool(node.getAttribute("sessionTimeout"), default=30) self.callFpdbHud = string_to_bool(node.getAttribute("callFpdbHud") , default=False)
self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False) self.fastStoreHudCache = string_to_bool(node.getAttribute("fastStoreHudCache"), default=False)
self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH") , default=False) self.saveStarsHH = string_to_bool(node.getAttribute("saveStarsHH") , default=False)
@ -631,7 +631,7 @@ class RawHands:
if node==None: if node==None:
self.save="error" self.save="error"
self.compression="none" self.compression="none"
print _("missing config section raw_hands") #print _("missing config section raw_hands")
else: else:
save=node.getAttribute("save") save=node.getAttribute("save")
if save in ("none", "error", "all"): if save in ("none", "error", "all"):
@ -657,7 +657,7 @@ class RawTourneys:
if node==None: if node==None:
self.save="error" self.save="error"
self.compression="none" self.compression="none"
print _("missing config section raw_tourneys") #print _("missing config section raw_tourneys")
else: else:
save=node.getAttribute("save") save=node.getAttribute("save")
if save in ("none", "error", "all"): if save in ("none", "error", "all"):
@ -720,7 +720,7 @@ class Config:
while added > 0 and n < 2: while added > 0 and n < 2:
n = n + 1 n = n + 1
log.info(_("Reading configuration file %s") % file) log.info(_("Reading configuration file %s") % file)
print (("\n"+_("Reading configuration file %s")+"\n") % file) #print (("\n"+_("Reading configuration file %s")+"\n") % file)
try: try:
doc = xml.dom.minidom.parse(file) doc = xml.dom.minidom.parse(file)
self.doc = doc self.doc = doc
@ -829,7 +829,7 @@ class Config:
for raw_tourneys_node in doc.getElementsByTagName('raw_tourneys'): for raw_tourneys_node in doc.getElementsByTagName('raw_tourneys'):
self.raw_tourneys = RawTourneys(raw_tourneys_node) self.raw_tourneys = RawTourneys(raw_tourneys_node)
print "" #print ""
#end def __init__ #end def __init__
def add_missing_elements(self, doc, example_file): def add_missing_elements(self, doc, example_file):

734
pyfpdb/Database.py
View File

@ -73,7 +73,7 @@ except ImportError:
use_numpy = False use_numpy = False
DB_VERSION = 150 DB_VERSION = 152
# Variance created as sqlite has a bunch of undefined aggregate functions. # Variance created as sqlite has a bunch of undefined aggregate functions.
@ -134,6 +134,9 @@ class Database:
, {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'gametypeId', 'drop':1}
, {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'playerId', 'drop':0}
, {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0}
, {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1}
, {'tab':'SessionsCache', 'col':'playerId', 'drop':0}
, {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0}
, {'tab':'Players', 'col':'siteId', 'drop':1} , {'tab':'Players', 'col':'siteId', 'drop':1}
#, {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped #, {'tab':'Players', 'col':'name', 'drop':0} unique indexes not dropped
, {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1}
@ -157,6 +160,9 @@ class Database:
, {'tab':'HudCache', 'col':'gametypeId', 'drop':1} , {'tab':'HudCache', 'col':'gametypeId', 'drop':1}
, {'tab':'HudCache', 'col':'playerId', 'drop':0} , {'tab':'HudCache', 'col':'playerId', 'drop':0}
, {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0} , {'tab':'HudCache', 'col':'tourneyTypeId', 'drop':0}
, {'tab':'SessionsCache', 'col':'gametypeId', 'drop':1}
, {'tab':'SessionsCache', 'col':'playerId', 'drop':0}
, {'tab':'SessionsCache', 'col':'tourneyTypeId', 'drop':0}
, {'tab':'Players', 'col':'siteId', 'drop':1} , {'tab':'Players', 'col':'siteId', 'drop':1}
, {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1} , {'tab':'Tourneys', 'col':'tourneyTypeId', 'drop':1}
, {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0} , {'tab':'TourneysPlayers', 'col':'playerId', 'drop':0}
@ -182,6 +188,9 @@ class Database:
, {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
, {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
, {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
, {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
] ]
, [ # foreign keys for postgres (index 3) , [ # foreign keys for postgres (index 3)
{'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
@ -193,6 +202,9 @@ class Database:
, {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0} , {'fktab':'HudCache', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
, {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1} , {'fktab':'HudCache', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
, {'fktab':'SessionsCache','fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'SessionsCache','fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':0}
, {'fktab':'SessionsCache','fkcol':'tourneyTypeId', 'rtab':'TourneyTypes', 'rcol':'id', 'drop':1}
] ]
, [ # no foreign keys in sqlite (index 4) , [ # no foreign keys in sqlite (index 4)
] ]
@ -261,6 +273,7 @@ class Database:
self.db_path = '' self.db_path = ''
gen = c.get_general_params() gen = c.get_general_params()
self.day_start = 0 self.day_start = 0
self._has_lock = False
if 'day_start' in gen: if 'day_start' in gen:
self.day_start = float(gen['day_start']) self.day_start = float(gen['day_start'])
@ -1253,6 +1266,12 @@ class Database:
c.execute(self.sql.query['createRawHands']) c.execute(self.sql.query['createRawHands'])
c.execute(self.sql.query['createRawTourneys']) c.execute(self.sql.query['createRawTourneys'])
# Create sessionscache indexes
log.debug("Creating SessionsCache indexes")
c.execute(self.sql.query['addSessionIdIndex'])
c.execute(self.sql.query['addHandsSessionIdIndex'])
c.execute(self.sql.query['addHandsGameSessionIdIndex'])
# Create unique indexes: # Create unique indexes:
log.debug("Creating unique indexes") log.debug("Creating unique indexes")
c.execute(self.sql.query['addTourneyIndex']) c.execute(self.sql.query['addTourneyIndex'])
@ -1618,10 +1637,86 @@ class Database:
print err print err
#end def rebuild_hudcache #end def rebuild_hudcache
def rebuild_sessionscache(self, h_start=None, v_start=None): def rebuild_sessionscache(self):
"""clears sessionscache and rebuilds from the individual handsplayers records""" """clears sessionscache and rebuilds from the individual records"""
#Will get to this soon heros = []
pass for site in self.config.get_supported_sites():
result = self.get_site_id(site)
if result:
site_id = result[0][0]
hero = self.config.supported_sites[site].screen_name
p_id = self.get_player_id(self.config, site, hero)
if p_id:
heros.append(int(p_id))
rebuildSessionsCache = self.sql.query['rebuildSessionsCache']
rebuildSessionsCacheSum = self.sql.query['rebuildSessionsCacheSum']
if len(heros) == 0:
where = '0'
where_summary = '0'
elif len(heros) > 0:
where = str(heros[0])
where_summary = str(heros[0])
if len(heros) > 1:
for i in heros:
if i != heros[0]:
where = where + ' OR HandsPlayers.playerId = %s' % str(i)
where_summary = where_summary + ' OR TourneysPlayers.playerId = %s' % str(i)
rebuildSessionsCache = rebuildSessionsCache.replace('<where_clause>', where)
rebuildSessionsCacheSum = rebuildSessionsCacheSum.replace('<where_clause>', where_summary)
c = self.get_cursor()
c.execute(self.sql.query['clearSessionsCache'])
self.commit()
sc, gsc = {'bk': []}, {'bk': []}
c.execute(rebuildSessionsCache)
tmp = c.fetchone()
while True:
pids, game, pdata = {}, {}, {}
pdata['pname'] = {}
id = tmp[0]
startTime = tmp[1]
pids['pname'] = tmp[2]
gid = tmp[3]
game['type'] = tmp[4]
pdata['pname']['totalProfit'] = tmp[5]
pdata['pname']['tourneyTypeId'] = tmp[6]
tmp = c.fetchone()
sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None)
gsc = self.storeSessionsCache(id, pids, startTime, game, gid, pdata, sc, gsc, None, heros, tmp == None)
if tmp == None:
for i, id in sc.iteritems():
if i!='bk':
sid = id['id']
gid = gsc[i]['id']
c.execute("UPDATE Hands SET sessionId = %s, gameSessionId = %s WHERE id = %s", (sid, gid, i))
break
self.commit()
sc, gsc = {'bk': []}, {'bk': []}
c.execute(rebuildSessionsCacheSum)
tmp = c.fetchone()
while True:
pids, game, info = {}, {}, {}
id = tmp[0]
startTime = tmp[1]
pids['pname'] = tmp[2]
game['type'] = 'summary'
info['tourneyTypeId'] = tmp[3]
info['winnings'] = {}
info['winnings']['pname'] = tmp[4]
info['winningsCurrency'] = {}
info['winningsCurrency']['pname'] = tmp[5]
info['buyinCurrency'] = tmp[6]
info['buyin'] = tmp[7]
info['fee'] = tmp[8]
tmp = c.fetchone()
sc = self.prepSessionsCache (id, pids, startTime, sc , heros, tmp == None)
gsc = self.storeSessionsCache(id, pids, startTime, game, None, info, sc, gsc, None, heros, tmp == None)
if tmp == None:
break
def get_hero_hudcache_start(self): def get_hero_hudcache_start(self):
"""fetches earliest stylekey from hudcache for one of hero's player ids""" """fetches earliest stylekey from hudcache for one of hero's player ids"""
@ -1701,6 +1796,34 @@ class Database:
# however the calling prog requires. Main aims: # however the calling prog requires. Main aims:
# - existing static routines from fpdb_simple just modified # - existing static routines from fpdb_simple just modified
def setThreadId(self, threadid):
self.threadId = threadid
def acquireLock(self, wait=True, retry_time=.01):
while not self._has_lock:
cursor = self.get_cursor()
cursor.execute(self.sql.query['selectLock'])
record = cursor.fetchall()
self.commit()
if not len(record):
cursor.execute(self.sql.query['switchLock'], (True, self.threadId))
self.commit()
self._has_lock = True
return True
else:
cursor.execute(self.sql.query['missedLock'], (1, self.threadId))
self.commit()
if not wait:
return False
sleep(retry_time)
def releaseLock(self):
if self._has_lock:
cursor = self.get_cursor()
num = cursor.execute(self.sql.query['switchLock'], (False, self.threadId))
self.commit()
self._has_lock = False
def lock_for_insert(self): def lock_for_insert(self):
"""Lock tables in MySQL to try to speed inserts up""" """Lock tables in MySQL to try to speed inserts up"""
try: try:
@ -1713,70 +1836,74 @@ class Database:
# NEWIMPORT CODE # NEWIMPORT CODE
########################### ###########################
def storeHand(self, p, printdata = False): def storeHand(self, hdata, hbulk, doinsert = False, printdata = False):
if printdata: if printdata:
print _("######## Hands ##########") print _("######## Hands ##########")
import pprint import pprint
pp = pprint.PrettyPrinter(indent=4) pp = pprint.PrettyPrinter(indent=4)
pp.pprint(p) pp.pprint(hdata)
print _("###### End Hands ########") print _("###### End Hands ########")
# Tablename can have odd charachers # Tablename can have odd charachers
p['tableName'] = Charset.to_db_utf8(p['tableName']) hdata['tableName'] = Charset.to_db_utf8(hdata['tableName'])
#stores into table hands: hbulk.append( [ hdata['tableName'],
q = self.sql.query['store_hand'] hdata['siteHandNo'],
hdata['tourneyId'],
q = q.replace('%s', self.sql.query['placeholder']) hdata['gameTypeId'],
hdata['sessionId'],
c = self.get_cursor() hdata['gameSessionId'],
hdata['startTime'],
c.execute(q, (
p['tableName'],
p['siteHandNo'],
p['tourneyId'],
p['gametypeId'],
p['sessionId'],
p['startTime'],
datetime.utcnow(), #importtime datetime.utcnow(), #importtime
p['seats'], hdata['seats'],
p['maxSeats'], hdata['maxSeats'],
p['texture'], hdata['texture'],
p['playersVpi'], hdata['playersVpi'],
p['boardcard1'], hdata['boardcard1'],
p['boardcard2'], hdata['boardcard2'],
p['boardcard3'], hdata['boardcard3'],
p['boardcard4'], hdata['boardcard4'],
p['boardcard5'], hdata['boardcard5'],
p['playersAtStreet1'], hdata['playersAtStreet1'],
p['playersAtStreet2'], hdata['playersAtStreet2'],
p['playersAtStreet3'], hdata['playersAtStreet3'],
p['playersAtStreet4'], hdata['playersAtStreet4'],
p['playersAtShowdown'], hdata['playersAtShowdown'],
p['street0Raises'], hdata['street0Raises'],
p['street1Raises'], hdata['street1Raises'],
p['street2Raises'], hdata['street2Raises'],
p['street3Raises'], hdata['street3Raises'],
p['street4Raises'], hdata['street4Raises'],
p['street1Pot'], hdata['street1Pot'],
p['street2Pot'], hdata['street2Pot'],
p['street3Pot'], hdata['street3Pot'],
p['street4Pot'], hdata['street4Pot'],
p['showdownPot'] hdata['showdownPot'],
)) hdata['id']
return self.get_last_insert_id(c) ])
# def storeHand
def storeHandsPlayers(self, hid, pids, pdata, hp_bulk = None, insert = False, printdata = False): if doinsert:
for h in hbulk:
id = h.pop()
if hdata['sc'] and hdata['gsc']:
h[4] = hdata['sc'][id]['id']
h[5] = hdata['gsc'][id]['id']
q = self.sql.query['store_hand']
q = q.replace('%s', self.sql.query['placeholder'])
c = self.get_cursor()
c.executemany(q, hbulk)
self.commit()
return hbulk
def storeHandsPlayers(self, hid, pids, pdata, hpbulk, doinsert = False, printdata = False):
#print "DEBUG: %s %s %s" %(hid, pids, pdata) #print "DEBUG: %s %s %s" %(hid, pids, pdata)
if printdata: if printdata:
import pprint import pprint
pp = pprint.PrettyPrinter(indent=4) pp = pprint.PrettyPrinter(indent=4)
pp.pprint(pdata) pp.pprint(pdata)
inserts = []
for p in pdata: for p in pdata:
inserts.append( (hid, hpbulk.append( ( hid,
pids[p], pids[p],
pdata[p]['startCash'], pdata[p]['startCash'],
pdata[p]['seatNo'], pdata[p]['seatNo'],
@ -1788,6 +1915,19 @@ class Database:
pdata[p]['card5'], pdata[p]['card5'],
pdata[p]['card6'], pdata[p]['card6'],
pdata[p]['card7'], pdata[p]['card7'],
pdata[p]['card8'],
pdata[p]['card9'],
pdata[p]['card10'],
pdata[p]['card11'],
pdata[p]['card12'],
pdata[p]['card13'],
pdata[p]['card14'],
pdata[p]['card15'],
pdata[p]['card16'],
pdata[p]['card17'],
pdata[p]['card18'],
pdata[p]['card19'],
pdata[p]['card20'],
pdata[p]['winnings'], pdata[p]['winnings'],
pdata[p]['rake'], pdata[p]['rake'],
pdata[p]['totalProfit'], pdata[p]['totalProfit'],
@ -1882,16 +2022,14 @@ class Database:
pdata[p]['street4Raises'] pdata[p]['street4Raises']
) ) ) )
if insert: if doinsert:
hp_bulk += inserts
q = self.sql.query['store_hands_players'] q = self.sql.query['store_hands_players']
q = q.replace('%s', self.sql.query['placeholder']) q = q.replace('%s', self.sql.query['placeholder'])
c = self.get_cursor() c = self.get_cursor()
c.executemany(q, hp_bulk) c.executemany(q, hpbulk)
return hpbulk
return inserts def storeHandsActions(self, hid, pids, adata, habulk, doinsert = False, printdata = False):
def storeHandsActions(self, hid, pids, adata, ha_bulk = None, insert = False, printdata = False):
#print "DEBUG: %s %s %s" %(hid, pids, adata) #print "DEBUG: %s %s %s" %(hid, pids, adata)
# This can be used to generate test data. Currently unused # This can be used to generate test data. Currently unused
@ -1900,9 +2038,8 @@ class Database:
# pp = pprint.PrettyPrinter(indent=4) # pp = pprint.PrettyPrinter(indent=4)
# pp.pprint(adata) # pp.pprint(adata)
inserts = []
for a in adata: for a in adata:
inserts.append( (hid, habulk.append( (hid,
pids[adata[a]['player']], pids[adata[a]['player']],
adata[a]['street'], adata[a]['street'],
adata[a]['actionNo'], adata[a]['actionNo'],
@ -1916,16 +2053,14 @@ class Database:
adata[a]['allIn'] adata[a]['allIn']
) ) ) )
if insert: if doinsert:
ha_bulk += inserts
q = self.sql.query['store_hands_actions'] q = self.sql.query['store_hands_actions']
q = q.replace('%s', self.sql.query['placeholder']) q = q.replace('%s', self.sql.query['placeholder'])
c = self.get_cursor() c = self.get_cursor()
c.executemany(q, ha_bulk) c.executemany(q, habulk)
return habulk
return inserts def storeHudCache(self, gid, pids, starttime, pdata, hcbulk, doinsert = False):
def storeHudCache(self, gid, pids, starttime, pdata):
"""Update cached statistics. If update fails because no record exists, do an insert.""" """Update cached statistics. If update fails because no record exists, do an insert."""
tz = datetime.utcnow() - datetime.today() tz = datetime.utcnow() - datetime.today()
@ -1948,11 +2083,9 @@ class Database:
insert_hudcache = insert_hudcache.replace('%s', self.sql.query['placeholder']) insert_hudcache = insert_hudcache.replace('%s', self.sql.query['placeholder'])
#print "DEBUG: %s %s %s" %(hid, pids, pdata) #print "DEBUG: %s %s %s" %(hid, pids, pdata)
inserts = []
for p in pdata: for p in pdata:
#NOTE: Insert new stats at right place because SQL needs strict order #NOTE: Insert new stats at right place because SQL needs strict order
line = [] line = []
line.append(1) # HDs line.append(1) # HDs
line.append(pdata[p]['street0VPI']) line.append(pdata[p]['street0VPI'])
line.append(pdata[p]['street0Aggr']) line.append(pdata[p]['street0Aggr'])
@ -2042,168 +2175,365 @@ class Database:
line.append(pdata[p]['street3Raises']) line.append(pdata[p]['street3Raises'])
line.append(pdata[p]['street4Raises']) line.append(pdata[p]['street4Raises'])
line.append(gid) # gametypeId hc, hcs = {}, []
line.append(pids[p]) # playerId hc['gametypeId'] = gid
line.append(len(pids)) # activeSeats hc['playerId'] = pids[p]
hc['activeSeats'] = len(pids)
pos = {'B':'B', 'S':'S', 0:'D', 1:'C', 2:'M', 3:'M', 4:'M', 5:'E', 6:'E', 7:'E', 8:'E', 9:'E' } pos = {'B':'B', 'S':'S', 0:'D', 1:'C', 2:'M', 3:'M', 4:'M', 5:'E', 6:'E', 7:'E', 8:'E', 9:'E' }
line.append(pos[pdata[p]['position']]) hc['position'] = pos[pdata[p]['position']]
line.append(pdata[p]['tourneyTypeId']) hc['tourneyTypeId'] = pdata[p]['tourneyTypeId']
line.append(styleKey) # styleKey hc['styleKey'] = styleKey
inserts.append(line) hc['line'] = line
hc['game'] = [hc['gametypeId']
,hc['playerId']
,hc['activeSeats']
,hc['position']
,hc['tourneyTypeId']
,hc['styleKey']]
hcs.append(hc)
for h in hcs:
match = False
for b in hcbulk:
if h['game']==b['game']:
b['line'] = [sum(l) for l in zip(b['line'], h['line'])]
match = True
if not match: hcbulk.append(h)
cursor = self.get_cursor() if doinsert:
inserts = []
for row in inserts: exists = []
#convert all True/False values to numeric 0/1 updates = []
# needed because columns in hudcache are not BOOL they are INT for hc in hcbulk:
# and are being summed if an existing hudcache entry exists row = hc['line'] + hc['game']
# psycopg2 module does not automatically convert these to numeric. if hc['game'] in exists:
# mantis bug #93 updates.append(row)
for ind in range(len(row)): continue
if row[ind] == True: row[ind] = 1 c = self.get_cursor()
if row[ind] == False: row[ind] = 0 num = c.execute(update_hudcache, row)
# Try to do the update first: # Try to do the update first. Do insert it did not work
num = cursor.execute(update_hudcache, row) if ((self.backend == self.PGSQL and c.statusmessage != "UPDATE 1")
#print "DEBUG: values: %s" % row[-6:]
# Test statusmessage to see if update worked, do insert if not
# num is a cursor in sqlite
if ((self.backend == self.PGSQL and cursor.statusmessage != "UPDATE 1")
or (self.backend == self.MYSQL_INNODB and num == 0) or (self.backend == self.MYSQL_INNODB and num == 0)
or (self.backend == self.SQLITE and num.rowcount == 0)): or (self.backend == self.SQLITE and num.rowcount == 0)):
#move the last 6 items in WHERE clause of row from the end of the array inserts.append(hc['game'] + hc['line'])
# to the beginning for the INSERT statement #row = hc['game'] + hc['line']
#print "DEBUG: using INSERT: %s" % num #num = c.execute(insert_hudcache, row)
row = row[-6:] + row[:-6]
num = cursor.execute(insert_hudcache, row)
#print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num #print "DEBUG: Successfully(?: %s) updated HudCacho using INSERT" % num
else: else:
exists.append(hc['game'])
#print "DEBUG: Successfully updated HudCacho using UPDATE" #print "DEBUG: Successfully updated HudCacho using UPDATE"
pass if inserts: c.executemany(insert_hudcache, inserts)
if updates: c.executemany(update_hudcache, updates)
def storeSessionsCache(self, pids, startTime, game, pdata): return hcbulk
def prepSessionsCache(self, hid, pids, startTime, sc, heros, doinsert = False):
"""Update cached sessions. If no record exists, do an insert""" """Update cached sessions. If no record exists, do an insert"""
THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60)) THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60))
select_sessionscache = self.sql.query['select_sessionscache'] select_prepSC = self.sql.query['select_prepSC'].replace('%s', self.sql.query['placeholder'])
select_sessionscache = select_sessionscache.replace('%s', self.sql.query['placeholder']) update_Hands_sid = self.sql.query['update_Hands_sid'].replace('%s', self.sql.query['placeholder'])
select_sessionscache_mid = self.sql.query['select_sessionscache_mid'] update_SC_sid = self.sql.query['update_SC_sid'].replace('%s', self.sql.query['placeholder'])
select_sessionscache_mid = select_sessionscache_mid.replace('%s', self.sql.query['placeholder']) update_prepSC = self.sql.query['update_prepSC'].replace('%s', self.sql.query['placeholder'])
select_sessionscache_start = self.sql.query['select_sessionscache_start']
select_sessionscache_start = select_sessionscache_start.replace('%s', self.sql.query['placeholder'])
update_sessionscache_mid = self.sql.query['update_sessionscache_mid'] #print "DEBUG: %s %s %s" %(hid, pids, pdata)
update_sessionscache_mid = update_sessionscache_mid.replace('%s', self.sql.query['placeholder']) hand = {}
update_sessionscache_start = self.sql.query['update_sessionscache_start'] for p, id in pids.iteritems():
update_sessionscache_start = update_sessionscache_start.replace('%s', self.sql.query['placeholder']) if id in heros:
update_sessionscache_end = self.sql.query['update_sessionscache_end'] hand['startTime'] = startTime.replace(tzinfo=None)
update_sessionscache_end = update_sessionscache_end.replace('%s', self.sql.query['placeholder']) hand['ids'] = []
insert_sessionscache = self.sql.query['insert_sessionscache'] if hand:
insert_sessionscache = insert_sessionscache.replace('%s', self.sql.query['placeholder']) id = []
merge_sessionscache = self.sql.query['merge_sessionscache'] lower = hand['startTime']-THRESHOLD
merge_sessionscache = merge_sessionscache.replace('%s', self.sql.query['placeholder']) upper = hand['startTime']+THRESHOLD
delete_sessions = self.sql.query['delete_sessions'] for i in range(len(sc['bk'])):
delete_sessions = delete_sessions.replace('%s', self.sql.query['placeholder']) if ((lower <= sc['bk'][i]['sessionEnd'])
and (upper >= sc['bk'][i]['sessionStart'])):
if ((hand['startTime'] <= sc['bk'][i]['sessionEnd'])
and (hand['startTime'] >= sc['bk'][i]['sessionStart'])):
id.append(i)
elif hand['startTime'] < sc['bk'][i]['sessionStart']:
sc['bk'][i]['sessionStart'] = hand['startTime']
id.append(i)
elif hand['startTime'] > sc['bk'][i]['sessionEnd']:
sc['bk'][i]['sessionEnd'] = hand['startTime']
id.append(i)
if len(id) == 1:
id = id[0]
sc['bk'][id]['ids'].append(hid)
elif len(id) == 2:
if sc['bk'][id[0]]['startTime'] < sc['bk'][id[1]]['startTime']:
sc['bk'][id[0]]['endTime'] = sc['bk'][id[1]]['endTime']
else:
sc['bk'][id[0]]['startTime'] = sc['bk'][id[1]]['startTime']
sc['bk'].pop[id[1]]
id = id[0]
sc['bk'][id]['ids'].append(hid)
elif len(id) == 0:
hand['id'] = None
hand['sessionStart'] = hand['startTime']
hand['sessionEnd'] = hand['startTime']
id = len(sc['bk'])
hand['ids'].append(hid)
sc['bk'].append(hand)
update_hands_sessionid = self.sql.query['update_hands_sessionid'] if doinsert:
update_hands_sessionid = update_hands_sessionid.replace('%s', self.sql.query['placeholder']) c = self.get_cursor()
c.execute("SELECT max(sessionId) FROM SessionsCache")
id = c.fetchone()[0]
if id: sid = id
else: sid = 0
for i in range(len(sc['bk'])):
lower = sc['bk'][i]['sessionStart'] - THRESHOLD
upper = sc['bk'][i]['sessionEnd'] + THRESHOLD
c.execute(select_prepSC, (lower, upper))
r = self.fetchallDict(c)
num = len(r)
if (num == 1):
start, end, update = r[0]['sessionStart'], r[0]['sessionEnd'], False
if sc['bk'][i]['sessionStart'] < start:
start, update = sc['bk'][i]['sessionStart'], True
if sc['bk'][i]['sessionEnd'] > end:
end, update = sc['bk'][i]['sessionEnd'], True
if update:
c.execute(update_prepSC, [start, end, r[0]['id']])
for h in sc['bk'][i]['ids']:
sc[h] = {'id': r[0]['id'], 'data': [start, end]}
elif (num > 1):
start, end, merge, merge_h, merge_sc = None, None, [], [], []
sid += 1
r.append(sc['bk'][i])
for n in r:
if start:
if start > n['sessionStart']:
start = n['sessionStart']
else: start = n['sessionStart']
if end:
if end < n['sessionEnd']:
end = n['sessionEnd']
else: end = n['sessionEnd']
for n in r:
if n['id']:
if n['id'] in merge: continue
merge.append(n['id'])
merge_h.append([sid, n['id']])
merge_sc.append([start, end, sid, n['id']])
c.executemany(update_Hands_sid, merge_h)
c.executemany(update_SC_sid, merge_sc)
for k, v in sc.iteritems():
if k!='bk' and v['id'] in merge:
sc[k]['id'] = sid
for h in sc['bk'][i]['ids']:
sc[h] = {'id': sid, 'data': [start, end]}
elif (num == 0):
sid += 1
start = sc['bk'][i]['sessionStart']
end = sc['bk'][i]['sessionEnd']
for h in sc['bk'][i]['ids']:
sc[h] = {'id': sid, 'data': [start, end]}
return sc
def storeSessionsCache(self, hid, pids, startTime, game, gid, pdata, sc, gsc, tz, heros, doinsert = False):
"""Update cached sessions. If no record exists, do an insert"""
if not tz:
tz_dt = datetime.utcnow() - datetime.today()
tz = tz_dt.seconds/3600
THRESHOLD = timedelta(seconds=int(self.sessionTimeout * 60))
local = startTime + timedelta(hours=int(tz))
date = "d%02d%02d%02d" % (local.year - 2000, local.month, local.day)
select_SC = self.sql.query['select_SC'].replace('%s', self.sql.query['placeholder'])
update_SC = self.sql.query['update_SC'].replace('%s', self.sql.query['placeholder'])
insert_SC = self.sql.query['insert_SC'].replace('%s', self.sql.query['placeholder'])
delete_SC = self.sql.query['delete_SC'].replace('%s', self.sql.query['placeholder'])
update_Hands_gsid = self.sql.query['update_Hands_gsid'].replace('%s', self.sql.query['placeholder'])
#print "DEBUG: %s %s %s" %(hid, pids, pdata)
hand = {}
for p, id in pids.iteritems():
if id in heros:
hand['hands'] = 0
hand['totalProfit'] = 0
hand['playerId'] = id
hand['gametypeId'] = None
hand['date'] = date
hand['startTime'] = startTime.replace(tzinfo=None)
hand['hid'] = hid
hand['tourneys'] = 0
hand['tourneyTypeId'] = None
hand['ids'] = []
if (game['type']=='summary'):
hand['type'] = 'tour'
hand['tourneys'] = 1
hand['tourneyTypeId'] = pdata['tourneyTypeId']
if pdata['buyinCurrency'] == pdata['winningsCurrency'][p]:
hand['totalProfit'] = pdata['winnings'][p] - (pdata['buyin'] + pdata['fee'])
else: hand['totalProfit'] = pdata['winnings'][p]
elif (game['type']=='ring'):
hand['type'] = game['type']
hand['hands'] = 1
hand['gametypeId'] = gid
hand['totalProfit'] = pdata[p]['totalProfit']
elif (game['type']=='tour'):
hand['type'] = game['type']
hand['hands'] = 1
hand['tourneyTypeId'] = pdata[p]['tourneyTypeId']
if hand:
id = []
lower = hand['startTime']-THRESHOLD
upper = hand['startTime']+THRESHOLD
for i in range(len(gsc['bk'])):
if ((hand['date'] == gsc['bk'][i]['date'])
and (hand['gametypeId'] == gsc['bk'][i]['gametypeId'])
and (hand['playerId'] == gsc['bk'][i]['playerId'])
and (hand['tourneyTypeId'] == gsc['bk'][i]['tourneyTypeId'])):
if ((lower <= gsc['bk'][i]['gameEnd'])
and (upper >= gsc['bk'][i]['gameStart'])):
if ((hand['startTime'] <= gsc['bk'][i]['gameEnd'])
and (hand['startTime'] >= gsc['bk'][i]['gameStart'])):
gsc['bk'][i]['hands'] += hand['hands']
gsc['bk'][i]['tourneys'] += hand['tourneys']
gsc['bk'][i]['totalProfit'] += hand['totalProfit']
elif hand['startTime'] < gsc['bk'][i]['gameStart']:
gsc['bk'][i]['hands'] += hand['hands']
gsc['bk'][i]['tourneys'] += hand['tourneys']
gsc['bk'][i]['totalProfit'] += hand['totalProfit']
gsc['bk'][i]['gameStart'] = hand['startTime']
elif hand['startTime'] > gsc['bk'][i]['gameEnd']:
gsc['bk'][i]['hands'] += hand['hands']
gsc['bk'][i]['tourneys'] += hand['tourneys']
gsc['bk'][i]['totalProfit'] += hand['totalProfit']
gsc['bk'][i]['gameEnd'] = hand['startTime']
id.append(i)
if len(id) == 1:
gsc['bk'][id[0]]['ids'].append(hid)
elif len(id) == 2:
if gsc['bk'][id[0]]['gameStart'] < gsc['bk'][id[1]]['gameStart']:
gsc['bk'][id[0]]['gameEnd'] = gsc['bk'][id[1]]['gameEnd']
else: gsc['bk'][id[0]]['gameStart'] = gsc['bk'][id[1]]['gameStart']
gsc['bk'][id[0]]['hands'] += hand['hands']
gsc['bk'][id[0]]['tourneys'] += hand['tourneys']
gsc['bk'][id[0]]['totalProfit'] += hand['totalProfit']
gsc['bk'].pop[id[1]]
gsc['bk'][id[0]]['ids'].append(hid)
elif len(id) == 0:
hand['gameStart'] = hand['startTime']
hand['gameEnd'] = hand['startTime']
id = len(gsc['bk'])
hand['ids'].append(hid)
gsc['bk'].append(hand)
if doinsert:
c = self.get_cursor()
for i in range(len(gsc['bk'])):
hid = gsc['bk'][i]['hid']
sid, start, end = sc[hid]['id'], sc[hid]['data'][0], sc[hid]['data'][1]
lower = gsc['bk'][i]['gameStart'] - THRESHOLD
upper = gsc['bk'][i]['gameEnd'] + THRESHOLD
game = [gsc['bk'][i]['date']
,gsc['bk'][i]['type']
,gsc['bk'][i]['gametypeId']
,gsc['bk'][i]['tourneyTypeId']
,gsc['bk'][i]['playerId']]
row = [lower, upper] + game
c.execute(select_SC, row)
r = self.fetchallDict(c)
num = len(r)
if (num == 1):
gstart, gend = r[0]['gameStart'], r[0]['gameEnd']
if gsc['bk'][i]['gameStart'] < gstart:
gstart = gsc['bk'][i]['gameStart']
if gsc['bk'][i]['gameEnd'] > gend:
gend = gsc['bk'][i]['gameEnd']
row = [start, end, gstart, gend
,gsc['bk'][i]['hands']
,gsc['bk'][i]['tourneys']
,gsc['bk'][i]['totalProfit']
,r[0]['id']]
c.execute(update_SC, row)
for h in gsc['bk'][i]['ids']: gsc[h] = {'id': r[0]['id']}
elif (num > 1):
gstart, gend, hands, tourneys, totalProfit, delete, merge = None, None, 0, 0, 0, [], []
for n in r: delete.append(n['id'])
delete.sort()
for d in delete: c.execute(delete_SC, d)
r.append(gsc['bk'][i])
for n in r:
if gstart:
if gstart > n['gameStart']:
gstart = n['gameStart']
else: gstart = n['gameStart']
if gend:
if gend < n['gameEnd']:
gend = n['gameEnd']
else: gend = n['gameEnd']
hands += n['hands']
tourneys += n['tourneys']
totalProfit += n['totalProfit']
row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit]
c.execute(insert_SC, row)
gsid = self.get_last_insert_id(c)
for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid}
for m in delete: merge.append([gsid, m])
c.executemany(update_Hands_gsid, merge)
elif (num == 0):
gstart = gsc['bk'][i]['gameStart']
gend = gsc['bk'][i]['gameEnd']
hands = gsc['bk'][i]['hands']
tourneys = gsc['bk'][i]['tourneys']
totalProfit = gsc['bk'][i]['totalProfit']
row = [start, end, gstart, gend, sid] + game + [hands, tourneys, totalProfit]
c.execute(insert_SC, row)
gsid = self.get_last_insert_id(c)
for h in gsc['bk'][i]['ids']: gsc[h] = {'id': gsid}
else:
# Something bad happened
pass
self.commit()
return gsc
def getHeroIds(self, pids, sitename):
#Grab playerIds using hero names in HUD_Config.xml #Grab playerIds using hero names in HUD_Config.xml
try: try:
# derive list of program owner's player ids # derive list of program owner's player ids
self.hero = {} # name of program owner indexed by site id hero = {} # name of program owner indexed by site id
self.hero_ids = [] hero_ids = []
# make sure at least two values in list # make sure at least two values in list
# so that tuple generation creates doesn't use # so that tuple generation creates doesn't use
# () or (1,) style # () or (1,) style
for site in self.config.get_supported_sites(): for site in self.config.get_supported_sites():
result = self.get_site_id(site) hero = self.config.supported_sites[site].screen_name
if result: for n, v in pids.iteritems():
site_id = result[0][0] if n == hero and sitename == site:
self.hero[site_id] = self.config.supported_sites[site].screen_name hero_ids.append(v)
p_id = self.get_player_id(self.config, site, self.hero[site_id])
if p_id:
self.hero_ids.append(int(p_id))
except: except:
err = traceback.extract_tb(sys.exc_info()[2])[-1] err = traceback.extract_tb(sys.exc_info()[2])[-1]
print _("Error aquiring hero ids:"), str(sys.exc_value) #print _("Error aquiring hero ids:"), str(sys.exc_value)
print err return hero_ids
inserts = [] def fetchallDict(self, cursor):
for p in pdata: data = cursor.fetchall()
if pids[p] in self.hero_ids: if not data: return []
line = [0]*5 desc = cursor.description
results = [0]*len(data)
if (game['type']=='ring'): line[0] = 1 # count ring hands for i in range(len(data)):
if (game['type']=='tour'): line[1] = 1 # count tour hands results[i] = {}
if (game['type']=='ring' and game['currency']=='USD'): line[2] = pdata[p]['totalProfit'] #sum of ring profit in USD for n in range(len(desc)):
if (game['type']=='ring' and game['currency']=='EUR'): line[3] = pdata[p]['totalProfit'] #sum of ring profit in EUR name = desc[n][0]
line[4] = startTime results[i][name] = data[i][n]
inserts.append(line) return results
cursor = self.get_cursor()
id = None
for row in inserts:
threshold = []
threshold.append(row[-1]-THRESHOLD)
threshold.append(row[-1]+THRESHOLD)
cursor.execute(select_sessionscache, threshold)
session_records = cursor.fetchall()
num = len(session_records)
if (num == 1):
id = session_records[0][0] #grab the sessionId
# Try to do the update first:
#print "DEBUG: found 1 record to update"
update_mid = row + row[-1:]
cursor.execute(select_sessionscache_mid, update_mid[-2:])
mid = len(cursor.fetchall())
if (mid == 0):
update_startend = row[-1:] + row + threshold
cursor.execute(select_sessionscache_start, update_startend[-3:])
start = len(cursor.fetchall())
if (start == 0):
#print "DEBUG:", start, " start record found. Update stats and start time"
cursor.execute(update_sessionscache_end, update_startend)
else:
#print "DEBUG: 1 end record found. Update stats and end time time"
cursor.execute(update_sessionscache_start, update_startend)
else:
#print "DEBUG: update stats mid-session"
cursor.execute(update_sessionscache_mid, update_mid)
elif (num > 1):
session_ids = [session_records[0][0], session_records[1][0]]
session_ids.sort()
# Multiple matches found - merge them into one session and update:
# - Obtain the session start and end times for the new combined session
cursor.execute(merge_sessionscache, session_ids)
merge = cursor.fetchone()
# - Delete the old records
for id in session_ids:
cursor.execute(delete_sessions, id)
# - Insert the new updated record
cursor.execute(insert_sessionscache, merge)
# - Obtain the new sessionId and write over the old ids in Hands
id = self.get_last_insert_id(cursor) #grab the sessionId
update_hands = [id] + session_ids
cursor.execute(update_hands_sessionid, update_hands)
# - Update the newly combined record in SessionsCache with data from this hand
update_mid = row + row[-1:]
cursor.execute(update_sessionscache_mid, update_mid)
elif (num == 0):
# No matches found, insert new session:
insert = row + row[-1:]
insert = insert[-2:] + insert[:-2]
#print "DEBUG: No matches found. Insert record", insert
cursor.execute(insert_sessionscache, insert)
id = self.get_last_insert_id(cursor) #grab the sessionId
else:
# Something bad happened
pass
def nextHandId(self):
c = self.get_cursor()
c.execute("SELECT max(id) FROM Hands")
id = c.fetchone()[0]
if not id: id = 0
id += 1
return id return id
def isDuplicate(self, gametypeID, siteHandNo): def isDuplicate(self, gametypeID, siteHandNo):

7
pyfpdb/DerivedStats.py
View File

@ -123,6 +123,7 @@ class DerivedStats():
self.hands['siteHandNo'] = hand.handid self.hands['siteHandNo'] = hand.handid
self.hands['gametypeId'] = None # Leave None, handled later after checking db self.hands['gametypeId'] = None # Leave None, handled later after checking db
self.hands['sessionId'] = None # Leave None, added later if caching sessions self.hands['sessionId'] = None # Leave None, added later if caching sessions
self.hands['gameSessionId'] = None # Leave None, added later if caching sessions
self.hands['startTime'] = hand.startTime # format this! self.hands['startTime'] = hand.startTime # format this!
self.hands['importTime'] = None self.hands['importTime'] = None
self.hands['seats'] = self.countPlayers(hand) self.hands['seats'] = self.countPlayers(hand)
@ -213,10 +214,10 @@ class DerivedStats():
for player in hand.players: for player in hand.players:
hcs = hand.join_holecards(player[1], asList=True) hcs = hand.join_holecards(player[1], asList=True)
hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x'] hcs = hcs + [u'0x']*18
#for i, card in enumerate(hcs[:7], 1): #Python 2.6 syntax #for i, card in enumerate(hcs[:20, 1): #Python 2.6 syntax
# self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card) # self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card)
for i, card in enumerate(hcs[:7]): for i, card in enumerate(hcs[:20]):
self.handsplayers[player[1]]['card%s' % (i+1)] = Card.encodeCard(card) self.handsplayers[player[1]]['card%s' % (i+1)] = Card.encodeCard(card)
self.handsplayers[player[1]]['startCards'] = Card.calcStartCards(hand, player[1]) self.handsplayers[player[1]]['startCards'] = Card.calcStartCards(hand, player[1])

2
pyfpdb/EverestToFpdb.py
View File

@ -35,7 +35,7 @@ class Everest(HandHistoryConverter):
siteID = 15 siteID = 15
substitutions = { substitutions = {
'LS' : u"\$|\xe2\x82\xac|\u20ac|", 'LS' : u"\$|\u20ac|",
'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename
} }

3
pyfpdb/FullTiltPokerSummary.py
View File

@ -25,7 +25,6 @@ import datetime
from Exceptions import FpdbParseError from Exceptions import FpdbParseError
from HandHistoryConverter import * from HandHistoryConverter import *
import PokerStarsToFpdb
from TourneySummary import * from TourneySummary import *
class FullTiltPokerSummary(TourneySummary): class FullTiltPokerSummary(TourneySummary):
@ -46,7 +45,7 @@ class FullTiltPokerSummary(TourneySummary):
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : "\$|\xe2\x82\xac|", # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20AC|", # legal currency symbols - Euro(cp1252, utf-8)
'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename 'TAB' : u"-\u2013'\s\da-zA-Z", # legal characters for tablename
'NUM' : u".,\d", # legal characters in number format 'NUM' : u".,\d", # legal characters in number format
} }

6
pyfpdb/FulltiltToFpdb.py
View File

@ -172,7 +172,7 @@ class Fulltilt(HandHistoryConverter):
self.re_BringIn = re.compile(r"^%(PLAYERS)s brings in for [%(LS)s]?(?P<BRINGIN>[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_BringIn = re.compile(r"^%(PLAYERS)s brings in for [%(LS)s]?(?P<BRINGIN>[%(NUM)s]+)" % self.substitutions, re.MULTILINE)
self.re_PostBoth = re.compile(r"^%(PLAYERS)s posts small \& big blinds \[[%(LS)s]? (?P<SBBB>[%(NUM)s]+)" % self.substitutions, re.MULTILINE) self.re_PostBoth = re.compile(r"^%(PLAYERS)s posts small \& big blinds \[[%(LS)s]? (?P<SBBB>[%(NUM)s]+)" % self.substitutions, re.MULTILINE)
self.re_HeroCards = re.compile(r"^Dealt to %s(?: \[(?P<OLDCARDS>.+?)\])?( \[(?P<NEWCARDS>.+?)\])" % player_re, re.MULTILINE) self.re_HeroCards = re.compile(r"^Dealt to %s(?: \[(?P<OLDCARDS>.+?)\])?( \[(?P<NEWCARDS>.+?)\])" % player_re, re.MULTILINE)
self.re_Action = re.compile(r"^%(PLAYERS)s(?P<ATYPE> bets| checks| raises to| completes it to| calls| folds)( [%(LS)s]?(?P<BET>[%(NUM)s]+))?" % self.substitutions, re.MULTILINE) self.re_Action = re.compile(r"^%(PLAYERS)s(?P<ATYPE> bets| checks| raises to| completes it to| calls| folds| discards| stands pat)( [%(LS)s]?(?P<BET>[%(NUM)s]+))?(\son|\scards?)?(\s\[(?P<CARDS>.+?)\])?" % self.substitutions, re.MULTILINE)
self.re_ShowdownAction = re.compile(r"^%s shows \[(?P<CARDS>.*)\]" % player_re, re.MULTILINE) self.re_ShowdownAction = re.compile(r"^%s shows \[(?P<CARDS>.*)\]" % player_re, re.MULTILINE)
self.re_CollectPot = re.compile(r"^Seat (?P<SEAT>[0-9]+): %(PLAYERS)s (\(button\) |\(small blind\) |\(big blind\) )?(collected|showed \[.*\] and won) \([%(LS)s]?(?P<POT>[%(NUM)s]+)\)(, mucked| with.*)?" % self.substitutions, re.MULTILINE) self.re_CollectPot = re.compile(r"^Seat (?P<SEAT>[0-9]+): %(PLAYERS)s (\(button\) |\(small blind\) |\(big blind\) )?(collected|showed \[.*\] and won) \([%(LS)s]?(?P<POT>[%(NUM)s]+)\)(, mucked| with.*)?" % self.substitutions, re.MULTILINE)
self.re_SitsOut = re.compile(r"^%s sits out" % player_re, re.MULTILINE) self.re_SitsOut = re.compile(r"^%s sits out" % player_re, re.MULTILINE)
@ -483,6 +483,10 @@ class Fulltilt(HandHistoryConverter):
hand.addFold( street, action.group('PNAME')) hand.addFold( street, action.group('PNAME'))
elif action.group('ATYPE') == ' checks': elif action.group('ATYPE') == ' checks':
hand.addCheck( street, action.group('PNAME')) hand.addCheck( street, action.group('PNAME'))
elif action.group('ATYPE') == ' discards':
hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS'))
elif action.group('ATYPE') == ' stands pat':
hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS'))
else: else:
print _("FullTilt: DEBUG: unimplemented readAction: '%s' '%s'") %(action.group('PNAME'),action.group('ATYPE'),) print _("FullTilt: DEBUG: unimplemented readAction: '%s' '%s'") %(action.group('PNAME'),action.group('ATYPE'),)

7
pyfpdb/GuiTourneyImport.py
View File

@ -223,13 +223,18 @@ class SummaryImporter:
print "Found %s summaries" %(len(summaryTexts)) print "Found %s summaries" %(len(summaryTexts))
errors = 0 errors = 0
imported = 0 imported = 0
####Lock Placeholder####
for j, summaryText in enumerate(summaryTexts, start=1): for j, summaryText in enumerate(summaryTexts, start=1):
sc, gsc = {'bk': []}, {'bk': []}
doinsert = len(summaryTexts)==j
try: try:
conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") conv = obj(db=self.database, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP")
sc, gsc = conv.updateSessionsCache(sc, gsc, None, doinsert)
except FpdbParseError, e: except FpdbParseError, e:
errors += 1 errors += 1
print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts)) print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts))
imported = j imported = j
####Lock Placeholder####
return (imported - errors, errors) return (imported - errors, errors)
def clearFileList(self): def clearFileList(self):

123
pyfpdb/Hand.py
View File

@ -57,6 +57,7 @@ class Hand(object):
#log.debug( _("Hand.init(): handText is ") + str(handText) ) #log.debug( _("Hand.init(): handText is ") + str(handText) )
self.config = config self.config = config
self.saveActions = self.config.get_import_parameters().get('saveActions') self.saveActions = self.config.get_import_parameters().get('saveActions')
self.callHud = self.config.get_import_parameters().get("callFpdbHud")
self.cacheSessions = self.config.get_import_parameters().get("cacheSessions") self.cacheSessions = self.config.get_import_parameters().get("cacheSessions")
#log = Configuration.get_logger("logging.conf", "db", log_dir=self.config.dir_log) #log = Configuration.get_logger("logging.conf", "db", log_dir=self.config.dir_log)
self.sitename = sitename self.sitename = sitename
@ -227,83 +228,77 @@ dealt whether they were seen in a 'dealt to' line
self.holecards[street][player] = [open, closed] self.holecards[street][player] = [open, closed]
def prepInsert(self, db, printtest = False): def prepInsert(self, db):
##### #####
# Players, Gametypes, TourneyTypes are all shared functions that are needed for additional tables # Players, Gametypes, TourneyTypes are all shared functions that are needed for additional tables
# These functions are intended for prep insert eventually # These functions are intended for prep insert eventually
##### #####
# Players - base playerid and siteid tuple
self.dbid_pids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId) self.dbid_pids = db.getSqlPlayerIDs([p[1] for p in self.players], self.siteId)
self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype)
#Gametypes
hilo = "h"
if self.gametype['category'] in ['studhilo', 'omahahilo']:
hilo = "s"
elif self.gametype['category'] in ['razz','27_3draw','badugi', '27_1draw']:
hilo = "l"
self.gametyperow = (self.siteId, self.gametype['currency'], self.gametype['type'], self.gametype['base'],
self.gametype['category'], self.gametype['limitType'], hilo,
int(Decimal(self.gametype['sb'])*100), int(Decimal(self.gametype['bb'])*100),
int(Decimal(self.gametype['bb'])*100), int(Decimal(self.gametype['bb'])*200))
# Note: the above data is calculated in db.getGameTypeId
# Only being calculated above so we can grab the testdata
self.dbid_gt = db.getGameTypeId(self.siteId, self.gametype, printdata = printtest)
if self.tourNo!=None: if self.tourNo!=None:
self.tourneyTypeId = db.createTourneyType(self) self.tourneyTypeId = db.createTourneyType(self)
db.commit()
self.tourneyId = db.createOrUpdateTourney(self, "HHC") self.tourneyId = db.createOrUpdateTourney(self, "HHC")
db.commit()
self.tourneysPlayersIds = db.createOrUpdateTourneysPlayers(self, "HHC") self.tourneysPlayersIds = db.createOrUpdateTourneysPlayers(self, "HHC")
db.commit() #db.commit() #commit these transactions'
#end def prepInsert
def insert(self, db, hp_data = None, ha_data = None, insert_data=False, printtest = False): def assembleHand(self):
self.stats.getStats(self)
self.hands = self.stats.getHands()
self.handsplayers = self.stats.getHandsPlayers()
def getHandId(self, db, id):
if db.isDuplicate(self.dbid_gt, self.hands['siteHandNo']):
#log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo'])
self.is_duplicate = True # i.e. don't update hudcache
next = id
raise FpdbHandDuplicate(self.hands['siteHandNo'])
else:
self.dbid_hands = id
self.hands['id'] = self.dbid_hands
next = id +1
return next
def insertHands(self, db, hbulk, doinsert = False, printtest = False):
""" Function to insert Hand into database """ Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected Database object""" db: a connected Database object"""
self.hands['gameTypeId'] = self.dbid_gt
self.hands['seats'] = len(self.dbid_pids)
hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest)
return hbulk
def insertHandsPlayers(self, db, hpbulk, doinsert = False, printtest = False):
""" Function to inserts HandsPlayers into database"""
hpbulk = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, self.handsplayers, hpbulk, doinsert, printtest)
return hpbulk
self.stats.getStats(self) def insertHandsActions(self, db, habulk, doinsert = False, printtest = False):
""" Function to inserts HandsActions into database"""
handsactions = self.stats.getHandsActions()
habulk = db.storeHandsActions(self.dbid_hands, self.dbid_pids, handsactions, habulk, doinsert, printtest)
return habulk
##### def updateHudCache(self, db, hcbulk, doinsert = False):
# End prep functions """ Function to update the HudCache"""
##### if self.callHud:
hh = self.stats.getHands() hcbulk = db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.handsplayers, hcbulk, doinsert)
hp_inserts, ha_inserts = [], [] return hcbulk
if not db.isDuplicate(self.dbid_gt, hh['siteHandNo']):
# Hands - Summary information of hand indexed by handId - gameinfo
hh['gametypeId'] = self.dbid_gt
# seats TINYINT NOT NULL,
hh['seats'] = len(self.dbid_pids)
hp = self.stats.getHandsPlayers()
def updateSessionsCache(self, db, sc, gsc, tz, doinsert = False):
""" Function to update the SessionsCache"""
if self.cacheSessions: if self.cacheSessions:
hh['sessionId'] = db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, hp) self.heros = db.getHeroIds(self.dbid_pids, self.sitename)
sc = db.prepSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, sc, self.heros, doinsert)
self.dbid_hands = db.storeHand(hh, printdata = printtest) gsc = db.storeSessionsCache(self.dbid_hands, self.dbid_pids, self.startTime, self.gametype
,self.dbid_gt, self.handsplayers, sc, gsc, tz, self.heros, doinsert)
hp_inserts = db.storeHandsPlayers(self.dbid_hands, self.dbid_pids, hp, if doinsert:
insert=insert_data, hp_bulk = hp_data, printdata = printtest) self.hands['sc'] = sc
self.hands['gsc'] = gsc
if self.saveActions:
ha_inserts = db.storeHandsActions(self.dbid_hands, self.dbid_pids, self.stats.getHandsActions(),
insert=insert_data, ha_bulk = ha_data, printdata = printtest)
else: else:
log.info(_("Hand.insert(): hid #: %s is a duplicate") % hh['siteHandNo']) self.hands['sc'] = None
self.is_duplicate = True # i.e. don't update hudcache self.hands['gsc'] = None
raise FpdbHandDuplicate(hh['siteHandNo']) return sc, gsc
return hp_inserts, ha_inserts
def updateHudCache(self, db):
db.storeHudCache(self.dbid_gt, self.dbid_pids, self.startTime, self.stats.getHandsPlayers())
def updateSessionsCache(self, db):
db.storeSessionsCache(self.dbid_pids, self.startTime, self.gametype, self.stats.getHandsPlayers())
def select(self, db, handId): def select(self, db, handId):
""" Function to create Hand object from database """ """ Function to create Hand object from database """
@ -666,10 +661,13 @@ Add a raise on [street] by [player] to [amountTo]
self.pot.addMoney(player, amount) self.pot.addMoney(player, amount)
def addStandsPat(self, street, player): def addStandsPat(self, street, player, cards):
self.checkPlayerExists(player) self.checkPlayerExists(player)
act = (player, 'stands pat') act = (player, 'stands pat')
self.actions[street].append(act) self.actions[street].append(act)
if cards:
cards = cards.split(' ')
self.addHoleCards(street, player, open=[], closed=cards)
def addFold(self, street, player): def addFold(self, street, player):
@ -1229,7 +1227,14 @@ class DrawHand(Hand):
def join_holecards(self, player, asList=False): def join_holecards(self, player, asList=False):
"""With asList = True it returns the set cards for a player including down cards if they aren't know""" """With asList = True it returns the set cards for a player including down cards if they aren't know"""
# FIXME: This should actually return # FIXME: This should actually return
holecards = [u'0x', u'0x', u'0x', u'0x', u'0x'] holecards = [u'0x']*20
for i, street in enumerate(self.holeStreets):
if player in self.holecards[street].keys():
allhole = self.holecards[street][player][0] + self.holecards[street][player][1]
for c in range(len(allhole)):
idx = c + (i*5)
holecards[idx] = allhole[c]
if asList == False: if asList == False:
return " ".join(holecards) return " ".join(holecards)

11
pyfpdb/HandHistoryConverter.py
View File

@ -50,9 +50,6 @@ import Hand
from Exceptions import FpdbParseError from Exceptions import FpdbParseError
import Configuration import Configuration
import pygtk
import gtk
class HandHistoryConverter(): class HandHistoryConverter():
READ_CHUNK_SIZE = 10000 # bytes to read at a time from file in tail mode READ_CHUNK_SIZE = 10000 # bytes to read at a time from file in tail mode
@ -128,9 +125,6 @@ If in follow mode, wait for more data to turn up.
Otherwise, finish at EOF. Otherwise, finish at EOF.
""" """
while gtk.events_pending():
gtk.main_iteration(False)
starttime = time.time() starttime = time.time()
if not self.sanityCheck(): if not self.sanityCheck():
log.warning(_("Failed sanity check")) log.warning(_("Failed sanity check"))
@ -183,6 +177,11 @@ Otherwise, finish at EOF.
if self.out_fh != sys.stdout: if self.out_fh != sys.stdout:
self.out_fh.close() self.out_fh.close()
def progressNotify(self):
"A callback to the interface while events are pending"
import gtk, pygtk
while gtk.events_pending():
gtk.main_iteration(False)
def tailHands(self): def tailHands(self):
"""Generator of handTexts from a tailed file: """Generator of handTexts from a tailed file:

6
pyfpdb/OnGameToFpdb.py
View File

@ -42,12 +42,12 @@ class OnGame(HandHistoryConverter):
siteId = 5 # Needs to match id entry in Sites database siteId = 5 # Needs to match id entry in Sites database
mixes = { } # Legal mixed games mixes = { } # Legal mixed games
sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20ac", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : u"\$|\xe2\x82\xac|\u20ac" # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\\u20AC" # legal currency symbols - Euro(cp1252, utf-8)
} }
currencies = { u'\u20ac':'EUR', u'\xe2\x82\xac':'EUR', '$':'USD', '':'T$' } currencies = { u'\u20AC':'EUR', '$':'USD', '':'T$' }
limits = { 'NO_LIMIT':'nl', 'POT_LIMIT':'pl', 'LIMIT':'fl'} limits = { 'NO_LIMIT':'nl', 'POT_LIMIT':'pl', 'LIMIT':'fl'}

4
pyfpdb/PacificPokerToFpdb.py
View File

@ -39,10 +39,10 @@ class PacificPoker(HandHistoryConverter):
siteId = 13 # Needs to match id entry in Sites database siteId = 13 # Needs to match id entry in Sites database
mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games
sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8)
} }
# translations from captured groups to fpdb info strings # translations from captured groups to fpdb info strings

4
pyfpdb/PartyPokerToFpdb.py
View File

@ -46,10 +46,10 @@ class PartyPoker(HandHistoryConverter):
siteId = 9 siteId = 9
filetype = "text" filetype = "text"
sym = {'USD': "\$", 'EUR': u"\u20ac", 'T$': ""} sym = {'USD': "\$", 'EUR': u"\u20ac", 'T$': ""}
currencies = {"\$": "USD", "$": "USD", u"\xe2\x82\xac": "EUR", u"\u20ac": "EUR", '': "T$"} currencies = {"\$": "USD", "$": "USD", u"\u20ac": "EUR", '': "T$"}
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR", # legal ISO currency codes
'LS' : u"\$|\u20ac|\xe2\x82\xac|", # Currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20ac|", # Currency symbols - Euro(cp1252, utf-8)
'NUM' : u".,\d", 'NUM' : u".,\d",
} }

2
pyfpdb/PokerStarsSummary.py
View File

@ -45,7 +45,7 @@ class PokerStarsSummary(TourneySummary):
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8)
} }
re_SplitTourneys = re.compile("PokerStars Tournament ") re_SplitTourneys = re.compile("PokerStars Tournament ")

11
pyfpdb/PokerStarsToFpdb.py
View File

@ -39,10 +39,10 @@ class PokerStars(HandHistoryConverter):
siteId = 2 # Needs to match id entry in Sites database siteId = 2 # Needs to match id entry in Sites database
mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games
sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3", "play": ""} # ADD Euro, Sterling, etc HERE
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8)
} }
# translations from captured groups to fpdb info strings # translations from captured groups to fpdb info strings
@ -145,7 +145,8 @@ class PokerStars(HandHistoryConverter):
(\s(%(CUR)s)?(?P<BET>[.\d]+))?(\sto\s%(CUR)s(?P<BETTO>[.\d]+))? # the number discarded goes in <BET> (\s(%(CUR)s)?(?P<BET>[.\d]+))?(\sto\s%(CUR)s(?P<BETTO>[.\d]+))? # the number discarded goes in <BET>
\s*(and\sis\sall.in)? \s*(and\sis\sall.in)?
(and\shas\sreached\sthe\s[%(CUR)s\d\.]+\scap)? (and\shas\sreached\sthe\s[%(CUR)s\d\.]+\scap)?
(\scards?(\s\[(?P<DISCARDED>.+?)\])?)?\s*$""" (\son|\scards?)?
(\s\[(?P<CARDS>.+?)\])?\s*$"""
% short_subst, re.MULTILINE|re.VERBOSE) % short_subst, re.MULTILINE|re.VERBOSE)
re_ShowdownAction = re.compile(r"^%s: shows \[(?P<CARDS>.*)\]" % short_subst['PLYR'], re.MULTILINE) re_ShowdownAction = re.compile(r"^%s: shows \[(?P<CARDS>.*)\]" % short_subst['PLYR'], re.MULTILINE)
re_sitsOut = re.compile("^%s sits out" % short_subst['PLYR'], re.MULTILINE) re_sitsOut = re.compile("^%s sits out" % short_subst['PLYR'], re.MULTILINE)
@ -432,9 +433,9 @@ class PokerStars(HandHistoryConverter):
elif action.group('ATYPE') == ' checks': elif action.group('ATYPE') == ' checks':
hand.addCheck( street, action.group('PNAME')) hand.addCheck( street, action.group('PNAME'))
elif action.group('ATYPE') == ' discards': elif action.group('ATYPE') == ' discards':
hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('DISCARDED')) hand.addDiscard(street, action.group('PNAME'), action.group('BET'), action.group('CARDS'))
elif action.group('ATYPE') == ' stands pat': elif action.group('ATYPE') == ' stands pat':
hand.addStandsPat( street, action.group('PNAME')) hand.addStandsPat( street, action.group('PNAME'), action.group('CARDS'))
else: else:
print (_("DEBUG: ") + _("Unimplemented readAction: '%s' '%s'") % (action.group('PNAME'),action.group('ATYPE'))) print (_("DEBUG: ") + _("Unimplemented readAction: '%s' '%s'") % (action.group('PNAME'),action.group('ATYPE')))

348
pyfpdb/SQL.py
View File

@ -108,6 +108,15 @@ class Sql:
self.query['createSettingsTable'] = """CREATE TABLE Settings self.query['createSettingsTable'] = """CREATE TABLE Settings
(version INTEGER NOT NULL) """ (version INTEGER NOT NULL) """
################################
# Create InsertLock
################################
if db_server == 'mysql':
self.query['createLockTable'] = """CREATE TABLE InsertLock (
id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id),
locked BOOLEAN NOT NULL DEFAULT FALSE)
ENGINE=INNODB"""
################################ ################################
# Create RawHands (this table is all but identical with RawTourneys) # Create RawHands (this table is all but identical with RawTourneys)
################################ ################################
@ -347,6 +356,7 @@ class Sql:
tourneyId INT UNSIGNED, tourneyId INT UNSIGNED,
gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
sessionId INT UNSIGNED, sessionId INT UNSIGNED,
gameSessionId INT UNSIGNED,
startTime DATETIME NOT NULL, startTime DATETIME NOT NULL,
importTime DATETIME NOT NULL, importTime DATETIME NOT NULL,
seats TINYINT NOT NULL, seats TINYINT NOT NULL,
@ -385,6 +395,7 @@ class Sql:
tourneyId INT, tourneyId INT,
gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
sessionId INT, sessionId INT,
gameSessionId INT,
startTime timestamp without time zone NOT NULL, startTime timestamp without time zone NOT NULL,
importTime timestamp without time zone NOT NULL, importTime timestamp without time zone NOT NULL,
seats SMALLINT NOT NULL, seats SMALLINT NOT NULL,
@ -422,6 +433,7 @@ class Sql:
tourneyId INT, tourneyId INT,
gametypeId INT NOT NULL, gametypeId INT NOT NULL,
sessionId INT, sessionId INT,
gameSessionId INT,
startTime REAL NOT NULL, startTime REAL NOT NULL,
importTime REAL NOT NULL, importTime REAL NOT NULL,
seats INT NOT NULL, seats INT NOT NULL,
@ -621,6 +633,19 @@ class Sql:
card5 smallint, card5 smallint,
card6 smallint, card6 smallint,
card7 smallint, card7 smallint,
card8 smallint, /* cards 8-20 for draw hands */
card9 smallint,
card10 smallint,
card11 smallint,
card12 smallint,
card13 smallint,
card14 smallint,
card15 smallint,
card16 smallint,
card17 smallint,
card18 smallint,
card19 smallint,
card20 smallint,
startCards smallint, startCards smallint,
ante INT, ante INT,
@ -748,6 +773,19 @@ class Sql:
card5 smallint, card5 smallint,
card6 smallint, card6 smallint,
card7 smallint, card7 smallint,
card8 smallint, /* cards 8-20 for draw hands */
card9 smallint,
card10 smallint,
card11 smallint,
card12 smallint,
card13 smallint,
card14 smallint,
card15 smallint,
card16 smallint,
card17 smallint,
card18 smallint,
card19 smallint,
card20 smallint,
startCards smallint, startCards smallint,
ante INT, ante INT,
@ -874,6 +912,19 @@ class Sql:
card5 INT, card5 INT,
card6 INT, card6 INT,
card7 INT, card7 INT,
card8 INT, /* cards 8-20 for draw hands */
card9 INT,
card10 INT,
card11 INT,
card12 INT,
card13 INT,
card14 INT,
card15 INT,
card16 INT,
card17 INT,
card18 INT,
card19 INT,
card20 INT,
startCards INT, startCards INT,
ante INT, ante INT,
@ -1430,33 +1481,62 @@ class Sql:
id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), id BIGINT UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id),
sessionStart DATETIME NOT NULL, sessionStart DATETIME NOT NULL,
sessionEnd DATETIME NOT NULL, sessionEnd DATETIME NOT NULL,
ringHDs INT NOT NULL, gameStart DATETIME NOT NULL,
tourHDs INT NOT NULL, gameEnd DATETIME NOT NULL,
ringProfitUSD INT NOT NULL, sessionId BIGINT,
ringProfitEUR INT NOT NULL) date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */
type char(7) NOT NULL,
gametypeId SMALLINT UNSIGNED, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
tourneyTypeId SMALLINT UNSIGNED, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
playerId INT UNSIGNED NOT NULL, FOREIGN KEY (playerId) REFERENCES Players(id),
hands INT NOT NULL,
tourneys INT NOT NULL,
totalProfit INT)
ENGINE=INNODB
"""
ENGINE=INNODB"""
elif db_server == 'postgresql': elif db_server == 'postgresql':
self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache (
id BIGSERIAL, PRIMARY KEY (id), id BIGSERIAL, PRIMARY KEY (id),
sessionStart REAL NOT NULL, sessionStart REAL NOT NULL,
sessionEnd REAL NOT NULL, sessionEnd REAL NOT NULL,
ringHDs INT NOT NULL, gameStart REAL NOT NULL,
tourHDs INT NOT NULL, gameEnd REAL NOT NULL,
ringProfitUSD INT NOT NULL, sessionId INT,
ringProfitEUR INT NOT NULL) date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */
type char(7),
gametypeId INT, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
tourneyTypeId INT, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
playerId INT, FOREIGN KEY (playerId) REFERENCES Players(id),
hands INT,
tourneys INT,
totalProfit INT)
""" """
elif db_server == 'sqlite': elif db_server == 'sqlite':
self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
sessionStart REAL NOT NULL, sessionStart REAL NOT NULL,
sessionEnd REAL NOT NULL, sessionEnd REAL NOT NULL,
ringHDs INT NOT NULL, gameStart REAL NOT NULL,
tourHDs INT NOT NULL, gameEnd REAL NOT NULL,
ringProfitUSD INT NOT NULL, sessionId INT,
ringProfitEUR INT NOT NULL) date TEXT NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */
type TEXT,
gametypeId INT,
tourneyTypeId INT,
playerId INT,
hands INT,
tourneys INT,
totalProfit INT)
""" """
self.query['addSessionIdIndex'] = """CREATE INDEX index_SessionId ON SessionsCache (sessionId)"""
self.query['addHandsSessionIdIndex'] = """CREATE INDEX index_handsSessionId ON Hands (sessionId)"""
self.query['addHandsGameSessionIdIndex'] = """CREATE INDEX index_handsGameSessionId ON Hands (gameSessionId)"""
if db_server == 'mysql': if db_server == 'mysql':
self.query['addTourneyIndex'] = """ALTER TABLE Tourneys ADD UNIQUE INDEX siteTourneyNo(siteTourneyNo, tourneyTypeId)""" self.query['addTourneyIndex'] = """ALTER TABLE Tourneys ADD UNIQUE INDEX siteTourneyNo(siteTourneyNo, tourneyTypeId)"""
elif db_server == 'postgresql': elif db_server == 'postgresql':
@ -1479,7 +1559,7 @@ class Sql:
self.query['addPlayersIndex'] = """CREATE UNIQUE INDEX name ON Players (name, siteId)""" self.query['addPlayersIndex'] = """CREATE UNIQUE INDEX name ON Players (name, siteId)"""
if db_server == 'mysql': if db_server == 'mysql':
self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX tourneyId(tourneyId, playerId)""" self.query['addTPlayersIndex'] = """ALTER TABLE TourneysPlayers ADD UNIQUE INDEX _tourneyId(tourneyId, playerId)"""
elif db_server == 'postgresql': elif db_server == 'postgresql':
self.query['addTPlayersIndex'] = """CREATE UNIQUE INDEX tourneyId ON TourneysPlayers (tourneyId, playerId)""" self.query['addTPlayersIndex'] = """CREATE UNIQUE INDEX tourneyId ON TourneysPlayers (tourneyId, playerId)"""
elif db_server == 'sqlite': elif db_server == 'sqlite':
@ -4142,7 +4222,7 @@ class Sql:
""" """
self.query['insert_hudcache'] = """ self.query['insert_hudcache'] = """
INSERT INTO HudCache ( insert into HudCache (
gametypeId, gametypeId,
playerId, playerId,
activeSeats, activeSeats,
@ -4237,7 +4317,7 @@ class Sql:
street2Raises, street2Raises,
street3Raises, street3Raises,
street4Raises) street4Raises)
VALUES (%s, %s, %s, %s, %s, values (%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
@ -4364,95 +4444,135 @@ class Sql:
# Queries to rebuild/modify sessionscache # Queries to rebuild/modify sessionscache
#################################### ####################################
self.query['select_sessionscache'] = """ self.query['clearSessionsCache'] = """DELETE FROM SessionsCache"""
self.query['rebuildSessionsCache'] = """
SELECT Hands.id as id,
Hands.startTime as startTime,
HandsPlayers.playerId as playerId,
Hands.gametypeId as gametypeId,
Gametypes.type as game,
HandsPlayers.totalProfit as totalProfit,
Tourneys.tourneyTypeId as tourneyTypeId
FROM Gametypes, HandsPlayers, Hands
LEFT JOIN Tourneys ON Hands.tourneyId = Tourneys.tourneyTypeId
WHERE HandsPlayers.handId = Hands.id
AND Hands.gametypeId = Gametypes.id
AND (case when HandsPlayers.playerId = <where_clause> then 1 else 0 end) = 1
ORDER BY Hands.startTime ASC"""
self.query['rebuildSessionsCacheSum'] = """
SELECT Tourneys.id as id,
Tourneys.startTime as startTime,
TourneysPlayers.playerId,
TourneyTypes.id as tourneyTypeId,
TourneysPlayers.winnings as winnings,
TourneysPlayers.winningsCurrency as winningsCurrency,
TourneyTypes.currency as buyinCurrency,
TourneyTypes.buyIn as buyIn,
TourneyTypes.fee as fee,
case when TourneyTypes.rebuy then TourneyTypes.rebuyCost else 0 end as rebuyCost,
case when TourneyTypes.rebuy then TourneyTypes.rebuyFee else 0 end as rebuyFee,
case when TourneyTypes.addOn then TourneyTypes.addOnCost else 0 end as addOnCost,
case when TourneyTypes.addOn then TourneyTypes.addOnFee else 0 end as addOnFee,
case when TourneyTypes.knockout then TourneyTypes.koBounty else 0 end as koBounty
FROM Tourneys, TourneyTypes, TourneysPlayers
WHERE Tourneys.tourneyTypeId = TourneyTypes.id
AND Tourneys.id = TourneysPlayers.tourneyId
AND (case when TourneysPlayers.playerId = <where_clause> then 1 else 0 end) = 1
ORDER BY Tourneys.startTime ASC"""
self.query['select_prepSC'] = """
SELECT sessionId as id,
sessionStart,
sessionEnd,
count(sessionId) as count
FROM SessionsCache
WHERE sessionEnd>=%s
AND sessionStart<=%s
GROUP BY sessionId, sessionStart, sessionEnd"""
self.query['update_prepSC'] = """
UPDATE SessionsCache SET
sessionStart=%s,
sessionEnd=%s
WHERE sessionId=%s"""
self.query['update_SC'] = """
UPDATE SessionsCache SET
sessionStart=%s,
sessionEnd=%s,
gameStart=%s,
gameEnd=%s,
hands=hands+%s,
tourneys=tourneys+%s,
totalProfit=totalProfit+%s
WHERE id=%s"""
self.query['select_SC'] = """
SELECT id, SELECT id,
sessionStart, sessionStart,
sessionEnd, sessionEnd,
ringHDs, gameStart,
tourHDs, gameEnd,
ringProfitUSD, sessionId,
ringProfitEUR date,
type,
gametypeId,
tourneyTypeId,
playerId,
hands,
tourneys,
totalProfit
FROM SessionsCache FROM SessionsCache
WHERE sessionEnd>=%s WHERE gameEnd>=%s
AND sessionStart<=%s""" AND gameStart<=%s
AND date=%s
AND type=%s
AND (case when gametypeId is NULL then 1 else
(case when gametypeId=%s then 1 else 0 end) end)=1
AND (case when tourneyTypeId is NULL then 1 else
(case when tourneyTypeId=%s then 1 else 0 end) end)=1
AND playerId=%s"""
self.query['select_sessionscache_mid'] = """ self.query['insert_SC'] = """
SELECT sessionStart, insert into SessionsCache (
sessionEnd,
ringHDs,
tourHDs,
ringProfitUSD,
ringProfitEUR
FROM SessionsCache
WHERE sessionEnd>=%s
AND sessionStart<=%s"""
self.query['select_sessionscache_start'] = """
SELECT sessionStart,
sessionEnd,
ringHDs,
tourHDs,
ringProfitUSD,
ringProfitEUR
FROM SessionsCache
WHERE sessionStart>%s
AND sessionEnd>=%s
AND sessionStart<=%s"""
self.query['update_sessionscache_mid'] = """
UPDATE SessionsCache SET
ringHDs=ringHDs+%s,
tourHDs=tourHDs+%s,
ringProfitUSD=ringProfitUSD+%s,
ringProfitEUR=ringProfitEUR+%s
WHERE sessionStart<=%s
AND sessionEnd>=%s"""
self.query['update_sessionscache_start'] = """
UPDATE SessionsCache SET
sessionStart=%s,
ringHDs=ringHDs+%s,
tourHDs=tourHDs+%s,
ringProfitUSD=ringProfitUSD+%s,
ringProfitEUR=ringProfitEUR+%s
WHERE sessionStart>%s
AND sessionEnd>=%s
AND sessionStart<=%s"""
self.query['update_sessionscache_end'] = """
UPDATE SessionsCache SET
sessionEnd=%s,
ringHDs=ringHDs+%s,
tourHDs=tourHDs+%s,
ringProfitUSD=ringProfitUSD+%s,
ringProfitEUR=ringProfitEUR+%s
WHERE sessionEnd<%s
AND sessionEnd>=%s
AND sessionStart<=%s"""
self.query['insert_sessionscache'] = """
INSERT INTO SessionsCache (
sessionStart, sessionStart,
sessionEnd, sessionEnd,
ringHDs, gameStart,
tourHDs, gameEnd,
ringProfitUSD, sessionId,
ringProfitEUR) date,
VALUES (%s, %s, %s, %s, %s, %s)""" type,
gametypeId,
tourneyTypeId,
playerId,
hands,
tourneys,
totalProfit)
values (%s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s)"""
self.query['merge_sessionscache'] = """ self.query['update_Hands_gsid'] = """
SELECT min(sessionStart), max(sessionEnd), sum(ringHDs), sum(tourHDs), sum(ringProfitUSD), sum(ringProfitEUR) UPDATE Hands SET
FROM SessionsCache gameSessionId=%s
WHERE (case when id=%s or id=%s then 1 else 0 end)=1""" WHERE gameSessionId=%s"""
self.query['delete_sessions'] = """ self.query['update_Hands_sid'] = """
DELETE FROM SessionsCache
WHERE id=%s"""
self.query['update_hands_sessionid'] = """
UPDATE Hands SET UPDATE Hands SET
sessionId=%s sessionId=%s
WHERE (case when sessionId=%s or sessionId=%s then 1 else 0 end)=1""" WHERE sessionId=%s"""
self.query['update_SC_sid'] = """
UPDATE SessionsCache SET
sessionStart=%s,
sessionEnd=%s,
sessionId=%s
WHERE sessionId=%s"""
self.query['delete_SC'] = """
DELETE FROM SessionsCache
WHERE id=%s"""
#################################### ####################################
# Database management queries # Database management queries
@ -4468,6 +4588,25 @@ class Sql:
elif db_server == 'sqlite': elif db_server == 'sqlite':
self.query['analyze'] = "analyze" self.query['analyze'] = "analyze"
if db_server == 'mysql':
self.query['selectLock'] = """
SELECT locked
FROM InsertLock
WHERE locked=True
LOCK IN SHARE MODE"""
if db_server == 'mysql':
self.query['switchLock'] = """
UPDATE InsertLock SET
locked=%s
WHERE id=%s"""
if db_server == 'mysql':
self.query['missedLock'] = """
UPDATE InsertLock SET
missed=missed+%s
WHERE id=%s"""
if db_server == 'mysql': if db_server == 'mysql':
self.query['lockForInsert'] = """ self.query['lockForInsert'] = """
lock tables Hands write, HandsPlayers write, HandsActions write, Players write lock tables Hands write, HandsPlayers write, HandsActions write, Players write
@ -4652,6 +4791,7 @@ class Sql:
tourneyId, tourneyId,
gametypeid, gametypeid,
sessionId, sessionId,
gameSessionId,
startTime, startTime,
importtime, importtime,
seats, seats,
@ -4682,7 +4822,7 @@ class Sql:
values values
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s)""" %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
self.query['store_hands_players'] = """insert into HandsPlayers ( self.query['store_hands_players'] = """insert into HandsPlayers (
@ -4698,6 +4838,19 @@ class Sql:
card5, card5,
card6, card6,
card7, card7,
card8,
card9,
card10,
card11,
card12,
card13,
card14,
card15,
card16,
card17,
card18,
card19,
card20,
winnings, winnings,
rake, rake,
totalProfit, totalProfit,
@ -4812,7 +4965,10 @@ class Sql:
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s
)""" )"""
self.query['store_hands_actions'] = """insert into HandsActions ( self.query['store_hands_actions'] = """insert into HandsActions (

18
pyfpdb/TourneySummary.py
View File

@ -288,6 +288,24 @@ winnings (int) the money the player ended the tourney with (can be 0, or -1 i
print "checkPlayerExists", player, "fail" print "checkPlayerExists", player, "fail"
raise FpdbParseError raise FpdbParseError
def updateSessionsCache(self, sc, gsc, tz, doinsert):
self.heros = self.db.getHeroIds(self.dbid_pids, self.siteName)
sc = self.db.prepSessionsCache(self.tourNo, self.dbid_pids, self.startTime, sc , self.heros, doinsert)
gsc = self.db.storeSessionsCache(self.tourNo, self.dbid_pids, self.startTime, {'type': 'summary'}
,None, self.assembleInfo(), sc, gsc, tz, self.heros, doinsert)
return sc, gsc
def assembleInfo(self):
info = {}
info['tourneyTypeId'] = self.tourneyTypeId
info['winnings'] = self.winnings
info['winningsCurrency'] = self.winningsCurrency
info['buyinCurrency'] = self.buyinCurrency
info['buyin'] = self.buyin
info['fee'] = self.fee
return info
def writeSummary(self, fh=sys.__stdout__): def writeSummary(self, fh=sys.__stdout__):
print >>fh, "Override me" print >>fh, "Override me"

2
pyfpdb/WinamaxSummary.py
View File

@ -45,7 +45,7 @@ class WinamaxSummary(TourneySummary):
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : u"\$|\xe2\x82\xac|\u20AC|" # legal currency symbols 'LS' : u"\$|\u20AC|" # legal currency symbols
} }
re_GameType = re.compile("""<h1>((?P<LIMIT>No Limit|Pot Limit) (?P<GAME>Hold\'em))</h1>""") re_GameType = re.compile("""<h1>((?P<LIMIT>No Limit|Pot Limit) (?P<GAME>Hold\'em))</h1>""")

4
pyfpdb/WinamaxToFpdb.py
View File

@ -52,10 +52,10 @@ class Winamax(HandHistoryConverter):
siteId = 14 # Needs to match id entry in Sites database siteId = 14 # Needs to match id entry in Sites database
mixes = { } # Legal mixed games mixes = { } # Legal mixed games
sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3"} # ADD Euro, Sterling, etc HERE sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": u"\u20AC", "GBP": u"\xa3"} # ADD Euro, Sterling, etc HERE
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) 'LS' : u"\$|\u20AC|" # legal currency symbols - Euro(cp1252, utf-8)
} }
limits = { 'no limit':'nl', 'pot limit' : 'pl','LIMIT':'fl'} limits = { 'no limit':'nl', 'pot limit' : 'pl','LIMIT':'fl'}

89
pyfpdb/fpdb_import.py
View File

@ -242,7 +242,7 @@ class Importer:
#print "dropInd =", self.settings['dropIndexes'], " dropHudCache =", self.settings['dropHudCache'] #print "dropInd =", self.settings['dropIndexes'], " dropHudCache =", self.settings['dropHudCache']
if self.settings['threads'] <= 0: if self.settings['threads'] <= 0:
(totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, None) (totstored, totdups, totpartial, toterrors) = self.importFiles(None)
else: else:
# create queue (will probably change to deque at some point): # create queue (will probably change to deque at some point):
self.writeq = Queue.Queue( self.settings['writeQSize'] ) self.writeq = Queue.Queue( self.settings['writeQSize'] )
@ -254,7 +254,7 @@ class Importer:
t.setDaemon(True) t.setDaemon(True)
t.start() t.start()
# read hands and write to q: # read hands and write to q:
(totstored, totdups, totpartial, toterrors) = self.importFiles(self.database, self.writeq) (totstored, totdups, totpartial, toterrors) = self.importFiles(self.writeq)
if self.writeq.empty(): if self.writeq.empty():
print _("writers finished already") print _("writers finished already")
@ -286,7 +286,7 @@ class Importer:
return (totstored, totdups, totpartial, toterrors, endtime-starttime) return (totstored, totdups, totpartial, toterrors, endtime-starttime)
# end def runImport # end def runImport
def importFiles(self, db, q): def importFiles(self, q):
""""Read filenames in self.filelist and pass to import_file_dict(). """"Read filenames in self.filelist and pass to import_file_dict().
Uses a separate database connection if created as a thread (caller Uses a separate database connection if created as a thread (caller
passes None or no param as db).""" passes None or no param as db)."""
@ -304,7 +304,7 @@ class Importer:
ProgressDialog.progress_update() ProgressDialog.progress_update()
(stored, duplicates, partial, errors, ttime) = self.import_file_dict(db, file (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file
,self.filelist[file][0], self.filelist[file][1], q) ,self.filelist[file][0], self.filelist[file][1], q)
totstored += stored totstored += stored
totdups += duplicates totdups += duplicates
@ -395,7 +395,7 @@ class Importer:
self.caller.addText("\n"+os.path.basename(file)) self.caller.addText("\n"+os.path.basename(file))
except KeyError: # TODO: What error happens here? except KeyError: # TODO: What error happens here?
pass pass
(stored, duplicates, partial, errors, ttime) = self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None) (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1], None)
try: try:
if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func
self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime)) self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime))
@ -426,68 +426,71 @@ class Importer:
#rulog.close() #rulog.close()
# This is now an internal function that should not be called directly. # This is now an internal function that should not be called directly.
def import_file_dict(self, db, file, site, filter, q=None): def import_file_dict(self, file, site, filter, q=None):
#print "import_file_dict"
if os.path.isdir(file): if os.path.isdir(file):
self.addToDirList[file] = [site] + [filter] self.addToDirList[file] = [site] + [filter]
return (0,0,0,0,0) return (0,0,0,0,0)
conv = None
(stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, time()) (stored, duplicates, partial, errors, ttime) = (0, 0, 0, 0, time())
# sc: is there any need to decode this? maybe easier to skip it than guess at the encoding?
#file = file.decode("utf-8") #(Configuration.LOCALE_ENCODING)
# Load filter, process file, pass returned filename to import_fpdb_file # Load filter, process file, pass returned filename to import_fpdb_file
if self.settings['threads'] > 0 and self.writeq is not None: if self.settings['threads'] > 0 and self.writeq is not None:
log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")") log.info((_("Converting %s") % file) + " (" + str(q.qsize()) + ")")
else: else: log.info(_("Converting %s") % file)
log.info(_("Converting %s") % file)
filter_name = filter.replace("ToFpdb", "") filter_name = filter.replace("ToFpdb", "")
mod = __import__(filter) mod = __import__(filter)
obj = getattr(mod, filter_name, None) obj = getattr(mod, filter_name, None)
if callable(obj): if callable(obj):
idx = 0
if file in self.pos_in_file: if file in self.pos_in_file: idx = self.pos_in_file[file]
idx = self.pos_in_file[file] else: self.pos_in_file[file], idx = 0, 0
else:
self.pos_in_file[file] = 0 hhc = obj( self.config, in_path = file, index = idx
hhc = obj( self.config, in_path = file, index = idx, starsArchive = self.settings['starsArchive'], ftpArchive = self.settings['ftpArchive'], sitename = site ) ,starsArchive = self.settings['starsArchive']
,ftpArchive = self.settings['ftpArchive']
,sitename = site )
if hhc.getStatus(): if hhc.getStatus():
if self.caller: hhc.progressNotify()
handlist = hhc.getProcessedHands() handlist = hhc.getProcessedHands()
self.pos_in_file[file] = hhc.getLastCharacterRead() self.pos_in_file[file] = hhc.getLastCharacterRead()
to_hud = [] (hbulk, hpbulk, habulk, hcbulk, phands, ihands) = ([], [], [], [], [], [])
hp_bulk = [] sc, gsc = {'bk': []}, {'bk': []}
ha_bulk = []
i = 0
####Lock Placeholder####
for hand in handlist: for hand in handlist:
i += 1 hand.prepInsert(self.database)
if hand is not None: self.database.commit()
hand.prepInsert(self.database, printtest = self.settings['testData']) phands.append(hand)
####Lock Placeholder####
for hand in phands:
hand.assembleHand()
####Lock Placeholder####
id = self.database.nextHandId()
for i in range(len(phands)):
doinsert = len(phands)==i+1
hand = phands[i]
try: try:
hp_inserts, ha_inserts = hand.insert(self.database, hp_data = hp_bulk, id = hand.getHandId(self.database, id)
ha_data = ha_bulk, insert_data = len(handlist)==i, sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert)
printtest = self.settings['testData']) hbulk = hand.insertHands(self.database, hbulk, doinsert, self.settings['testData'])
hp_bulk += hp_inserts hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert)
ha_bulk += ha_inserts ihands.append(hand)
to_hud.append(id)
except Exceptions.FpdbHandDuplicate: except Exceptions.FpdbHandDuplicate:
duplicates += 1 duplicates += 1
else: self.database.commit()
if self.callHud and hand.dbid_hands != 0: ####Lock Placeholder####
to_hud.append(hand.dbid_hands)
else: # TODO: Treat empty as an error, or just ignore?
log.error(_("Hand processed but empty"))
# Call hudcache update if not in bulk import mode for i in range(len(ihands)):
# FIXME: Need to test for bulk import that isn't rebuilding the cache doinsert = len(ihands)==i+1
if self.callHud: hand = ihands[i]
for hand in handlist: hpbulk = hand.insertHandsPlayers(self.database, hpbulk, doinsert, self.settings['testData'])
if hand is not None and not hand.is_duplicate: habulk = hand.insertHandsActions(self.database, habulk, doinsert, self.settings['testData'])
hand.updateHudCache(self.database)
self.database.commit() self.database.commit()
#pipe the Hands.id out to the HUD #pipe the Hands.id out to the HUD