evilny0/fpdb
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added methods for storing the filename and import statistics in a new 'Files' table which links with the Hands table via Hands.fileId. Extended support for storing mixed game information for Full Tilt Poker coming from the filename, and updated mixed game dictionaries for PokerStars. Added a field in GameTypes to track this mixed game variable entitled 'mix' which takes on the string value of 'none' if not a mix game hand.

Browse Source
This commit is contained in:
Chaz Littlejohn 2011-04-02 23:56:52 +00:00
parent 148777bebd
commit f55cd5569d
8 changed files with 231 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
pyfpdb/Database.py
View File

@ -73,7 +73,7 @@ except ImportError:
use_numpy = False use_numpy = False
DB_VERSION = 152 DB_VERSION = 153
# Variance created as sqlite has a bunch of undefined aggregate functions. # Variance created as sqlite has a bunch of undefined aggregate functions.
@ -124,6 +124,7 @@ class Database:
, [ # indexes for postgres (list index 3) , [ # indexes for postgres (list index 3)
{'tab':'Gametypes', 'col':'siteId', 'drop':0} {'tab':'Gametypes', 'col':'siteId', 'drop':0}
, {'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09 , {'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09
, {'tab':'Hands', 'col':'fileId', 'drop':0} # mct 22/3/09
#, {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped #, {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped
, {'tab':'HandsActions', 'col':'handId', 'drop':1} , {'tab':'HandsActions', 'col':'handId', 'drop':1}
, {'tab':'HandsActions', 'col':'playerId', 'drop':1} , {'tab':'HandsActions', 'col':'playerId', 'drop':1}
@ -151,6 +152,7 @@ class Database:
] ]
, [ # indexes for sqlite (list index 4) , [ # indexes for sqlite (list index 4)
{'tab':'Hands', 'col':'gametypeId', 'drop':0} {'tab':'Hands', 'col':'gametypeId', 'drop':0}
, {'tab':'Hands', 'col':'fileId', 'drop':0}
, {'tab':'HandsPlayers', 'col':'handId', 'drop':0} , {'tab':'HandsPlayers', 'col':'handId', 'drop':0}
, {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'playerId', 'drop':0}
, {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0} , {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0}
@ -179,6 +181,7 @@ class Database:
, [ ] # no db with index 1 , [ ] # no db with index 1
, [ # foreign keys for mysql (index 2) , [ # foreign keys for mysql (index 2)
{'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'Hands', 'fkcol':'fileId', 'rtab':'Files', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1}
@ -194,6 +197,7 @@ class Database:
] ]
, [ # foreign keys for postgres (index 3) , [ # foreign keys for postgres (index 3)
{'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1}
, {'fktab':'Hands', 'fkcol':'fileId', 'rtab':'Files', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1}
, {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1}
, {'fktab':'HandsActions', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsActions', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1}
@ -332,7 +336,7 @@ class Database:
tables=self.cursor.execute(self.sql.query['list_tables']) tables=self.cursor.execute(self.sql.query['list_tables'])
tables=self.cursor.fetchall() tables=self.cursor.fetchall()
for table in (u'Actions', u'Autorates', u'Backings', u'Gametypes', u'Hands', u'HandsActions', u'HandsPlayers', u'HudCache', u'SessionsCache', u'Players', u'RawHands', u'RawTourneys', u'Settings', u'Sites', u'TourneyTypes', u'Tourneys', u'TourneysPlayers'): for table in (u'Actions', u'Autorates', u'Backings', u'Gametypes', u'Hands', u'HandsActions', u'HandsPlayers', u'Files', u'HudCache', u'SessionsCache', u'Players', u'RawHands', u'RawTourneys', u'Settings', u'Sites', u'TourneyTypes', u'Tourneys', u'TourneysPlayers'):
print "table:", table print "table:", table
result+="###################\nTable "+table+"\n###################\n" result+="###################\nTable "+table+"\n###################\n"
rows=self.cursor.execute(self.sql.query['get'+table]) rows=self.cursor.execute(self.sql.query['get'+table])
@ -1257,6 +1261,7 @@ class Database:
c.execute(self.sql.query['createActionsTable']) c.execute(self.sql.query['createActionsTable'])
c.execute(self.sql.query['createSitesTable']) c.execute(self.sql.query['createSitesTable'])
c.execute(self.sql.query['createGametypesTable']) c.execute(self.sql.query['createGametypesTable'])
c.execute(self.sql.query['createFilesTable'])
c.execute(self.sql.query['createPlayersTable']) c.execute(self.sql.query['createPlayersTable'])
c.execute(self.sql.query['createAutoratesTable']) c.execute(self.sql.query['createAutoratesTable'])
c.execute(self.sql.query['createHandsTable']) c.execute(self.sql.query['createHandsTable'])
@ -1858,6 +1863,7 @@ class Database:
hdata['gametypeId'], hdata['gametypeId'],
hdata['sessionId'], hdata['sessionId'],
hdata['gameSessionId'], hdata['gameSessionId'],
hdata['fileId'],
hdata['startTime'], hdata['startTime'],
datetime.utcnow(), #importtime datetime.utcnow(), #importtime
hdata['seats'], hdata['seats'],
@ -2501,6 +2507,52 @@ class Database:
return gsc return gsc
def getGameTypeId(self, siteid, game, printdata = False):
c = self.get_cursor()
#FIXME: Fixed for NL at the moment
c.execute(self.sql.query['getGametypeNL'], (siteid, game['type'], game['category'], game['limitType'], game['currency'],
game['mix'], int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100)))
tmp = c.fetchone()
if (tmp == None):
hilo = "h"
if game['category'] in ['studhilo', 'omahahilo']:
hilo = "s"
elif game['category'] in ['razz','27_3draw','badugi', '27_1draw']:
hilo = "l"
#FIXME: recognise currency
#TODO: this wont work for non-standard structures
tmp = self.insertGameTypes( (siteid, game['currency'], game['type'], game['base'], game['category'], game['limitType'], hilo,
game['mix'], int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100),
int(Decimal(game['bb'])*100), int(Decimal(game['bb'])*200)), printdata = printdata)
return tmp[0]
def insertGameTypes(self, row, printdata = False):
if printdata:
print _("######## Gametype ##########")
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(row)
print _("###### End Gametype ########")
c = self.get_cursor()
c.execute( self.sql.query['insertGameTypes'], row )
return [self.get_last_insert_id(c)]
def storeFile(self, fdata):
q = self.sql.query['store_file']
q = q.replace('%s', self.sql.query['placeholder'])
c = self.get_cursor()
c.execute(q, fdata)
id = self.get_last_insert_id(c)
return id
def updateFile(self, fdata):
q = self.sql.query['update_file']
q = q.replace('%s', self.sql.query['placeholder'])
c = self.get_cursor()
c.execute(q, fdata)
def getHeroIds(self, pids, sitename): def getHeroIds(self, pids, sitename):
#Grab playerIds using hero names in HUD_Config.xml #Grab playerIds using hero names in HUD_Config.xml
try: try:
@ -2550,40 +2602,6 @@ class Database:
dup = True dup = True
return dup return dup
def getGameTypeId(self, siteid, game, printdata = False):
c = self.get_cursor()
#FIXME: Fixed for NL at the moment
c.execute(self.sql.query['getGametypeNL'], (siteid, game['type'], game['category'], game['limitType'], game['currency'],
int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100)))
tmp = c.fetchone()
if (tmp == None):
hilo = "h"
if game['category'] in ['studhilo', 'omahahilo']:
hilo = "s"
elif game['category'] in ['razz','27_3draw','badugi', '27_1draw']:
hilo = "l"
#FIXME: recognise currency
#TODO: this wont work for non-standard structures
tmp = self.insertGameTypes( (siteid, game['currency'], game['type'], game['base'], game['category'], game['limitType'], hilo,
int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100),
int(Decimal(game['bb'])*100), int(Decimal(game['bb'])*200)), printdata = printdata)
return tmp[0]
def insertGameTypes(self, row, printdata = False):
if printdata:
print _("######## Gametype ##########")
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(row)
print _("###### End Gametype ########")
c = self.get_cursor()
c.execute( self.sql.query['insertGameTypes'], row )
return [self.get_last_insert_id(c)]
################################# #################################
# Finish of NEWIMPORT CODE # Finish of NEWIMPORT CODE
################################# #################################

33
pyfpdb/FulltiltToFpdb.py
View File

@ -145,17 +145,11 @@ class Fulltilt(HandHistoryConverter):
##Total Prize Pool: 1,500 Play Chips ##Total Prize Pool: 1,500 Play Chips
# These regexes are for FTP only # These regexes are for FTP only
re_Mixed = re.compile(r'\s\-\s(?P<MIXED>HA|HORSE|HOSE)\s\-\s', re.VERBOSE) re_Mixed = re.compile(r'\s\-\s(?P<MIXED>7\-Game|8\-Game|9\-Game|10\-Game|HA|HEROS|HO|HOE|HORSE|HOSE|OA|OE|SE)\s\-\s', re.VERBOSE)
re_Max = re.compile("(?P<MAX>\d+)( max)?", re.MULTILINE) re_Max = re.compile("(?P<MAX>\d+)( max)?", re.MULTILINE)
# NB: if we ever match "Full Tilt Poker" we should also match "FullTiltPoker", which PT Stud erroneously exports. # NB: if we ever match "Full Tilt Poker" we should also match "FullTiltPoker", which PT Stud erroneously exports.
re_DateTime = re.compile("""((?P<H>[0-9]+):(?P<MIN>[0-9]+):(?P<S>[0-9]+)\s(?P<TZ>\w+)\s-\s(?P<Y>[0-9]{4})\/(?P<M>[0-9]{2})\/(?P<D>[0-9]{2})|(?P<H2>[0-9]+):(?P<MIN2>[0-9]+)\s(?P<TZ2>\w+)\s-\s\w+\,\s(?P<M2>\w+)\s(?P<D2>\d+)\,\s(?P<Y2>[0-9]{4}))(?P<PARTIAL>\s\(partial\))?""", re.MULTILINE) re_DateTime = re.compile("""((?P<H>[0-9]+):(?P<MIN>[0-9]+):(?P<S>[0-9]+)\s(?P<TZ>\w+)\s-\s(?P<Y>[0-9]{4})\/(?P<M>[0-9]{2})\/(?P<D>[0-9]{2})|(?P<H2>[0-9]+):(?P<MIN2>[0-9]+)\s(?P<TZ2>\w+)\s-\s\w+\,\s(?P<M2>\w+)\s(?P<D2>\d+)\,\s(?P<Y2>[0-9]{4}))(?P<PARTIAL>\s\(partial\))?""", re.MULTILINE)
mixes = { 'HORSE': 'horse', '7-Game': '7game', 'HOSE': 'hose', 'HA': 'ha'}
def compilePlayerRegexs(self, hand): def compilePlayerRegexs(self, hand):
players = set([player[1] for player in hand.players]) players = set([player[1] for player in hand.players])
if not players <= self.compiledPlayers: # x <= y means 'x is subset of y' if not players <= self.compiledPlayers: # x <= y means 'x is subset of y'
@ -224,6 +218,21 @@ class Fulltilt(HandHistoryConverter):
'Badugi' : ('draw','badugi'), 'Badugi' : ('draw','badugi'),
'2-7 Single Draw' : ('draw','27_1draw') '2-7 Single Draw' : ('draw','27_1draw')
} }
mixes = {
'7-Game' : '7game',
'8-Game' : '8game',
'9-Game' : '9game',
'10-Game' : '10game',
'HA' : 'ha',
'HEROS' : 'heros',
'HO' : 'ho',
'HOE' : 'hoe',
'HORSE' : 'horse',
'HOSE' : 'hose',
'OA' : 'oa',
'OE' : 'oe',
'SE' : 'se'
}
currencies = { u'€':'EUR', '$':'USD', '':'T$' } currencies = { u'€':'EUR', '$':'USD', '':'T$' }
if 'SB' in mg: if 'SB' in mg:
@ -254,6 +263,9 @@ class Fulltilt(HandHistoryConverter):
if mg['CURRENCY'] is not None: if mg['CURRENCY'] is not None:
info['currency'] = currencies[mg['CURRENCY']] info['currency'] = currencies[mg['CURRENCY']]
# NB: SB, BB must be interpreted as blinds or bets depending on limit type. # NB: SB, BB must be interpreted as blinds or bets depending on limit type.
m = self.re_Mixed.search(self.in_path)
if m: info['mix'] = mixes[m.groupdict()['MIXED']]
return info return info
def readHandInfo(self, hand): def readHandInfo(self, hand):
@ -527,13 +539,6 @@ class Fulltilt(HandHistoryConverter):
if mo <= 6: return 6 if mo <= 6: return 6
return 9 return 9
def readOther(self, hand):
m = self.re_Mixed.search(self.in_path)
if m is None:
hand.mixed = None
else:
hand.mixed = self.mixes[m.groupdict()['MIXED']]
def readSummaryInfo(self, summaryInfoList): def readSummaryInfo(self, summaryInfoList):
self.status = True self.status = True

3
pyfpdb/Hand.py
View File

@ -273,12 +273,13 @@ dealt whether they were seen in a 'dealt to' line
next = id +1 next = id +1
return next return next
def insertHands(self, db, hbulk, doinsert = False, printtest = False): def insertHands(self, db, hbulk, fileId, doinsert = False, printtest = False):
""" Function to insert Hand into database """ Function to insert Hand into database
Should not commit, and do minimal selects. Callers may want to cache commits Should not commit, and do minimal selects. Callers may want to cache commits
db: a connected Database object""" db: a connected Database object"""
self.hands['gametypeId'] = self.dbid_gt self.hands['gametypeId'] = self.dbid_gt
self.hands['seats'] = len(self.dbid_pids) self.hands['seats'] = len(self.dbid_pids)
self.hands['fileId'] = fileId
hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest) hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest)
return hbulk return hbulk

3
pyfpdb/HandHistoryConverter.py
View File

@ -289,6 +289,7 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py.
self.numErrors += 1 self.numErrors += 1
else: else:
# See if gametype is supported. # See if gametype is supported.
if 'mix' not in gametype: gametype['mix'] = 'none'
type = gametype['type'] type = gametype['type']
base = gametype['base'] base = gametype['base']
limit = gametype['limitType'] limit = gametype['limitType']
@ -332,6 +333,7 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py.
'base' in ('hold', 'stud', 'draw') 'base' in ('hold', 'stud', 'draw')
'category' in ('holdem', 'omahahi', omahahilo', 'razz', 'studhi', 'studhilo', 'fivedraw', '27_1draw', '27_3draw', 'badugi') 'category' in ('holdem', 'omahahi', omahahilo', 'razz', 'studhi', 'studhilo', 'fivedraw', '27_1draw', '27_3draw', 'badugi')
'hilo' in ('h','l','s') 'hilo' in ('h','l','s')
'mix' in (site specific, or 'none')
'smallBlind' int? 'smallBlind' int?
'bigBlind' int? 'bigBlind' int?
'smallBet' 'smallBet'
@ -446,6 +448,7 @@ or None if we fail to get the info """
def readCollectPot(self, hand): abstract def readCollectPot(self, hand): abstract
def readShownCards(self, hand): abstract def readShownCards(self, hand): abstract
# EDIT: readOther is depreciated
# Some sites do odd stuff that doesn't fall in to the normal HH parsing. # Some sites do odd stuff that doesn't fall in to the normal HH parsing.
# e.g., FTP doesn't put mixed game info in the HH, but puts in in the # e.g., FTP doesn't put mixed game info in the HH, but puts in in the
# file name. Use readOther() to clean up those messes. # file name. Use readOther() to clean up those messes.

17
pyfpdb/PokerStarsToFpdb.py
View File

@ -37,8 +37,6 @@ class PokerStars(HandHistoryConverter):
filetype = "text" filetype = "text"
codepage = ("utf8", "cp1252") codepage = ("utf8", "cp1252")
siteId = 2 # Needs to match id entry in Sites database siteId = 2 # Needs to match id entry in Sites database
mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games
sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE
substitutions = { substitutions = {
'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes
@ -80,6 +78,15 @@ class PokerStars(HandHistoryConverter):
'Single Draw 2-7 Lowball' : ('draw','27_1draw'), 'Single Draw 2-7 Lowball' : ('draw','27_1draw'),
'5 Card Draw' : ('draw','fivedraw') '5 Card Draw' : ('draw','fivedraw')
} }
mixes = {
'HORSE': 'horse',
'8-Game': '8game',
'HOSE': 'hose',
'Mixed PLH/PLO': 'plh_plo',
'Mixed Omaha H/L': 'plo_lo',
"Mixed Hold'em": 'lh_nlh',
'Triple Stud': '3stud'
} # Legal mixed games
currencies = { u'€':'EUR', '$':'USD', '':'T$' } currencies = { u'€':'EUR', '$':'USD', '':'T$' }
# Static regexes # Static regexes
@ -90,7 +97,7 @@ class PokerStars(HandHistoryConverter):
# here's how I plan to use LS # here's how I plan to use LS
(?P<BUYIN>(?P<BIAMT>[%(LS)s\d\.]+)?\+?(?P<BIRAKE>[%(LS)s\d\.]+)?\+?(?P<BOUNTY>[%(LS)s\d\.]+)?\s?(?P<TOUR_ISO>%(LEGAL_ISO)s)?|Freeroll)\s+)? (?P<BUYIN>(?P<BIAMT>[%(LS)s\d\.]+)?\+?(?P<BIRAKE>[%(LS)s\d\.]+)?\+?(?P<BOUNTY>[%(LS)s\d\.]+)?\s?(?P<TOUR_ISO>%(LEGAL_ISO)s)?|Freeroll)\s+)?
# close paren of tournament info # close paren of tournament info
(?P<MIXED>HORSE|8\-Game|HOSE|Mixed PLH/PLO)?\s?\(? (?P<MIXED>HORSE|8\-Game|HOSE|Mixed Omaha H/L|Mixed Hold\'em|Mixed PLH/PLO|Triple Stud)?\s?\(?
(?P<GAME>Hold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|Single\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s (?P<GAME>Hold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|Single\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s
(?P<LIMIT>No\sLimit|Limit|LIMIT|Pot\sLimit)\)?,?\s (?P<LIMIT>No\sLimit|Limit|LIMIT|Pot\sLimit)\)?,?\s
(-\s)? (-\s)?
@ -198,6 +205,8 @@ class PokerStars(HandHistoryConverter):
info['bb'] = mg['BB'] info['bb'] = mg['BB']
if 'CURRENCY' in mg: if 'CURRENCY' in mg:
info['currency'] = self.currencies[mg['CURRENCY']] info['currency'] = self.currencies[mg['CURRENCY']]
if 'MIXED' in mg:
if mg['MIXED'] is not None: info['mix'] = self.mixes[mg['MIXED']]
if 'TOURNO' in mg and mg['TOURNO'] is None: if 'TOURNO' in mg and mg['TOURNO'] is None:
info['type'] = 'ring' info['type'] = 'ring'
@ -299,8 +308,6 @@ class PokerStars(HandHistoryConverter):
if key == 'MAX' and info[key] != None: if key == 'MAX' and info[key] != None:
hand.maxseats = int(info[key]) hand.maxseats = int(info[key])
if key == 'MIXED':
hand.mixed = self.mixes[info[key]] if info[key] is not None else None
if key == 'PLAY' and info['PLAY'] is not None: if key == 'PLAY' and info['PLAY'] is not None:
# hand.currency = 'play' # overrides previously set value # hand.currency = 'play' # overrides previously set value
hand.gametype['currency'] = 'play' hand.gametype['currency'] = 'play'

114
pyfpdb/SQL.py
View File

@ -245,6 +245,7 @@ class Sql:
category varchar(9) NOT NULL, category varchar(9) NOT NULL,
limitType char(2) NOT NULL, limitType char(2) NOT NULL,
hiLo char(1) NOT NULL, hiLo char(1) NOT NULL,
mix varchar(9) NOT NULL,
smallBlind int, smallBlind int,
bigBlind int, bigBlind int,
smallBet int NOT NULL, smallBet int NOT NULL,
@ -260,6 +261,7 @@ class Sql:
category varchar(9), category varchar(9),
limitType char(2), limitType char(2),
hiLo char(1), hiLo char(1),
mix char(9),
smallBlind int, smallBlind int,
bigBlind int, bigBlind int,
smallBet int, smallBet int,
@ -274,6 +276,7 @@ class Sql:
category TEXT, category TEXT,
limitType TEXT, limitType TEXT,
hiLo TEXT, hiLo TEXT,
mix TEXT,
smallBlind INTEGER, smallBlind INTEGER,
bigBlind INTEGER, bigBlind INTEGER,
smallBet INTEGER, smallBet INTEGER,
@ -357,6 +360,7 @@ class Sql:
gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
sessionId INT UNSIGNED, sessionId INT UNSIGNED,
gameSessionId INT UNSIGNED, gameSessionId INT UNSIGNED,
fileId INT(10) UNSIGNED NOT NULL, FOREIGN KEY (fileId) REFERENCES Files(id),
startTime DATETIME NOT NULL, startTime DATETIME NOT NULL,
importTime DATETIME NOT NULL, importTime DATETIME NOT NULL,
seats TINYINT NOT NULL, seats TINYINT NOT NULL,
@ -396,6 +400,7 @@ class Sql:
gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id),
sessionId INT, sessionId INT,
gameSessionId INT, gameSessionId INT,
fileId BIGINT NOT NULL, FOREIGN KEY (fileId) REFERENCES Files(id),
startTime timestamp without time zone NOT NULL, startTime timestamp without time zone NOT NULL,
importTime timestamp without time zone NOT NULL, importTime timestamp without time zone NOT NULL,
seats SMALLINT NOT NULL, seats SMALLINT NOT NULL,
@ -434,6 +439,7 @@ class Sql:
gametypeId INT NOT NULL, gametypeId INT NOT NULL,
sessionId INT, sessionId INT,
gameSessionId INT, gameSessionId INT,
fileId INT NOT NULL,
startTime REAL NOT NULL, startTime REAL NOT NULL,
importTime REAL NOT NULL, importTime REAL NOT NULL,
seats INT NOT NULL, seats INT NOT NULL,
@ -1136,6 +1142,60 @@ class Sql:
allIn BOOLEAN allIn BOOLEAN
)""" )"""
################################
# Create Files
################################
if db_server == 'mysql':
self.query['createFilesTable'] = """CREATE TABLE Files (
id INT(10) UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id),
file text NOT NULL,
site VARCHAR(32),
type VARCHAR(7),
startTime DATETIME NOT NULL,
lastUpdate DATETIME NOT NULL,
endTime DATETIME,
hands INT,
stored INT,
dups INT,
partial INT,
errs INT,
ttime100 INT,
finished BOOLEAN)
ENGINE=INNODB"""
elif db_server == 'postgresql':
self.query['createFilesTable'] = """CREATE TABLE Files (
id BIGSERIAL, PRIMARY KEY (id),
file TEXT NOT NULL,
site VARCHAR(32),
type VARCHAR(7),timestamp without time zone
startTime timestamp without time zone NOT NULL,
lastUpdate timestamp without time zone NOT NULL,
endTime timestamp without time zone,
hands INT,
stored INT,
dups INT,
partial INT,
errs INT,
ttime100 INT,
finished BOOLEAN)"""
elif db_server == 'sqlite':
self.query['createFilesTable'] = """CREATE TABLE Files (
id INTEGER PRIMARY KEY,
file TEXT NOT NULL,
site VARCHAR(32),
type VARCHAR(7),
startTime timestamp NOT NULL,
lastUpdate timestamp NOT NULL,
endTime timestamp,
hands INT,
stored INT,
dups INT,
partial INT,
errs INT,
ttime100 INT,
finished BOOLEAN
)"""
################################ ################################
# Create HudCache # Create HudCache
@ -1498,10 +1558,10 @@ class Sql:
elif db_server == 'postgresql': elif db_server == 'postgresql':
self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache (
id BIGSERIAL, PRIMARY KEY (id), id BIGSERIAL, PRIMARY KEY (id),
sessionStart REAL NOT NULL, sessionStart timestamp without time zone NOT NULL,
sessionEnd REAL NOT NULL, sessionEnd timestamp without time zone NOT NULL,
gameStart REAL NOT NULL, gameStart timestamp without time zone NOT NULL,
gameEnd REAL NOT NULL, gameEnd timestamp without time zone NOT NULL,
sessionId INT, sessionId INT,
date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */
type char(7), type char(7),
@ -3481,7 +3541,7 @@ class Sql:
<limit_test> <limit_test>
<game_test> <game_test>
AND hp.tourneysPlayersId IS NULL AND hp.tourneysPlayersId IS NULL
GROUP BY h.startTime, hp.handId, hp.sawShowdown, ( hp.totalProfit / ( gt.bigBlind * 2 ) ) * 100 GROUP BY h.startTime, hp.handId, hp.sawShowdown, hp.totalProfit
ORDER BY h.startTime""" ORDER BY h.startTime"""
self.query['getRingProfitAllHandsPlayerIdSiteInDollars'] = """ self.query['getRingProfitAllHandsPlayerIdSiteInDollars'] = """
@ -4645,14 +4705,15 @@ class Sql:
AND category=%s AND category=%s
AND limitType=%s AND limitType=%s
AND currency=%s AND currency=%s
AND mix=%s
AND smallBlind=%s AND smallBlind=%s
AND bigBlind=%s AND bigBlind=%s
""" """
self.query['insertGameTypes'] = """INSERT INTO Gametypes self.query['insertGameTypes'] = """INSERT INTO Gametypes
(siteId, currency, type, base, category, limitType (siteId, currency, type, base, category, limitType
,hiLo, smallBlind, bigBlind, smallBet, bigBet) ,hiLo, mix, smallBlind, bigBlind, smallBet, bigBet)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
self.query['isAlreadyInDB'] = """SELECT id FROM Hands self.query['isAlreadyInDB'] = """SELECT id FROM Hands
WHERE gametypeId=%s AND siteHandNo=%s WHERE gametypeId=%s AND siteHandNo=%s
@ -4792,6 +4853,7 @@ class Sql:
gametypeid, gametypeid,
sessionId, sessionId,
gameSessionId, gameSessionId,
fileId,
startTime, startTime,
importtime, importtime,
seats, seats,
@ -4822,7 +4884,7 @@ class Sql:
values values
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
self.query['store_hands_players'] = """insert into HandsPlayers ( self.query['store_hands_players'] = """insert into HandsPlayers (
@ -4991,6 +5053,42 @@ class Sql:
%s, %s %s, %s
)""" )"""
################################
# queries for Files Table
################################
self.query['store_file'] = """ insert into Files (
file,
site,
startTime,
lastUpdate,
hands,
stored,
dups,
partial,
errs,
ttime100,
finished)
values (
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s
)"""
self.query['update_file'] = """
UPDATE Files SET
type=%s,
lastUpdate=%s,
endTime=%s,
hands=hands+%s,
stored=stored+%s,
dups=dups+%s,
partial=partial+%s,
errs=errs+%s,
ttime100=ttime100+%s,
finished=%s
WHERE id=%s"""
################################ ################################
# Counts for DB stats window # Counts for DB stats window
################################ ################################

6
pyfpdb/TestHandsPlayers.py
View File

@ -160,7 +160,11 @@ def compare_hands_file(filename, importer, errors):
pass pass
else: else:
# Stats don't match. # Stats don't match.
if datum == "gametypeId" or datum == 'sessionId' or datum == 'tourneyId' or datum == 'gameSessionId': if (datum == "gametypeId"
or datum == 'sessionId'
or datum == 'tourneyId'
or datum == 'gameSessionId'
or datum == 'fileId'):
# Not an error. gametypeIds are dependent on the order added to the db. # Not an error. gametypeIds are dependent on the order added to the db.
#print "DEBUG: Skipping mismatched gamtypeId" #print "DEBUG: Skipping mismatched gamtypeId"
pass pass

28
pyfpdb/fpdb_import.py
View File

@ -166,6 +166,18 @@ class Importer:
for i in xrange(len(self.writerdbs)): for i in xrange(len(self.writerdbs)):
self.writerdbs[i].disconnect() self.writerdbs[i].disconnect()
def logImport(self, type, file, stored, dups, partial, errs, ttime, id):
hands = stored + dups + partial + errs
now = datetime.utcnow()
ttime100 = ttime * 100
self.database.updateFile([type, now, now, hands, stored, dups, partial, errs, ttime100, True, id])
def addFileToList(self, file, site, filter):
now = datetime.utcnow()
file = os.path.splitext(os.path.basename(file))[0]
id = self.database.storeFile([file, site, now, now, 0, 0, 0, 0, 0, 0, False])
return [site] + [filter] + [id]
#Add an individual file to filelist #Add an individual file to filelist
def addImportFile(self, filename, site = "default", filter = "passthrough"): def addImportFile(self, filename, site = "default", filter = "passthrough"):
#TODO: test it is a valid file -> put that in config!! #TODO: test it is a valid file -> put that in config!!
@ -173,7 +185,7 @@ class Importer:
# filename not guaranteed to be unicode # filename not guaranteed to be unicode
if filename in self.filelist or not os.path.exists(filename): if filename in self.filelist or not os.path.exists(filename):
return return
self.filelist[filename] = [site] + [filter] self.filelist[filename] = self.addFileToList(filename, site, filter)
if site not in self.siteIds: if site not in self.siteIds:
# Get id from Sites table in DB # Get id from Sites table in DB
result = self.database.get_site_id(site) result = self.database.get_site_id(site)
@ -303,13 +315,15 @@ class Importer:
ProgressDialog.progress_update() ProgressDialog.progress_update()
(stored, duplicates, partial, errors, ttime) = self.import_file_dict(file (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0]
,self.filelist[file][0], self.filelist[file][1], q) ,self.filelist[file][1], self.filelist[file][2], q)
totstored += stored totstored += stored
totdups += duplicates totdups += duplicates
totpartial += partial totpartial += partial
toterrors += errors toterrors += errors
self.logImport('bulk', file, stored, duplicates, partial, errors, ttime, self.filelist[file][2])
self.database.commit()
del ProgressDialog del ProgressDialog
for i in xrange( self.settings['threads'] ): for i in xrange( self.settings['threads'] ):
@ -394,7 +408,9 @@ class Importer:
self.caller.addText("\n"+os.path.basename(file)) self.caller.addText("\n"+os.path.basename(file))
except KeyError: # TODO: What error happens here? except KeyError: # TODO: What error happens here?
pass pass
(stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1], None) (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0]
,self.filelist[file][1], self.filelist[file][2], None)
self.logImport('auto', file, stored, duplicates, partial, errors, ttime, self.filelist[file][2])
try: try:
if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func
self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime)) self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime))
@ -425,7 +441,7 @@ class Importer:
#rulog.close() #rulog.close()
# This is now an internal function that should not be called directly. # This is now an internal function that should not be called directly.
def import_file_dict(self, file, site, filter, q=None): def import_file_dict(self, file, site, filter, fileId, q=None):
if os.path.isdir(file): if os.path.isdir(file):
self.addToDirList[file] = [site] + [filter] self.addToDirList[file] = [site] + [filter]
@ -476,7 +492,7 @@ class Importer:
try: try:
id = hand.getHandId(self.database, id) id = hand.getHandId(self.database, id)
sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert) sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert)
hbulk = hand.insertHands(self.database, hbulk, doinsert, self.settings['testData']) hbulk = hand.insertHands(self.database, hbulk, fileId, doinsert, self.settings['testData'])
hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert)
ihands.append(hand) ihands.append(hand)
to_hud.append(id) to_hud.append(id)