diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 4131d4ff..0bf5bd5b 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -73,7 +73,7 @@ except ImportError: use_numpy = False -DB_VERSION = 152 +DB_VERSION = 153 # Variance created as sqlite has a bunch of undefined aggregate functions. @@ -124,6 +124,7 @@ class Database: , [ # indexes for postgres (list index 3) {'tab':'Gametypes', 'col':'siteId', 'drop':0} , {'tab':'Hands', 'col':'gametypeId', 'drop':0} # mct 22/3/09 + , {'tab':'Hands', 'col':'fileId', 'drop':0} # mct 22/3/09 #, {'tab':'Hands', 'col':'siteHandNo', 'drop':0} unique indexes not dropped , {'tab':'HandsActions', 'col':'handId', 'drop':1} , {'tab':'HandsActions', 'col':'playerId', 'drop':1} @@ -151,6 +152,7 @@ class Database: ] , [ # indexes for sqlite (list index 4) {'tab':'Hands', 'col':'gametypeId', 'drop':0} + , {'tab':'Hands', 'col':'fileId', 'drop':0} , {'tab':'HandsPlayers', 'col':'handId', 'drop':0} , {'tab':'HandsPlayers', 'col':'playerId', 'drop':0} , {'tab':'HandsPlayers', 'col':'tourneysPlayersId', 'drop':0} @@ -179,6 +181,7 @@ class Database: , [ ] # no db with index 1 , [ # foreign keys for mysql (index 2) {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'Hands', 'fkcol':'fileId', 'rtab':'Files', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1} @@ -194,6 +197,7 @@ class Database: ] , [ # foreign keys for postgres (index 3) {'fktab':'Hands', 'fkcol':'gametypeId', 'rtab':'Gametypes', 'rcol':'id', 'drop':1} + , {'fktab':'Hands', 'fkcol':'fileId', 'rtab':'Files', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} , {'fktab':'HandsPlayers', 'fkcol':'playerId', 'rtab':'Players', 'rcol':'id', 'drop':1} , {'fktab':'HandsActions', 'fkcol':'handId', 'rtab':'Hands', 'rcol':'id', 'drop':1} @@ -332,7 +336,7 @@ class Database: tables=self.cursor.execute(self.sql.query['list_tables']) tables=self.cursor.fetchall() - for table in (u'Actions', u'Autorates', u'Backings', u'Gametypes', u'Hands', u'HandsActions', u'HandsPlayers', u'HudCache', u'SessionsCache', u'Players', u'RawHands', u'RawTourneys', u'Settings', u'Sites', u'TourneyTypes', u'Tourneys', u'TourneysPlayers'): + for table in (u'Actions', u'Autorates', u'Backings', u'Gametypes', u'Hands', u'HandsActions', u'HandsPlayers', u'Files', u'HudCache', u'SessionsCache', u'Players', u'RawHands', u'RawTourneys', u'Settings', u'Sites', u'TourneyTypes', u'Tourneys', u'TourneysPlayers'): print "table:", table result+="###################\nTable "+table+"\n###################\n" rows=self.cursor.execute(self.sql.query['get'+table]) @@ -1257,6 +1261,7 @@ class Database: c.execute(self.sql.query['createActionsTable']) c.execute(self.sql.query['createSitesTable']) c.execute(self.sql.query['createGametypesTable']) + c.execute(self.sql.query['createFilesTable']) c.execute(self.sql.query['createPlayersTable']) c.execute(self.sql.query['createAutoratesTable']) c.execute(self.sql.query['createHandsTable']) @@ -1857,7 +1862,8 @@ class Database: hdata['tourneyId'], hdata['gametypeId'], hdata['sessionId'], - hdata['gameSessionId'], + hdata['gameSessionId'], + hdata['fileId'], hdata['startTime'], datetime.utcnow(), #importtime hdata['seats'], @@ -2500,7 +2506,53 @@ class Database: self.commit() return gsc + + def getGameTypeId(self, siteid, game, printdata = False): + c = self.get_cursor() + #FIXME: Fixed for NL at the moment + c.execute(self.sql.query['getGametypeNL'], (siteid, game['type'], game['category'], game['limitType'], game['currency'], + game['mix'], int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100))) + tmp = c.fetchone() + if (tmp == None): + hilo = "h" + if game['category'] in ['studhilo', 'omahahilo']: + hilo = "s" + elif game['category'] in ['razz','27_3draw','badugi', '27_1draw']: + hilo = "l" + #FIXME: recognise currency + #TODO: this wont work for non-standard structures + tmp = self.insertGameTypes( (siteid, game['currency'], game['type'], game['base'], game['category'], game['limitType'], hilo, + game['mix'], int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100), + int(Decimal(game['bb'])*100), int(Decimal(game['bb'])*200)), printdata = printdata) + return tmp[0] + + + def insertGameTypes(self, row, printdata = False): + if printdata: + print _("######## Gametype ##########") + import pprint + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(row) + print _("###### End Gametype ########") + + c = self.get_cursor() + c.execute( self.sql.query['insertGameTypes'], row ) + return [self.get_last_insert_id(c)] + def storeFile(self, fdata): + q = self.sql.query['store_file'] + q = q.replace('%s', self.sql.query['placeholder']) + c = self.get_cursor() + c.execute(q, fdata) + id = self.get_last_insert_id(c) + return id + + def updateFile(self, fdata): + q = self.sql.query['update_file'] + q = q.replace('%s', self.sql.query['placeholder']) + c = self.get_cursor() + c.execute(q, fdata) + def getHeroIds(self, pids, sitename): #Grab playerIds using hero names in HUD_Config.xml try: @@ -2550,40 +2602,6 @@ class Database: dup = True return dup - def getGameTypeId(self, siteid, game, printdata = False): - c = self.get_cursor() - #FIXME: Fixed for NL at the moment - c.execute(self.sql.query['getGametypeNL'], (siteid, game['type'], game['category'], game['limitType'], game['currency'], - int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100))) - tmp = c.fetchone() - if (tmp == None): - hilo = "h" - if game['category'] in ['studhilo', 'omahahilo']: - hilo = "s" - elif game['category'] in ['razz','27_3draw','badugi', '27_1draw']: - hilo = "l" - #FIXME: recognise currency - #TODO: this wont work for non-standard structures - tmp = self.insertGameTypes( (siteid, game['currency'], game['type'], game['base'], game['category'], game['limitType'], hilo, - int(Decimal(game['sb'])*100), int(Decimal(game['bb'])*100), - int(Decimal(game['bb'])*100), int(Decimal(game['bb'])*200)), printdata = printdata) - return tmp[0] - - - def insertGameTypes(self, row, printdata = False): - if printdata: - print _("######## Gametype ##########") - import pprint - pp = pprint.PrettyPrinter(indent=4) - pp.pprint(row) - print _("###### End Gametype ########") - - c = self.get_cursor() - c.execute( self.sql.query['insertGameTypes'], row ) - return [self.get_last_insert_id(c)] - - - ################################# # Finish of NEWIMPORT CODE ################################# diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 57a2984c..4e956650 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -145,17 +145,11 @@ class Fulltilt(HandHistoryConverter): ##Total Prize Pool: 1,500 Play Chips # These regexes are for FTP only - re_Mixed = re.compile(r'\s\-\s(?PHA|HORSE|HOSE)\s\-\s', re.VERBOSE) + re_Mixed = re.compile(r'\s\-\s(?P7\-Game|8\-Game|9\-Game|10\-Game|HA|HEROS|HO|HOE|HORSE|HOSE|OA|OE|SE)\s\-\s', re.VERBOSE) re_Max = re.compile("(?P\d+)( max)?", re.MULTILINE) # NB: if we ever match "Full Tilt Poker" we should also match "FullTiltPoker", which PT Stud erroneously exports. re_DateTime = re.compile("""((?P[0-9]+):(?P[0-9]+):(?P[0-9]+)\s(?P\w+)\s-\s(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})|(?P

[0-9]+):(?P[0-9]+)\s(?P\w+)\s-\s\w+\,\s(?P\w+)\s(?P\d+)\,\s(?P[0-9]{4}))(?P\s\(partial\))?""", re.MULTILINE) - - - - mixes = { 'HORSE': 'horse', '7-Game': '7game', 'HOSE': 'hose', 'HA': 'ha'} - - def compilePlayerRegexs(self, hand): players = set([player[1] for player in hand.players]) if not players <= self.compiledPlayers: # x <= y means 'x is subset of y' @@ -224,6 +218,21 @@ class Fulltilt(HandHistoryConverter): 'Badugi' : ('draw','badugi'), '2-7 Single Draw' : ('draw','27_1draw') } + mixes = { + '7-Game' : '7game', + '8-Game' : '8game', + '9-Game' : '9game', + '10-Game' : '10game', + 'HA' : 'ha', + 'HEROS' : 'heros', + 'HO' : 'ho', + 'HOE' : 'hoe', + 'HORSE' : 'horse', + 'HOSE' : 'hose', + 'OA' : 'oa', + 'OE' : 'oe', + 'SE' : 'se' + } currencies = { u'€':'EUR', '$':'USD', '':'T$' } if 'SB' in mg: @@ -254,6 +263,9 @@ class Fulltilt(HandHistoryConverter): if mg['CURRENCY'] is not None: info['currency'] = currencies[mg['CURRENCY']] # NB: SB, BB must be interpreted as blinds or bets depending on limit type. + m = self.re_Mixed.search(self.in_path) + if m: info['mix'] = mixes[m.groupdict()['MIXED']] + return info def readHandInfo(self, hand): @@ -527,13 +539,6 @@ class Fulltilt(HandHistoryConverter): if mo <= 6: return 6 return 9 - def readOther(self, hand): - m = self.re_Mixed.search(self.in_path) - if m is None: - hand.mixed = None - else: - hand.mixed = self.mixes[m.groupdict()['MIXED']] - def readSummaryInfo(self, summaryInfoList): self.status = True diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 6dd09ba4..4e5249a3 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -273,12 +273,13 @@ dealt whether they were seen in a 'dealt to' line next = id +1 return next - def insertHands(self, db, hbulk, doinsert = False, printtest = False): + def insertHands(self, db, hbulk, fileId, doinsert = False, printtest = False): """ Function to insert Hand into database Should not commit, and do minimal selects. Callers may want to cache commits db: a connected Database object""" self.hands['gametypeId'] = self.dbid_gt self.hands['seats'] = len(self.dbid_pids) + self.hands['fileId'] = fileId hbulk = db.storeHand(self.hands, hbulk, doinsert, printtest) return hbulk diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index 6d25d6aa..a1fab62b 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -68,7 +68,7 @@ class HandHistoryConverter(): # maybe archive params should be one archive param, then call method in specific converter. if archive: convert_archive() def __init__( self, config, in_path = '-', out_path = '-', follow=False, index=0 - , autostart=True, starsArchive=False, ftpArchive=False, sitename="PokerStars" ): + , autostart=True, starsArchive=False, ftpArchive=False, sitename="PokerStars"): """\ in_path (default '-' = sys.stdin) out_path (default '-' = sys.stdout) @@ -289,6 +289,7 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py. self.numErrors += 1 else: # See if gametype is supported. + if 'mix' not in gametype: gametype['mix'] = 'none' type = gametype['type'] base = gametype['base'] limit = gametype['limitType'] @@ -332,6 +333,7 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py. 'base' in ('hold', 'stud', 'draw') 'category' in ('holdem', 'omahahi', omahahilo', 'razz', 'studhi', 'studhilo', 'fivedraw', '27_1draw', '27_3draw', 'badugi') 'hilo' in ('h','l','s') + 'mix' in (site specific, or 'none') 'smallBlind' int? 'bigBlind' int? 'smallBet' @@ -445,7 +447,8 @@ or None if we fail to get the info """ def readAction(self, hand, street): abstract def readCollectPot(self, hand): abstract def readShownCards(self, hand): abstract - + + # EDIT: readOther is depreciated # Some sites do odd stuff that doesn't fall in to the normal HH parsing. # e.g., FTP doesn't put mixed game info in the HH, but puts in in the # file name. Use readOther() to clean up those messes. diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 103b45b0..aa2348fa 100644 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -37,8 +37,6 @@ class PokerStars(HandHistoryConverter): filetype = "text" codepage = ("utf8", "cp1252") siteId = 2 # Needs to match id entry in Sites database - - mixes = { 'HORSE': 'horse', '8-Game': '8game', 'HOSE': 'hose'} # Legal mixed games sym = {'USD': "\$", 'CAD': "\$", 'T$': "", "EUR": "\xe2\x82\xac", "GBP": "\xa3", "play": ""} # ADD Euro, Sterling, etc HERE substitutions = { 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes @@ -80,6 +78,15 @@ class PokerStars(HandHistoryConverter): 'Single Draw 2-7 Lowball' : ('draw','27_1draw'), '5 Card Draw' : ('draw','fivedraw') } + mixes = { + 'HORSE': 'horse', + '8-Game': '8game', + 'HOSE': 'hose', + 'Mixed PLH/PLO': 'plh_plo', + 'Mixed Omaha H/L': 'plo_lo', + "Mixed Hold'em": 'lh_nlh', + 'Triple Stud': '3stud' + } # Legal mixed games currencies = { u'€':'EUR', '$':'USD', '':'T$' } # Static regexes @@ -90,7 +97,7 @@ class PokerStars(HandHistoryConverter): # here's how I plan to use LS (?P(?P[%(LS)s\d\.]+)?\+?(?P[%(LS)s\d\.]+)?\+?(?P[%(LS)s\d\.]+)?\s?(?P%(LEGAL_ISO)s)?|Freeroll)\s+)? # close paren of tournament info - (?PHORSE|8\-Game|HOSE|Mixed PLH/PLO)?\s?\(? + (?PHORSE|8\-Game|HOSE|Mixed Omaha H/L|Mixed Hold\'em|Mixed PLH/PLO|Triple Stud)?\s?\(? (?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|Single\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s (?PNo\sLimit|Limit|LIMIT|Pot\sLimit)\)?,?\s (-\s)? @@ -198,7 +205,9 @@ class PokerStars(HandHistoryConverter): info['bb'] = mg['BB'] if 'CURRENCY' in mg: info['currency'] = self.currencies[mg['CURRENCY']] - + if 'MIXED' in mg: + if mg['MIXED'] is not None: info['mix'] = self.mixes[mg['MIXED']] + if 'TOURNO' in mg and mg['TOURNO'] is None: info['type'] = 'ring' else: @@ -299,8 +308,6 @@ class PokerStars(HandHistoryConverter): if key == 'MAX' and info[key] != None: hand.maxseats = int(info[key]) - if key == 'MIXED': - hand.mixed = self.mixes[info[key]] if info[key] is not None else None if key == 'PLAY' and info['PLAY'] is not None: # hand.currency = 'play' # overrides previously set value hand.gametype['currency'] = 'play' diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 06c4830d..cf054d7e 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -245,6 +245,7 @@ class Sql: category varchar(9) NOT NULL, limitType char(2) NOT NULL, hiLo char(1) NOT NULL, + mix varchar(9) NOT NULL, smallBlind int, bigBlind int, smallBet int NOT NULL, @@ -260,6 +261,7 @@ class Sql: category varchar(9), limitType char(2), hiLo char(1), + mix char(9), smallBlind int, bigBlind int, smallBet int, @@ -274,6 +276,7 @@ class Sql: category TEXT, limitType TEXT, hiLo TEXT, + mix TEXT, smallBlind INTEGER, bigBlind INTEGER, smallBet INTEGER, @@ -356,7 +359,8 @@ class Sql: tourneyId INT UNSIGNED, gametypeId SMALLINT UNSIGNED NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), sessionId INT UNSIGNED, - gameSessionId INT UNSIGNED, + gameSessionId INT UNSIGNED, + fileId INT(10) UNSIGNED NOT NULL, FOREIGN KEY (fileId) REFERENCES Files(id), startTime DATETIME NOT NULL, importTime DATETIME NOT NULL, seats TINYINT NOT NULL, @@ -396,6 +400,7 @@ class Sql: gametypeId INT NOT NULL, FOREIGN KEY (gametypeId) REFERENCES Gametypes(id), sessionId INT, gameSessionId INT, + fileId BIGINT NOT NULL, FOREIGN KEY (fileId) REFERENCES Files(id), startTime timestamp without time zone NOT NULL, importTime timestamp without time zone NOT NULL, seats SMALLINT NOT NULL, @@ -434,6 +439,7 @@ class Sql: gametypeId INT NOT NULL, sessionId INT, gameSessionId INT, + fileId INT NOT NULL, startTime REAL NOT NULL, importTime REAL NOT NULL, seats INT NOT NULL, @@ -1135,7 +1141,61 @@ class Sql: cardsDiscarded TEXT, allIn BOOLEAN )""" - + + ################################ + # Create Files + ################################ + + if db_server == 'mysql': + self.query['createFilesTable'] = """CREATE TABLE Files ( + id INT(10) UNSIGNED AUTO_INCREMENT NOT NULL, PRIMARY KEY (id), + file text NOT NULL, + site VARCHAR(32), + type VARCHAR(7), + startTime DATETIME NOT NULL, + lastUpdate DATETIME NOT NULL, + endTime DATETIME, + hands INT, + stored INT, + dups INT, + partial INT, + errs INT, + ttime100 INT, + finished BOOLEAN) + ENGINE=INNODB""" + elif db_server == 'postgresql': + self.query['createFilesTable'] = """CREATE TABLE Files ( + id BIGSERIAL, PRIMARY KEY (id), + file TEXT NOT NULL, + site VARCHAR(32), + type VARCHAR(7),timestamp without time zone + startTime timestamp without time zone NOT NULL, + lastUpdate timestamp without time zone NOT NULL, + endTime timestamp without time zone, + hands INT, + stored INT, + dups INT, + partial INT, + errs INT, + ttime100 INT, + finished BOOLEAN)""" + elif db_server == 'sqlite': + self.query['createFilesTable'] = """CREATE TABLE Files ( + id INTEGER PRIMARY KEY, + file TEXT NOT NULL, + site VARCHAR(32), + type VARCHAR(7), + startTime timestamp NOT NULL, + lastUpdate timestamp NOT NULL, + endTime timestamp, + hands INT, + stored INT, + dups INT, + partial INT, + errs INT, + ttime100 INT, + finished BOOLEAN + )""" ################################ # Create HudCache @@ -1498,10 +1558,10 @@ class Sql: elif db_server == 'postgresql': self.query['createSessionsCacheTable'] = """CREATE TABLE SessionsCache ( id BIGSERIAL, PRIMARY KEY (id), - sessionStart REAL NOT NULL, - sessionEnd REAL NOT NULL, - gameStart REAL NOT NULL, - gameEnd REAL NOT NULL, + sessionStart timestamp without time zone NOT NULL, + sessionEnd timestamp without time zone NOT NULL, + gameStart timestamp without time zone NOT NULL, + gameEnd timestamp without time zone NOT NULL, sessionId INT, date CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ type char(7), @@ -3481,7 +3541,7 @@ class Sql: AND hp.tourneysPlayersId IS NULL - GROUP BY h.startTime, hp.handId, hp.sawShowdown, ( hp.totalProfit / ( gt.bigBlind * 2 ) ) * 100 + GROUP BY h.startTime, hp.handId, hp.sawShowdown, hp.totalProfit ORDER BY h.startTime""" self.query['getRingProfitAllHandsPlayerIdSiteInDollars'] = """ @@ -4645,14 +4705,15 @@ class Sql: AND category=%s AND limitType=%s AND currency=%s + AND mix=%s AND smallBlind=%s AND bigBlind=%s """ self.query['insertGameTypes'] = """INSERT INTO Gametypes (siteId, currency, type, base, category, limitType - ,hiLo, smallBlind, bigBlind, smallBet, bigBet) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" + ,hiLo, mix, smallBlind, bigBlind, smallBet, bigBet) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" self.query['isAlreadyInDB'] = """SELECT id FROM Hands WHERE gametypeId=%s AND siteHandNo=%s @@ -4792,6 +4853,7 @@ class Sql: gametypeid, sessionId, gameSessionId, + fileId, startTime, importtime, seats, @@ -4822,7 +4884,7 @@ class Sql: values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""" self.query['store_hands_players'] = """insert into HandsPlayers ( @@ -4990,6 +5052,42 @@ class Sql: %s, %s, %s, %s, %s, %s, %s )""" + + ################################ + # queries for Files Table + ################################ + + self.query['store_file'] = """ insert into Files ( + file, + site, + startTime, + lastUpdate, + hands, + stored, + dups, + partial, + errs, + ttime100, + finished) + values ( + %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, + %s + )""" + + self.query['update_file'] = """ + UPDATE Files SET + type=%s, + lastUpdate=%s, + endTime=%s, + hands=hands+%s, + stored=stored+%s, + dups=dups+%s, + partial=partial+%s, + errs=errs+%s, + ttime100=ttime100+%s, + finished=%s + WHERE id=%s""" ################################ # Counts for DB stats window diff --git a/pyfpdb/TestHandsPlayers.py b/pyfpdb/TestHandsPlayers.py index 6917b77e..72f99bfe 100755 --- a/pyfpdb/TestHandsPlayers.py +++ b/pyfpdb/TestHandsPlayers.py @@ -160,7 +160,11 @@ def compare_hands_file(filename, importer, errors): pass else: # Stats don't match. - if datum == "gametypeId" or datum == 'sessionId' or datum == 'tourneyId' or datum == 'gameSessionId': + if (datum == "gametypeId" + or datum == 'sessionId' + or datum == 'tourneyId' + or datum == 'gameSessionId' + or datum == 'fileId'): # Not an error. gametypeIds are dependent on the order added to the db. #print "DEBUG: Skipping mismatched gamtypeId" pass diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index a95f8627..845de2e1 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -165,6 +165,18 @@ class Importer: self.database.disconnect() for i in xrange(len(self.writerdbs)): self.writerdbs[i].disconnect() + + def logImport(self, type, file, stored, dups, partial, errs, ttime, id): + hands = stored + dups + partial + errs + now = datetime.utcnow() + ttime100 = ttime * 100 + self.database.updateFile([type, now, now, hands, stored, dups, partial, errs, ttime100, True, id]) + + def addFileToList(self, file, site, filter): + now = datetime.utcnow() + file = os.path.splitext(os.path.basename(file))[0] + id = self.database.storeFile([file, site, now, now, 0, 0, 0, 0, 0, 0, False]) + return [site] + [filter] + [id] #Add an individual file to filelist def addImportFile(self, filename, site = "default", filter = "passthrough"): @@ -173,7 +185,7 @@ class Importer: # filename not guaranteed to be unicode if filename in self.filelist or not os.path.exists(filename): return - self.filelist[filename] = [site] + [filter] + self.filelist[filename] = self.addFileToList(filename, site, filter) if site not in self.siteIds: # Get id from Sites table in DB result = self.database.get_site_id(site) @@ -303,13 +315,15 @@ class Importer: ProgressDialog.progress_update() - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file - ,self.filelist[file][0], self.filelist[file][1], q) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0] + ,self.filelist[file][1], self.filelist[file][2], q) totstored += stored totdups += duplicates totpartial += partial toterrors += errors - + + self.logImport('bulk', file, stored, duplicates, partial, errors, ttime, self.filelist[file][2]) + self.database.commit() del ProgressDialog for i in xrange( self.settings['threads'] ): @@ -394,7 +408,9 @@ class Importer: self.caller.addText("\n"+os.path.basename(file)) except KeyError: # TODO: What error happens here? pass - (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1], None) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(file, self.filelist[file][0] + ,self.filelist[file][1], self.filelist[file][2], None) + self.logImport('auto', file, stored, duplicates, partial, errors, ttime, self.filelist[file][2]) try: if not os.path.isdir(file): # Note: This assumes that whatever calls us has an "addText" func self.caller.addText(" %d stored, %d duplicates, %d partial, %d errors (time = %f)" % (stored, duplicates, partial, errors, ttime)) @@ -425,7 +441,7 @@ class Importer: #rulog.close() # This is now an internal function that should not be called directly. - def import_file_dict(self, file, site, filter, q=None): + def import_file_dict(self, file, site, filter, fileId, q=None): if os.path.isdir(file): self.addToDirList[file] = [site] + [filter] @@ -449,7 +465,7 @@ class Importer: hhc = obj( self.config, in_path = file, index = idx ,starsArchive = self.settings['starsArchive'] ,ftpArchive = self.settings['ftpArchive'] - ,sitename = site ) + ,sitename = site) if hhc.getStatus(): if self.caller: hhc.progressNotify() @@ -476,7 +492,7 @@ class Importer: try: id = hand.getHandId(self.database, id) sc, gsc = hand.updateSessionsCache(self.database, sc, gsc, None, doinsert) - hbulk = hand.insertHands(self.database, hbulk, doinsert, self.settings['testData']) + hbulk = hand.insertHands(self.database, hbulk, fileId, doinsert, self.settings['testData']) hcbulk = hand.updateHudCache(self.database, hcbulk, doinsert) ihands.append(hand) to_hud.append(id)