Retab/whitespace demangly fpdb_import.py
This commit is contained in:
Mika Bostrom
parent
19c0328955
commit
415eb4a3c6
|
|
@ -20,16 +20,16 @@
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import MySQLdb
|
import MySQLdb
|
||||||
mysqlLibFound=True
|
mysqlLibFound=True
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import psycopg2
|
import psycopg2
|
||||||
pgsqlLibFound=True
|
pgsqlLibFound=True
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
import traceback
|
import traceback
|
||||||
import math
|
import math
|
||||||
|
|
@ -42,274 +42,276 @@ from time import time
|
||||||
|
|
||||||
class Importer:
|
class Importer:
|
||||||
|
|
||||||
def __init__(self, caller, settings, config):
|
def __init__(self, caller, settings, config):
|
||||||
"""Constructor"""
|
"""Constructor"""
|
||||||
self.settings=settings
|
self.settings=settings
|
||||||
self.caller=caller
|
self.caller=caller
|
||||||
self.config = config
|
self.config = config
|
||||||
self.db = None
|
self.db = None
|
||||||
self.cursor = None
|
self.cursor = None
|
||||||
self.filelist = {}
|
self.filelist = {}
|
||||||
self.dirlist = {}
|
self.dirlist = {}
|
||||||
self.monitor = False
|
self.monitor = False
|
||||||
self.updated = {} #Time last import was run {file:mtime}
|
self.updated = {} #Time last import was run {file:mtime}
|
||||||
self.lines = None
|
self.lines = None
|
||||||
self.faobs = None #File as one big string
|
self.faobs = None #File as one big string
|
||||||
self.pos_in_file = {} # dict to remember how far we have read in the file
|
self.pos_in_file = {} # dict to remember how far we have read in the file
|
||||||
#Set defaults
|
#Set defaults
|
||||||
self.callHud = self.config.get_import_parameters().get("callFpdbHud")
|
self.callHud = self.config.get_import_parameters().get("callFpdbHud")
|
||||||
if not self.settings.has_key('minPrint'):
|
if not self.settings.has_key('minPrint'):
|
||||||
self.settings['minPrint'] = 30
|
self.settings['minPrint'] = 30
|
||||||
self.dbConnect()
|
self.dbConnect()
|
||||||
|
|
||||||
def dbConnect(self):
|
# XXX: Why is this here, when fpdb_db.connect() already does the
|
||||||
#connect to DB
|
# same?
|
||||||
if self.settings['db-backend'] == 2:
|
def dbConnect(self):
|
||||||
if not mysqlLibFound:
|
#connect to DB
|
||||||
raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file")
|
if self.settings['db-backend'] == 2:
|
||||||
self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'],
|
if not mysqlLibFound:
|
||||||
self.settings['db-password'], self.settings['db-databaseName'])
|
raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file")
|
||||||
elif self.settings['db-backend'] == 3:
|
self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'],
|
||||||
if not pgsqlLibFound:
|
self.settings['db-password'], self.settings['db-databaseName'])
|
||||||
raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file")
|
elif self.settings['db-backend'] == 3:
|
||||||
print self.settings
|
if not pgsqlLibFound:
|
||||||
self.db = psycopg2.connect(host = self.settings['db-host'],
|
raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file")
|
||||||
user = self.settings['db-user'],
|
print self.settings
|
||||||
password = self.settings['db-password'],
|
self.db = psycopg2.connect(host = self.settings['db-host'],
|
||||||
database = self.settings['db-databaseName'])
|
user = self.settings['db-user'],
|
||||||
elif self.settings['db-backend'] == 4:
|
password = self.settings['db-password'],
|
||||||
pass
|
database = self.settings['db-databaseName'])
|
||||||
else:
|
elif self.settings['db-backend'] == 4:
|
||||||
pass
|
pass
|
||||||
self.cursor = self.db.cursor()
|
else:
|
||||||
|
pass
|
||||||
|
self.cursor = self.db.cursor()
|
||||||
|
|
||||||
#Set functions
|
#Set functions
|
||||||
def setCallHud(self, value):
|
def setCallHud(self, value):
|
||||||
self.callHud = value
|
self.callHud = value
|
||||||
|
|
||||||
def setMinPrint(self, value):
|
def setMinPrint(self, value):
|
||||||
self.settings['minPrint'] = int(value)
|
self.settings['minPrint'] = int(value)
|
||||||
|
|
||||||
def setHandCount(self, value):
|
def setHandCount(self, value):
|
||||||
self.settings['handCount'] = int(value)
|
self.settings['handCount'] = int(value)
|
||||||
|
|
||||||
def setQuiet(self, value):
|
def setQuiet(self, value):
|
||||||
self.settings['quiet'] = value
|
self.settings['quiet'] = value
|
||||||
|
|
||||||
def setFailOnError(self, value):
|
def setFailOnError(self, value):
|
||||||
self.settings['failOnError'] = value
|
self.settings['failOnError'] = value
|
||||||
|
|
||||||
# def setWatchTime(self):
|
# def setWatchTime(self):
|
||||||
# self.updated = time()
|
# self.updated = time()
|
||||||
|
|
||||||
def clearFileList(self):
|
def clearFileList(self):
|
||||||
self.filelist = {}
|
self.filelist = {}
|
||||||
|
|
||||||
#Add an individual file to filelist
|
#Add an individual file to filelist
|
||||||
def addImportFile(self, filename, site = "default", filter = "passthrough"):
|
def addImportFile(self, filename, site = "default", filter = "passthrough"):
|
||||||
#TODO: test it is a valid file
|
#TODO: test it is a valid file
|
||||||
self.filelist[filename] = [site] + [filter]
|
self.filelist[filename] = [site] + [filter]
|
||||||
|
|
||||||
#Add a directory of files to filelist
|
#Add a directory of files to filelist
|
||||||
#Only one import directory per site supported.
|
#Only one import directory per site supported.
|
||||||
#dirlist is a hash of lists:
|
#dirlist is a hash of lists:
|
||||||
#dirlist{ 'PokerStars' => ["/path/to/import/", "filtername"] }
|
#dirlist{ 'PokerStars' => ["/path/to/import/", "filtername"] }
|
||||||
def addImportDirectory(self,dir,monitor = False, site = "default", filter = "passthrough"):
|
def addImportDirectory(self,dir,monitor = False, site = "default", filter = "passthrough"):
|
||||||
if os.path.isdir(dir):
|
if os.path.isdir(dir):
|
||||||
if monitor == True:
|
if monitor == True:
|
||||||
self.monitor = True
|
self.monitor = True
|
||||||
self.dirlist[site] = [dir] + [filter]
|
self.dirlist[site] = [dir] + [filter]
|
||||||
|
|
||||||
for file in os.listdir(dir):
|
for file in os.listdir(dir):
|
||||||
self.addImportFile(os.path.join(dir, file), site, filter)
|
self.addImportFile(os.path.join(dir, file), site, filter)
|
||||||
else:
|
else:
|
||||||
print "Warning: Attempted to add: '" + str(dir) + "' as an import directory"
|
print "Warning: Attempted to add: '" + str(dir) + "' as an import directory"
|
||||||
|
|
||||||
#Run full import on filelist
|
#Run full import on filelist
|
||||||
def runImport(self):
|
def runImport(self):
|
||||||
for file in self.filelist:
|
for file in self.filelist:
|
||||||
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
||||||
|
|
||||||
#Run import on updated files, then store latest update time.
|
#Run import on updated files, then store latest update time.
|
||||||
def runUpdated(self):
|
def runUpdated(self):
|
||||||
#Check for new files in directory
|
#Check for new files in directory
|
||||||
#todo: make efficient - always checks for new file, should be able to use mtime of directory
|
#todo: make efficient - always checks for new file, should be able to use mtime of directory
|
||||||
# ^^ May not work on windows
|
# ^^ May not work on windows
|
||||||
for site in self.dirlist:
|
for site in self.dirlist:
|
||||||
self.addImportDirectory(self.dirlist[site][0], False, site, self.dirlist[site][1])
|
self.addImportDirectory(self.dirlist[site][0], False, site, self.dirlist[site][1])
|
||||||
|
|
||||||
for file in self.filelist:
|
for file in self.filelist:
|
||||||
stat_info = os.stat(file)
|
stat_info = os.stat(file)
|
||||||
try:
|
try:
|
||||||
lastupdate = self.updated[file]
|
lastupdate = self.updated[file]
|
||||||
if stat_info.st_mtime > lastupdate:
|
if stat_info.st_mtime > lastupdate:
|
||||||
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
||||||
self.updated[file] = time()
|
self.updated[file] = time()
|
||||||
except:
|
except:
|
||||||
self.updated[file] = time()
|
self.updated[file] = time()
|
||||||
# This codepath only runs first time the file is found, if modified in the last
|
# This codepath only runs first time the file is found, if modified in the last
|
||||||
# minute run an immediate import.
|
# minute run an immediate import.
|
||||||
if (time() - stat_info.st_mtime) < 60:
|
if (time() - stat_info.st_mtime) < 60:
|
||||||
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1])
|
||||||
|
|
||||||
# This is now an internal function that should not be called directly.
|
# This is now an internal function that should not be called directly.
|
||||||
def import_file_dict(self, file, site, filter):
|
def import_file_dict(self, file, site, filter):
|
||||||
if(filter == "passthrough"):
|
if(filter == "passthrough"):
|
||||||
self.import_fpdb_file(file, site)
|
self.import_fpdb_file(file, site)
|
||||||
else:
|
else:
|
||||||
#Load filter, and run filtered file though main importer
|
#Load filter, and run filtered file though main importer
|
||||||
self.import_fpdb_file(file, site)
|
self.import_fpdb_file(file, site)
|
||||||
|
|
||||||
|
|
||||||
def import_fpdb_file(self, file, site):
|
def import_fpdb_file(self, file, site):
|
||||||
starttime = time()
|
starttime = time()
|
||||||
last_read_hand=0
|
last_read_hand=0
|
||||||
loc = 0
|
loc = 0
|
||||||
if (file=="stdin"):
|
if (file=="stdin"):
|
||||||
inputFile=sys.stdin
|
inputFile=sys.stdin
|
||||||
else:
|
else:
|
||||||
inputFile=open(file, "rU")
|
inputFile=open(file, "rU")
|
||||||
try: loc = self.pos_in_file[file]
|
try: loc = self.pos_in_file[file]
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
# Read input file into class and close file
|
# Read input file into class and close file
|
||||||
inputFile.seek(loc)
|
inputFile.seek(loc)
|
||||||
self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines())
|
self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines())
|
||||||
self.pos_in_file[file] = inputFile.tell()
|
self.pos_in_file[file] = inputFile.tell()
|
||||||
inputFile.close()
|
inputFile.close()
|
||||||
|
|
||||||
try: # sometimes we seem to be getting an empty self.lines, in which case, we just want to return.
|
try: # sometimes we seem to be getting an empty self.lines, in which case, we just want to return.
|
||||||
firstline = self.lines[0]
|
firstline = self.lines[0]
|
||||||
except:
|
except:
|
||||||
# print "import_fpdb_file", file, site, self.lines, "\n"
|
# print "import_fpdb_file", file, site, self.lines, "\n"
|
||||||
return
|
return
|
||||||
|
|
||||||
if firstline.find("Tournament Summary")!=-1:
|
if firstline.find("Tournament Summary")!=-1:
|
||||||
print "TODO: implement importing tournament summaries"
|
print "TODO: implement importing tournament summaries"
|
||||||
#self.faobs = readfile(inputFile)
|
#self.faobs = readfile(inputFile)
|
||||||
#self.parseTourneyHistory()
|
#self.parseTourneyHistory()
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
site=fpdb_simple.recogniseSite(firstline)
|
site=fpdb_simple.recogniseSite(firstline)
|
||||||
category=fpdb_simple.recogniseCategory(firstline)
|
category=fpdb_simple.recogniseCategory(firstline)
|
||||||
|
|
||||||
startpos=0
|
startpos=0
|
||||||
stored=0 #counter
|
stored=0 #counter
|
||||||
duplicates=0 #counter
|
duplicates=0 #counter
|
||||||
partial=0 #counter
|
partial=0 #counter
|
||||||
errors=0 #counter
|
errors=0 #counter
|
||||||
|
|
||||||
for i in range (len(self.lines)): #main loop, iterates through the lines of a file and calls the appropriate parser method
|
for i in range (len(self.lines)): #main loop, iterates through the lines of a file and calls the appropriate parser method
|
||||||
if (len(self.lines[i])<2):
|
if (len(self.lines[i])<2):
|
||||||
endpos=i
|
endpos=i
|
||||||
hand=self.lines[startpos:endpos]
|
hand=self.lines[startpos:endpos]
|
||||||
|
|
||||||
if (len(hand[0])<2):
|
if (len(hand[0])<2):
|
||||||
hand=hand[1:]
|
hand=hand[1:]
|
||||||
|
|
||||||
cancelled=False
|
cancelled=False
|
||||||
damaged=False
|
damaged=False
|
||||||
if (site=="ftp"):
|
if (site=="ftp"):
|
||||||
for i in range (len(hand)):
|
for i in range (len(hand)):
|
||||||
if (hand[i].endswith(" has been canceled")): #this is their typo. this is a typo, right?
|
if (hand[i].endswith(" has been canceled")): #this is their typo. this is a typo, right?
|
||||||
cancelled=True
|
cancelled=True
|
||||||
|
|
||||||
seat1=hand[i].find("Seat ") #todo: make this recover by skipping this line
|
seat1=hand[i].find("Seat ") #todo: make this recover by skipping this line
|
||||||
if (seat1!=-1):
|
if (seat1!=-1):
|
||||||
if (hand[i].find("Seat ", seat1+3)!=-1):
|
if (hand[i].find("Seat ", seat1+3)!=-1):
|
||||||
damaged=True
|
damaged=True
|
||||||
|
|
||||||
if (len(hand)<3):
|
if (len(hand)<3):
|
||||||
pass
|
pass
|
||||||
#todo: the above 2 lines are kind of a dirty hack, the mentioned circumstances should be handled elsewhere but that doesnt work with DOS/Win EOL. actually this doesnt work.
|
#todo: the above 2 lines are kind of a dirty hack, the mentioned circumstances should be handled elsewhere but that doesnt work with DOS/Win EOL. actually this doesnt work.
|
||||||
elif (hand[0].endswith(" (partial)")): #partial hand - do nothing
|
elif (hand[0].endswith(" (partial)")): #partial hand - do nothing
|
||||||
partial+=1
|
partial+=1
|
||||||
elif (hand[1].find("Seat")==-1 and hand[2].find("Seat")==-1 and hand[3].find("Seat")==-1):#todo: should this be or instead of and?
|
elif (hand[1].find("Seat")==-1 and hand[2].find("Seat")==-1 and hand[3].find("Seat")==-1):#todo: should this be or instead of and?
|
||||||
partial+=1
|
partial+=1
|
||||||
elif (cancelled or damaged):
|
elif (cancelled or damaged):
|
||||||
partial+=1
|
partial+=1
|
||||||
else: #normal processing
|
else: #normal processing
|
||||||
isTourney=fpdb_simple.isTourney(hand[0])
|
isTourney=fpdb_simple.isTourney(hand[0])
|
||||||
if not isTourney:
|
if not isTourney:
|
||||||
fpdb_simple.filterAnteBlindFold(site,hand)
|
fpdb_simple.filterAnteBlindFold(site,hand)
|
||||||
hand=fpdb_simple.filterCrap(site, hand, isTourney)
|
hand=fpdb_simple.filterCrap(site, hand, isTourney)
|
||||||
self.hand=hand
|
self.hand=hand
|
||||||
|
|
||||||
try:
|
try:
|
||||||
handsId=fpdb_parse_logic.mainParser(self.db, self.cursor, site, category, hand)
|
handsId=fpdb_parse_logic.mainParser(self.db, self.cursor, site, category, hand)
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
|
|
||||||
stored+=1
|
stored+=1
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
if self.callHud:
|
if self.callHud:
|
||||||
#print "call to HUD here. handsId:",handsId
|
#print "call to HUD here. handsId:",handsId
|
||||||
#pipe the Hands.id out to the HUD
|
#pipe the Hands.id out to the HUD
|
||||||
self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
|
self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
|
||||||
except fpdb_simple.DuplicateError:
|
except fpdb_simple.DuplicateError:
|
||||||
duplicates+=1
|
duplicates+=1
|
||||||
except (ValueError), fe:
|
except (ValueError), fe:
|
||||||
errors+=1
|
errors+=1
|
||||||
self.printEmailErrorMessage(errors, file, hand)
|
self.printEmailErrorMessage(errors, file, hand)
|
||||||
|
|
||||||
if (self.settings['failOnError']):
|
if (self.settings['failOnError']):
|
||||||
self.db.commit() #dont remove this, in case hand processing was cancelled.
|
self.db.commit() #dont remove this, in case hand processing was cancelled.
|
||||||
raise
|
raise
|
||||||
except (fpdb_simple.FpdbError), fe:
|
except (fpdb_simple.FpdbError), fe:
|
||||||
errors+=1
|
errors+=1
|
||||||
self.printEmailErrorMessage(errors, file, hand)
|
self.printEmailErrorMessage(errors, file, hand)
|
||||||
|
|
||||||
#fe.printStackTrace() #todo: get stacktrace
|
#fe.printStackTrace() #todo: get stacktrace
|
||||||
self.db.rollback()
|
self.db.rollback()
|
||||||
|
|
||||||
if (self.settings['failOnError']):
|
if (self.settings['failOnError']):
|
||||||
self.db.commit() #dont remove this, in case hand processing was cancelled.
|
self.db.commit() #dont remove this, in case hand processing was cancelled.
|
||||||
raise
|
raise
|
||||||
if (self.settings['minPrint']!=0):
|
if (self.settings['minPrint']!=0):
|
||||||
if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0):
|
if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0):
|
||||||
print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors
|
print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors
|
||||||
|
|
||||||
if (self.settings['handCount']!=0):
|
if (self.settings['handCount']!=0):
|
||||||
if ((stored+duplicates+partial+errors)>=self.settings['handCount']):
|
if ((stored+duplicates+partial+errors)>=self.settings['handCount']):
|
||||||
if (not self.settings['quiet']):
|
if (not self.settings['quiet']):
|
||||||
print "quitting due to reaching the amount of hands to be imported"
|
print "quitting due to reaching the amount of hands to be imported"
|
||||||
print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime)
|
print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
startpos=endpos
|
startpos=endpos
|
||||||
print "Total stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors, " time:", (time() - starttime)
|
print "Total stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors, " time:", (time() - starttime)
|
||||||
|
|
||||||
if stored==0:
|
if stored==0:
|
||||||
if duplicates>0:
|
if duplicates>0:
|
||||||
for line_no in range(len(self.lines)):
|
for line_no in range(len(self.lines)):
|
||||||
if self.lines[line_no].find("Game #")!=-1:
|
if self.lines[line_no].find("Game #")!=-1:
|
||||||
final_game_line=self.lines[line_no]
|
final_game_line=self.lines[line_no]
|
||||||
handsId=fpdb_simple.parseSiteHandNo(final_game_line)
|
handsId=fpdb_simple.parseSiteHandNo(final_game_line)
|
||||||
else:
|
else:
|
||||||
print "failed to read a single hand from file:", inputFile
|
print "failed to read a single hand from file:", inputFile
|
||||||
handsId=0
|
handsId=0
|
||||||
#todo: this will cause return of an unstored hand number if the last hand was error or partial
|
#todo: this will cause return of an unstored hand number if the last hand was error or partial
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
self.handsId=handsId
|
self.handsId=handsId
|
||||||
return handsId
|
return handsId
|
||||||
#end def import_file_dict
|
#end def import_file_dict
|
||||||
|
|
||||||
def parseTourneyHistory(self):
|
def parseTourneyHistory(self):
|
||||||
print "Tourney history parser stub"
|
print "Tourney history parser stub"
|
||||||
#Find tournament boundaries.
|
#Find tournament boundaries.
|
||||||
#print self.foabs
|
#print self.foabs
|
||||||
|
|
||||||
|
|
||||||
def printEmailErrorMessage(self, errors, filename, line):
|
def printEmailErrorMessage(self, errors, filename, line):
|
||||||
traceback.print_exc(file=sys.stderr)
|
traceback.print_exc(file=sys.stderr)
|
||||||
print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it."
|
print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it."
|
||||||
print "Filename:", filename
|
print "Filename:", filename
|
||||||
print "Here is the first line so you can identify it. Please mention that the error was a ValueError:"
|
print "Here is the first line so you can identify it. Please mention that the error was a ValueError:"
|
||||||
print self.hand[0]
|
print self.hand[0]
|
||||||
print "Hand logged to hand-errors.txt"
|
print "Hand logged to hand-errors.txt"
|
||||||
logfile = open('hand-errors.txt', 'a')
|
logfile = open('hand-errors.txt', 'a')
|
||||||
for s in self.hand:
|
for s in self.hand:
|
||||||
logfile.write(str(s) + "\n")
|
logfile.write(str(s) + "\n")
|
||||||
logfile.write("\n")
|
logfile.write("\n")
|
||||||
logfile.close()
|
logfile.close()
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
print "CLI for fpdb_import is now available as CliFpdb.py"
|
print "CLI for fpdb_import is now available as CliFpdb.py"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user