Merge git://git.assembla.com/fpdboz

This commit is contained in:
steffen123 2008-10-13 22:27:32 +01:00
commit 79b8f4616a
6 changed files with 171 additions and 119 deletions

View File

@ -54,5 +54,5 @@ if __name__ == "__main__":
(options, sys.argv) = parser.parse_args()
settings={'imp-callFpdbHud':False, 'db-backend':2}
importer = fpdb_import.Importer()
importer.import_file_dict(options, settings)
importer = fpdb_import.Importer(options,settings)
importer.import_file_dict()

View File

@ -50,16 +50,7 @@ class GuiAutoImport (threading.Thread):
def do_import(self):
"""Callback for timer to do an import iteration."""
for file in os.listdir(self.path):
if os.path.isdir(file):
print "AutoImport is not recursive - please select the final directory in which the history files are"
else:
self.inputFile = os.path.join(self.path, file)
stat_info = os.stat(self.inputFile)
if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]:
self.importer.import_file_dict(self, self.settings)
self.import_files[self.inputFile] = stat_info.st_mtime
self.importer.runUpdated()
print "GuiAutoImport.import_dir done"
return True
@ -96,17 +87,8 @@ class GuiAutoImport (threading.Thread):
# self.pipe_to_hud = os.popen(command, 'w')
self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter())
# Iniitally populate the self.import_files dict, which keeps mtimes for the files watched
self.import_files = {}
for file in os.listdir(self.path):
if os.path.isdir(file):
pass # skip subdirs for now
else:
inputFile = os.path.join(self.path, file)
stat_info = os.stat(inputFile)
self.import_files[inputFile] = stat_info.st_mtime
# Add directory to importer object and set the initial mtime reference.
self.importer.addImportDirectory(self.path, True)
self.do_import()
interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter()))
@ -121,17 +103,18 @@ class GuiAutoImport (threading.Thread):
def __init__(self, settings, debug=True):
"""Constructor for GuiAutoImport"""
self.settings=settings
self.importer = fpdb_import.Importer()
self.importer = fpdb_import.Importer(self,self.settings)
self.importer.setCallHud(True)
self.importer.setMinPrint(30)
self.importer.setQuiet(False)
self.importer.setFailOnError(False)
self.importer.setHandCount(0)
self.importer.setWatchTime()
self.server=settings['db-host']
self.user=settings['db-user']
self.password=settings['db-password']
self.database=settings['db-databaseName']
self.quiet=False
self.failOnError=False
self.minPrint=30
self.handCount=0
self.mainVBox=gtk.VBox(False,1)
self.mainVBox.show()

View File

@ -27,12 +27,8 @@ class GuiBulkImport (threading.Thread):
def import_dir(self):
"""imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it"""
self.path=self.inputFile
for file in os.listdir(self.path):
if os.path.isdir(file):
print "BulkImport is not recursive - please select the final directory in which the history files are"
else:
self.inputFile=self.path+os.sep+file
self.importer.import_file_dict(self, self.settings)
self.importer.addImportDirectory(self.path)
self.importer.runImport()
print "GuiBulkImport.import_dir done"
def load_clicked(self, widget, data=None):
@ -40,36 +36,36 @@ class GuiBulkImport (threading.Thread):
self.handCount=self.hand_count_tbuffer.get_text(self.hand_count_tbuffer.get_start_iter(), self.hand_count_tbuffer.get_end_iter())
if (self.handCount=="unlimited" or self.handCount=="Unlimited"):
self.handCount=0
self.importer.setHandCount(0)
else:
self.handCount=int(self.handCount)
self.importer.setHandCount(int(self.handCount))
self.errorFile="failed.txt"
self.minPrint=self.min_print_tbuffer.get_text(self.min_print_tbuffer.get_start_iter(), self.min_print_tbuffer.get_end_iter())
if (self.minPrint=="never" or self.minPrint=="Never"):
self.minPrint=0
self.importer.setMinPrint(0)
else:
self.minPrint=int(self.minPrint)
self.importer.setMinPrint=int(self.minPrint)
self.quiet=self.info_tbuffer.get_text(self.info_tbuffer.get_start_iter(), self.info_tbuffer.get_end_iter())
if (self.quiet=="yes"):
self.quiet=False
self.importer.setQuiet(False)
else:
self.quiet=True
self.importer.setQuiet(True)
self.failOnError=self.fail_error_tbuffer.get_text(self.fail_error_tbuffer.get_start_iter(), self.fail_error_tbuffer.get_end_iter())
if (self.failOnError=="no"):
self.failOnError=False
self.importer.setFailOnError(False)
else:
self.failOnError=True
self.server, self.database, self.user, self.password=self.db.get_db_info()
self.importer.setFailOnError(True)
if os.path.isdir(self.inputFile):
self.import_dir()
else:
self.importer.import_file_dict(self, self.settings)
self.importer.addImportFile()
self.importer.runImport()
self.importer.clearFileList()
def get_vbox(self):
"""returns the vbox of this thread"""
@ -83,7 +79,7 @@ class GuiBulkImport (threading.Thread):
def __init__(self, db, settings):
self.db=db
self.settings=settings
self.importer = fpdb_import.Importer()
self.importer = fpdb_import.Importer(self,self.settings)
self.vbox=gtk.VBox(False,1)
self.vbox.show()

View File

@ -251,13 +251,13 @@ class GuiTableViewer (threading.Thread):
self.user=self.db.user
self.password=self.db.password
self.quiet=False
self.failOnError=False
self.minPrint=0
self.handCount=0
self.importer = fpdb_import.Importer()
self.importer = fpdb_import.Importer(self, self.settings)
self.importer.setMinPrint(0)
self.importer.setQuiet(False)
self.importer.setFailOnError(False)
self.importer.setHandCount(0)
self.last_read_hand_id=self.importer.import_file_dict(self, self.settings)
self.last_read_hand_id=self.importer.import_file_dict()
#end def table_viewer.import_clicked
def all_clicked(self, widget, data):

View File

@ -25,6 +25,7 @@ import os
import sys
import fpdb_db
import fpdb_import
import FpdbSQLQueries
import unittest
@ -33,20 +34,25 @@ class TestSequenceFunctions(unittest.TestCase):
def setUp(self):
"""Configure MySQL settings/database and establish connection"""
self.mysql_settings={ 'db-host':"localhost", 'db-backend':2, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
self.mysql_settings={ 'db-host':"localhost",
'db-backend':2,
'db-databaseName':"fpdbtest",
'db-user':"fpdb",
'db-password':"fpdb"}
self.mysql_db = fpdb_db.fpdb_db()
self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'],
self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'],
self.mysql_settings['db-password'])
self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB')
self.mysqlimporter = fpdb_import.Importer(self, self.mysql_settings)
"""Configure Postgres settings/database and establish connection"""
self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
self.pg_db = fpdb_db.fpdb_db()
self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'],
self.pg_settings['db-databaseName'], self.pg_settings['db-user'],
self.pg_settings['db-password'])
self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL')
# """Configure Postgres settings/database and establish connection"""
# self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
# self.pg_db = fpdb_db.fpdb_db()
# self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'],
# self.pg_settings['db-databaseName'], self.pg_settings['db-user'],
# self.pg_settings['db-password'])
# self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL')
def testDatabaseConnection(self):
@ -54,10 +60,8 @@ class TestSequenceFunctions(unittest.TestCase):
self.result = self.mysql_db.cursor.execute(self.mysqldict.query['list_tables'])
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
print self.pgdict.query['list_tables']
self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
def testMySQLRecreateTables(self):
"""Test droping then recreating fpdb table schema"""
@ -65,11 +69,18 @@ class TestSequenceFunctions(unittest.TestCase):
self.result = self.mysql_db.cursor.execute("SHOW TABLES")
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
def testPostgresSQLRecreateTables(self):
"""Test droping then recreating fpdb table schema"""
self.pg_db.recreate_tables()
self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
def testImportHandHistoryFiles(self):
"""Test import of single HH file"""
self.mysqlimporter.addImportFile("regression-test-files/hand-histories/ps-lhe-ring-3hands.txt")
self.mysqlimporter.runImport()
self.mysqlimporter.addImportDirectory("regression-test-files/hand-histories")
self.mysqlimporter.runImport()
# def testPostgresSQLRecreateTables(self):
# """Test droping then recreating fpdb table schema"""
# self.pg_db.recreate_tables()
# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
if __name__ == '__main__':
unittest.main()

View File

@ -40,63 +40,132 @@ from time import time
class Importer:
def __init__(self):
def __init__(self, caller, settings):
"""Constructor"""
self.settings={'imp-callFpdbHud':False}
self.settings=settings
self.caller=caller
self.db = None
self.cursor = None
self.options = None
self.filelist = []
self.dirlist = []
self.monitor = False
self.updated = 0 #Time last import was run, used as mtime reference
self.callHud = False
self.lines = None
self.pos_in_file = {} # dict to remember how far we have read in the file
#Set defaults
if not self.settings.has_key('imp-callFpdbHud'):
self.settings['imp-callFpdbHud'] = False
if not self.settings.has_key('minPrint'):
self.settings['minPrint'] = 30
self.dbConnect()
def dbConnect(self, options, settings):
def dbConnect(self):
#connect to DB
if settings['db-backend'] == 2:
if self.settings['db-backend'] == 2:
if not mysqlLibFound:
raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file")
self.db = MySQLdb.connect(host = options.server, user = options.user,
passwd = options.password, db = options.database)
elif settings['db-backend'] == 3:
self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'],
self.settings['db-password'], self.settings['db-databaseName'])
elif self.settings['db-backend'] == 3:
if not pgsqlLibFound:
raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file")
self.db = psycopg2.connect(host = options.server, user = options.user,
password = options.password, database = options.database)
elif settings['db-backend'] == 4:
self.db = psycopg2.connect(self.settings['db-host'], self.settings['db-user'],
self.settings['db-password'], self.settings['db-databaseName'])
elif self.settings['db-backend'] == 4:
pass
else:
pass
self.cursor = self.db.cursor()
#Set functions
def setCallHud(self, value):
self.callHud = value
def import_file_dict(self, options, settings):
self.options=options
def setMinPrint(self, value):
self.settings['minPrint'] = int(value)
def setHandCount(self, value):
self.settings['handCount'] = int(value)
def setQuiet(self, value):
self.settings['quiet'] = value
def setFailOnError(self, value):
self.settings['failOnError'] = value
def setWatchTime(self):
self.updated = time()
def clearFileList(self):
self.filelist = []
#Add an individual file to filelist
def addImportFile(self, filename):
#todo: test it is a valid file
self.filelist = self.filelist + [filename]
#Remove duplicates
self.filelist = list(set(self.filelist))
#Add a directory of files to filelist
def addImportDirectory(self,dir,monitor = False):
#todo: test it is a valid directory
if monitor == True:
self.monitor = True
self.dirlist = self.dirlist + [dir]
for file in os.listdir(dir):
if os.path.isdir(file):
print "BulkImport is not recursive - please select the final directory in which the history files are"
else:
self.filelist = self.filelist + [os.path.join(dir, file)]
#Remove duplicates
self.filelist = list(set(self.filelist))
#Run full import on filelist
def runImport(self):
for file in self.filelist:
self.import_file_dict(file)
#Run import on updated files, then store latest update time.
def runUpdated(self):
#Check for new files in directory
#todo: make efficient - always checks for new file, should be able to use mtime of directory
# ^^ May not work on windows
for dir in self.dirlist:
for file in os.listdir(dir):
self.filelist = self.filelist + [dir+os.sep+file]
self.filelist = list(set(self.filelist))
for file in self.filelist:
stat_info = os.stat(file)
if stat_info.st_mtime > self.updated:
self.import_file_dict(file)
self.updated = time()
# This is now an internal function that should not be called directly.
def import_file_dict(self, file):
starttime = time()
last_read_hand=0
loc = 0
if (options.inputFile=="stdin"):
if (file=="stdin"):
inputFile=sys.stdin
else:
inputFile=open(options.inputFile, "rU")
try: loc = self.pos_in_file[options.inputFile]
inputFile=open(file, "rU")
try: loc = self.pos_in_file[file]
except: pass
self.dbConnect(options,settings)
# Read input file into class and close file
inputFile.seek(loc)
self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines())
self.pos_in_file[options.inputFile] = inputFile.tell()
self.pos_in_file[file] = inputFile.tell()
inputFile.close()
firstline = self.lines[0]
if firstline.find("Tournament Summary")!=-1:
print "TODO: implement importing tournament summaries"
self.cursor.close()
self.db.close()
return 0
site=fpdb_simple.recogniseSite(firstline)
@ -151,41 +220,36 @@ class Importer:
stored+=1
self.db.commit()
# if settings['imp-callFpdbHud'] and self.callHud and os.sep=='/':
if settings['imp-callFpdbHud'] and self.callHud:
if self.settings['imp-callFpdbHud'] and self.callHud:
#print "call to HUD here. handsId:",handsId
#pipe the Hands.id out to the HUD
# options.pipe_to_hud.write("%s" % (handsId) + os.linesep)
options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
except fpdb_simple.DuplicateError:
duplicates+=1
except (ValueError), fe:
errors+=1
self.printEmailErrorMessage(errors, options.inputFile, hand[0])
self.printEmailErrorMessage(errors, file, hand[0])
if (options.failOnError):
self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends.
self.cursor.close()
self.db.close()
if (self.settings['failOnError']):
self.db.commit() #dont remove this, in case hand processing was cancelled.
raise
except (fpdb_simple.FpdbError), fe:
errors+=1
self.printEmailErrorMessage(errors, options.inputFile, hand[0])
self.printEmailErrorMessage(errors, file, hand[0])
#fe.printStackTrace() #todo: get stacktrace
self.db.rollback()
if (options.failOnError):
self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends.
self.cursor.close()
self.db.close()
if (self.settings['failOnError']):
self.db.commit() #dont remove this, in case hand processing was cancelled.
raise
if (options.minPrint!=0):
if ((stored+duplicates+partial+errors)%options.minPrint==0):
if (self.settings['minPrint']!=0):
if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0):
print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors
if (options.handCount!=0):
if ((stored+duplicates+partial+errors)>=options.handCount):
if (not options.quiet):
if (self.settings['handCount']!=0):
if ((stored+duplicates+partial+errors)>=self.settings['handCount']):
if (not self.settings['quiet']):
print "quitting due to reaching the amount of hands to be imported"
print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime)
sys.exit(0)
@ -203,14 +267,12 @@ class Importer:
handsId=0
#todo: this will cause return of an unstored hand number if the last hand was error or partial
self.db.commit()
self.cursor.close()
self.db.close()
return handsId
#end def import_file_dict
def printEmailErrorMessage(self, errors, filename, line):
print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it."
print "Filename:", self.options.inputFile
print "Filename:", filename
print "Here is the first line so you can identify it. Please mention that the error was a ValueError:"
print self.hand[0]