Changes import_file_dict interface to Importer internal.
Users of Importer should now addImportFile and addImportDirectory, set paramaters, then run either runImport - all files or runUpdated - modified files. Comments out postgres regression test until fixed Adds initial code for importing regression test files.
This commit is contained in:
Worros
parent
638a6d6dab
commit
6c4e2f3eb9
|
|
@ -50,16 +50,7 @@ class GuiAutoImport (threading.Thread):
|
|||
|
||||
def do_import(self):
|
||||
"""Callback for timer to do an import iteration."""
|
||||
for file in os.listdir(self.path):
|
||||
if os.path.isdir(file):
|
||||
print "AutoImport is not recursive - please select the final directory in which the history files are"
|
||||
else:
|
||||
self.inputFile = os.path.join(self.path, file)
|
||||
stat_info = os.stat(self.inputFile)
|
||||
if not self.import_files.has_key(self.inputFile) or stat_info.st_mtime > self.import_files[self.inputFile]:
|
||||
self.importer.import_file_dict()
|
||||
self.import_files[self.inputFile] = stat_info.st_mtime
|
||||
|
||||
self.importer.runUpdated()
|
||||
print "GuiAutoImport.import_dir done"
|
||||
return True
|
||||
|
||||
|
|
@ -96,17 +87,9 @@ class GuiAutoImport (threading.Thread):
|
|||
# self.pipe_to_hud = os.popen(command, 'w')
|
||||
self.path=self.pathTBuffer.get_text(self.pathTBuffer.get_start_iter(), self.pathTBuffer.get_end_iter())
|
||||
|
||||
# Iniitally populate the self.import_files dict, which keeps mtimes for the files watched
|
||||
|
||||
self.import_files = {}
|
||||
for file in os.listdir(self.path):
|
||||
if os.path.isdir(file):
|
||||
pass # skip subdirs for now
|
||||
else:
|
||||
inputFile = os.path.join(self.path, file)
|
||||
stat_info = os.stat(inputFile)
|
||||
self.import_files[inputFile] = stat_info.st_mtime
|
||||
|
||||
# Add directory to importer object and set the initial mtime reference.
|
||||
self.importer.addImportDirectory(self.path)
|
||||
self.importer.setWatchTime()
|
||||
self.do_import()
|
||||
|
||||
interval=int(self.intervalTBuffer.get_text(self.intervalTBuffer.get_start_iter(), self.intervalTBuffer.get_end_iter()))
|
||||
|
|
|
|||
|
|
@ -27,12 +27,8 @@ class GuiBulkImport (threading.Thread):
|
|||
def import_dir(self):
|
||||
"""imports a directory, non-recursive. todo: move this to fpdb_import so CLI can use it"""
|
||||
self.path=self.inputFile
|
||||
for file in os.listdir(self.path):
|
||||
if os.path.isdir(file):
|
||||
print "BulkImport is not recursive - please select the final directory in which the history files are"
|
||||
else:
|
||||
self.inputFile=self.path+os.sep+file
|
||||
self.importer.import_file_dict()
|
||||
self.importer.addImportDirectory(self.path)
|
||||
self.importer.runImport()
|
||||
print "GuiBulkImport.import_dir done"
|
||||
|
||||
def load_clicked(self, widget, data=None):
|
||||
|
|
@ -67,7 +63,9 @@ class GuiBulkImport (threading.Thread):
|
|||
if os.path.isdir(self.inputFile):
|
||||
self.import_dir()
|
||||
else:
|
||||
self.importer.import_file_dict()
|
||||
self.importer.addImportFile()
|
||||
self.importer.runImport()
|
||||
self.importer.clearFileList()
|
||||
|
||||
def get_vbox(self):
|
||||
"""returns the vbox of this thread"""
|
||||
|
|
|
|||
|
|
@ -25,51 +25,62 @@ import os
|
|||
import sys
|
||||
|
||||
import fpdb_db
|
||||
import fpdb_import
|
||||
import FpdbSQLQueries
|
||||
|
||||
import unittest
|
||||
|
||||
class TestSequenceFunctions(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
"""Configure MySQL settings/database and establish connection"""
|
||||
self.mysql_settings={ 'db-host':"localhost", 'db-backend':2, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
|
||||
self.mysql_db = fpdb_db.fpdb_db()
|
||||
self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'],
|
||||
self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'],
|
||||
self.mysql_settings['db-password'])
|
||||
self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB')
|
||||
def setUp(self):
|
||||
"""Configure MySQL settings/database and establish connection"""
|
||||
self.mysql_settings={ 'db-host':"localhost",
|
||||
'db-backend':2,
|
||||
'db-databaseName':"fpdbtest",
|
||||
'db-user':"fpdb",
|
||||
'db-password':"fpdb"}
|
||||
self.mysql_db = fpdb_db.fpdb_db()
|
||||
self.mysql_db.connect(self.mysql_settings['db-backend'], self.mysql_settings['db-host'],
|
||||
self.mysql_settings['db-databaseName'], self.mysql_settings['db-user'],
|
||||
self.mysql_settings['db-password'])
|
||||
self.mysqldict = FpdbSQLQueries.FpdbSQLQueries('MySQL InnoDB')
|
||||
self.mysqlimporter = fpdb_import.Importer(self, self.mysql_settings)
|
||||
|
||||
"""Configure Postgres settings/database and establish connection"""
|
||||
self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
|
||||
self.pg_db = fpdb_db.fpdb_db()
|
||||
self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'],
|
||||
self.pg_settings['db-databaseName'], self.pg_settings['db-user'],
|
||||
self.pg_settings['db-password'])
|
||||
self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL')
|
||||
# """Configure Postgres settings/database and establish connection"""
|
||||
# self.pg_settings={ 'db-host':"localhost", 'db-backend':3, 'db-databaseName':"fpdbtest", 'db-user':"fpdb", 'db-password':"fpdb"}
|
||||
# self.pg_db = fpdb_db.fpdb_db()
|
||||
# self.pg_db.connect(self.pg_settings['db-backend'], self.pg_settings['db-host'],
|
||||
# self.pg_settings['db-databaseName'], self.pg_settings['db-user'],
|
||||
# self.pg_settings['db-password'])
|
||||
# self.pgdict = FpdbSQLQueries.FpdbSQLQueries('PostgreSQL')
|
||||
|
||||
|
||||
def testDatabaseConnection(self):
|
||||
"""Test all supported DBs"""
|
||||
self.result = self.mysql_db.cursor.execute(self.mysqldict.query['list_tables'])
|
||||
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
|
||||
print self.pgdict.query['list_tables']
|
||||
|
||||
self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
|
||||
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
|
||||
# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
|
||||
# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
|
||||
def testMySQLRecreateTables(self):
|
||||
"""Test droping then recreating fpdb table schema"""
|
||||
self.mysql_db.recreate_tables()
|
||||
self.result = self.mysql_db.cursor.execute("SHOW TABLES")
|
||||
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
|
||||
def testPostgresSQLRecreateTables(self):
|
||||
"""Test droping then recreating fpdb table schema"""
|
||||
self.pg_db.recreate_tables()
|
||||
self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
|
||||
self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
def testImportHandHistoryFiles(self):
|
||||
"""Test import of single HH file"""
|
||||
self.mysqlimporter.addImportFile("regression-test-files/hand-histories/ps-lhe-ring-3hands.txt")
|
||||
self.mysqlimporter.runImport()
|
||||
self.mysqlimporter.addImportDirectory("regression-test-files/hand-histories")
|
||||
self.mysqlimporter.runImport()
|
||||
|
||||
# def testPostgresSQLRecreateTables(self):
|
||||
# """Test droping then recreating fpdb table schema"""
|
||||
# self.pg_db.recreate_tables()
|
||||
# self.result = self.pg_db.cursor.execute(self.pgdict.query['list_tables'])
|
||||
# self.failUnless(self.result==13, "Number of tables in database incorrect. Expected 13 got " + str(self.result))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -46,6 +46,9 @@ class Importer:
|
|||
self.caller=caller
|
||||
self.db = None
|
||||
self.cursor = None
|
||||
self.filelist = []
|
||||
self.queued = []
|
||||
self.updated = 0 #Time last import was run, used as mtime reference
|
||||
self.callHud = False
|
||||
self.lines = None
|
||||
self.pos_in_file = {} # dict to remember how far we have read in the file
|
||||
|
|
@ -74,12 +77,10 @@ class Importer:
|
|||
pass
|
||||
self.cursor = self.db.cursor()
|
||||
|
||||
#Set functions
|
||||
def setCallHud(self, value):
|
||||
self.callHud = value
|
||||
|
||||
def addImportFile(self, filename):
|
||||
self.caller.inputFile = filename
|
||||
|
||||
def setMinPrint(self, value):
|
||||
self.settings['minPrint'] = int(value)
|
||||
|
||||
|
|
@ -92,21 +93,62 @@ class Importer:
|
|||
def setFailOnError(self, value):
|
||||
self.settings['failOnError'] = value
|
||||
|
||||
def import_file_dict(self):
|
||||
def setWatchTime(self):
|
||||
self.updated = time()
|
||||
|
||||
def clearFileList(self):
|
||||
self.filelist = []
|
||||
|
||||
#Add an individual file to filelist
|
||||
def addImportFile(self, filename):
|
||||
#todo: test it is a valid file
|
||||
self.filelist = self.filelist + [filename]
|
||||
print "Filelist in addImportFile: ", self.filelist
|
||||
#Remove duplicates
|
||||
set(filelist)
|
||||
|
||||
#Add a directory of files to filelist
|
||||
def addImportDirectory(self,dir):
|
||||
#todo: test it is a valid directory
|
||||
for file in os.listdir(dir):
|
||||
if os.path.isdir(file):
|
||||
print "BulkImport is not recursive - please select the final directory in which the history files are"
|
||||
else:
|
||||
blah = [dir+os.sep+file]
|
||||
self.filelist = self.filelist + [dir+os.sep+file]
|
||||
#Remove duplicates
|
||||
set(self.filelist)
|
||||
|
||||
#Run full import on filelist
|
||||
def runImport(self):
|
||||
for file in self.filelist:
|
||||
print "Importing file: ", file
|
||||
self.import_file_dict(file)
|
||||
|
||||
#Run import on updated files, then store latest update time.
|
||||
def runUpdated(self):
|
||||
for file in self.filelist:
|
||||
stat_info = os.stat(file)
|
||||
if stat_info.st_mtime > self.updated:
|
||||
self.import_file_dict(file)
|
||||
self.updated = time()
|
||||
|
||||
# This is now an internal function that should not be called directly.
|
||||
def import_file_dict(self, file):
|
||||
starttime = time()
|
||||
last_read_hand=0
|
||||
loc = 0
|
||||
if (self.caller.inputFile=="stdin"):
|
||||
if (file=="stdin"):
|
||||
inputFile=sys.stdin
|
||||
else:
|
||||
inputFile=open(self.caller.inputFile, "rU")
|
||||
try: loc = self.pos_in_file[self.caller.inputFile]
|
||||
inputFile=open(file, "rU")
|
||||
try: loc = self.pos_in_file[file]
|
||||
except: pass
|
||||
|
||||
# Read input file into class and close file
|
||||
inputFile.seek(loc)
|
||||
self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines())
|
||||
self.pos_in_file[self.caller.inputFile] = inputFile.tell()
|
||||
self.pos_in_file[file] = inputFile.tell()
|
||||
inputFile.close()
|
||||
|
||||
firstline = self.lines[0]
|
||||
|
|
@ -175,14 +217,14 @@ class Importer:
|
|||
duplicates+=1
|
||||
except (ValueError), fe:
|
||||
errors+=1
|
||||
self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0])
|
||||
self.printEmailErrorMessage(errors, file, hand[0])
|
||||
|
||||
if (self.settings['failOnError']):
|
||||
self.db.commit() #dont remove this, in case hand processing was cancelled.
|
||||
raise
|
||||
except (fpdb_simple.FpdbError), fe:
|
||||
errors+=1
|
||||
self.printEmailErrorMessage(errors, self.caller.inputFile, hand[0])
|
||||
self.printEmailErrorMessage(errors, file, hand[0])
|
||||
|
||||
#fe.printStackTrace() #todo: get stacktrace
|
||||
self.db.rollback()
|
||||
|
|
@ -219,7 +261,7 @@ class Importer:
|
|||
|
||||
def printEmailErrorMessage(self, errors, filename, line):
|
||||
print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it."
|
||||
print "Filename:", self.caller.inputFile
|
||||
print "Filename:", filename
|
||||
print "Here is the first line so you can identify it. Please mention that the error was a ValueError:"
|
||||
print self.hand[0]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user