Merge branch 'master' of git://git.assembla.com/fpdboz

Conflicts:

	pyfpdb/fpdb_import.py
This commit is contained in:
Ray 2008-10-16 15:21:03 -04:00
commit 1fde7655ea
7 changed files with 135 additions and 51 deletions

View File

@ -1,7 +1,7 @@
# Copyright 1999-2008 Gentoo Foundation
# Gentoo had nothing to do with the production of this ebuild, but I'm pre-emptively transferring all copyrights (as far as legally possible under my local jurisdiction) to them.
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/games-util/fpdb/fpdb-1.0_alpha5_p110.ebuild,v 1.0 2008/09/26 steffen@sycamoretest.info Exp $
# $Header: /var/cvsroot/gentoo-x86/games-util/fpdb/fpdb-1.0_alpha7_p136.ebuild,v 1.0 2008/10/15 steffen@sycamoretest.info Exp $
NEED_PYTHON=2.3

View File

@ -59,6 +59,7 @@ if __name__ == "__main__":
settings['db-password']=options.password
settings['db-databaseName']=options.database
settings['handCount']=options.handCount
settings['failOnError']=options.failOnError
importer = fpdb_import.Importer(options, settings)
importer.addImportFile(options.inputFile)

View File

@ -109,7 +109,7 @@ class GuiAutoImport (threading.Thread):
self.importer.setQuiet(False)
self.importer.setFailOnError(False)
self.importer.setHandCount(0)
self.importer.setWatchTime()
# self.importer.setWatchTime()
self.server=settings['db-host']
self.user=settings['db-user']

View File

@ -220,7 +220,7 @@ class GuiTableViewer (threading.Thread):
#print "start of table_viewer.read_names_clicked"
self.db.reconnect()
self.cursor=self.db.cursor
self.hands_id=self.last_read_hand_id
#self.hands_id=self.last_read_hand_id
self.db.cursor.execute("SELECT gametypeId FROM Hands WHERE id=%s", (self.hands_id, ))
self.gametype_id=self.db.cursor.fetchone()[0]
@ -257,7 +257,9 @@ class GuiTableViewer (threading.Thread):
self.importer.setFailOnError(False)
self.importer.setHandCount(0)
self.last_read_hand_id=self.importer.import_file_dict()
self.importer.addImportFile(self.inputFile)
self.importer.runImport()
self.hands_id=self.importer.handsId
#end def table_viewer.import_clicked
def all_clicked(self, widget, data):

View File

@ -426,7 +426,7 @@ This program is licensed under the AGPL3, see docs"""+os.sep+"agpl-3.0.txt")
self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
self.window.connect("delete_event", self.delete_event)
self.window.connect("destroy", self.destroy)
self.window.set_title("Free Poker DB - version: alpha6+, p131 or higher")
self.window.set_title("Free Poker DB - version: alpha7, p136")
self.window.set_border_width(1)
self.window.set_size_request(1020,400)
self.window.set_resizable(True)

View File

@ -34,69 +34,148 @@ except:
import math
import os
import datetime
import re
import fpdb_simple
import fpdb_parse_logic
from optparse import OptionParser
from time import time
class Importer:
def __init__(self):
def __init__(self, caller, settings):
"""Constructor"""
self.settings={'imp-callFpdbHud':False}
self.settings=settings
self.caller=caller
self.db = None
self.cursor = None
self.options = None
self.filelist = []
self.dirlist = []
self.monitor = False
self.updated = {} #Time last import was run {file:mtime}
self.callHud = False
self.lines = None
self.faobs = None #File as one big string
self.pos_in_file = {} # dict to remember how far we have read in the file
#Set defaults
if not self.settings.has_key('imp-callFpdbHud'):
self.settings['imp-callFpdbHud'] = False
if not self.settings.has_key('minPrint'):
self.settings['minPrint'] = 30
self.dbConnect()
def dbConnect(self, options, settings):
def dbConnect(self):
#connect to DB
if settings['db-backend'] == 2:
if self.settings['db-backend'] == 2:
if not mysqlLibFound:
raise fpdb_simple.FpdbError("interface library MySQLdb not found but MySQL selected as backend - please install the library or change the config file")
self.db = MySQLdb.connect(host = options.server, user = options.user,
passwd = options.password, db = options.database)
elif settings['db-backend'] == 3:
self.db = MySQLdb.connect(self.settings['db-host'], self.settings['db-user'],
self.settings['db-password'], self.settings['db-databaseName'])
elif self.settings['db-backend'] == 3:
if not pgsqlLibFound:
raise fpdb_simple.FpdbError("interface library psycopg2 not found but PostgreSQL selected as backend - please install the library or change the config file")
self.db = psycopg2.connect(host = options.server, user = options.user,
password = options.password, database = options.database)
elif settings['db-backend'] == 4:
self.db = psycopg2.connect(self.settings['db-host'], self.settings['db-user'],
self.settings['db-password'], self.settings['db-databaseName'])
elif self.settings['db-backend'] == 4:
pass
else:
pass
self.cursor = self.db.cursor()
#Set functions
def setCallHud(self, value):
self.callHud = value
def import_file_dict(self, options, settings):
def setMinPrint(self, value):
self.settings['minPrint'] = int(value)
def setHandCount(self, value):
self.settings['handCount'] = int(value)
def setQuiet(self, value):
self.settings['quiet'] = value
def setFailOnError(self, value):
self.settings['failOnError'] = value
# def setWatchTime(self):
# self.updated = time()
def clearFileList(self):
self.filelist = []
#Add an individual file to filelist
def addImportFile(self, filename):
#todo: test it is a valid file
self.filelist = self.filelist + [filename]
#Remove duplicates
self.filelist = list(set(self.filelist))
#Add a directory of files to filelist
def addImportDirectory(self,dir,monitor = False):
#todo: test it is a valid directory
if monitor == True:
self.monitor = True
self.dirlist = self.dirlist + [dir]
for file in os.listdir(dir):
if os.path.isdir(file):
print "BulkImport is not recursive - please select the final directory in which the history files are"
else:
self.filelist = self.filelist + [os.path.join(dir, file)]
#Remove duplicates
self.filelist = list(set(self.filelist))
#Run full import on filelist
def runImport(self):
for file in self.filelist:
self.import_file_dict(file)
#Run import on updated files, then store latest update time.
def runUpdated(self):
#Check for new files in directory
#todo: make efficient - always checks for new file, should be able to use mtime of directory
# ^^ May not work on windows
for dir in self.dirlist:
for file in os.listdir(dir):
self.filelist = self.filelist + [dir+os.sep+file]
self.filelist = list(set(self.filelist))
for file in self.filelist:
stat_info = os.stat(file)
try:
lastupdate = self.updated[file]
# print "Is " + str(stat_info.st_mtime) + " > " + str(lastupdate)
if stat_info.st_mtime > lastupdate:
self.import_file_dict(file)
self.updated[file] = time()
except:
# print "Adding " + str(file) + " at approx " + str(time())
self.updated[file] = time()
# This is now an internal function that should not be called directly.
def import_file_dict(self, file):
starttime = time()
last_read_hand=0
loc = 0
if (options.inputFile=="stdin"):
if (file=="stdin"):
inputFile=sys.stdin
else:
inputFile=open(options.inputFile, "rU")
try: loc = self.pos_in_file[options.inputFile]
inputFile=open(file, "rU")
try: loc = self.pos_in_file[file]
except: pass
self.dbConnect(options,settings)
# Read input file into class and close file
inputFile.seek(loc)
self.lines=fpdb_simple.removeTrailingEOL(inputFile.readlines())
self.pos_in_file[options.inputFile] = inputFile.tell()
self.pos_in_file[file] = inputFile.tell()
inputFile.close()
firstline = self.lines[0]
if firstline.find("Tournament Summary")!=-1:
print "TODO: implement importing tournament summaries"
self.cursor.close()
self.db.close()
self.faobs = readfile(inputFile)
self.parseTourneyHistory()
return 0
site=fpdb_simple.recogniseSite(firstline)
@ -142,7 +221,8 @@ class Importer:
if not isTourney:
fpdb_simple.filterAnteBlindFold(site,hand)
hand=fpdb_simple.filterCrap(site, hand, isTourney)
self.hand=hand
try:
handsId=fpdb_parse_logic.mainParser(self.db, self.cursor, site, category, hand)
self.db.commit()
@ -150,42 +230,36 @@ class Importer:
stored+=1
self.db.commit()
# if settings['imp-callFpdbHud'] and self.callHud and os.sep=='/':
if settings['imp-callFpdbHud'] and self.callHud:
if self.settings['imp-callFpdbHud'] and self.callHud:
#print "call to HUD here. handsId:",handsId
#pipe the Hands.id out to the HUD
# options.pipe_to_hud.write("%s" % (handsId) + os.linesep)
print "handsID = ", handsID
options.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
self.caller.pipe_to_hud.stdin.write("%s" % (handsId) + os.linesep)
except fpdb_simple.DuplicateError:
duplicates+=1
except (ValueError), fe:
errors+=1
self.printEmailErrorMessage(errors, options.inputFile, hand[0])
self.printEmailErrorMessage(errors, file, hand[0])
if (options.failOnError):
self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends.
self.cursor.close()
self.db.close()
if (self.settings['failOnError']):
self.db.commit() #dont remove this, in case hand processing was cancelled.
raise
except (fpdb_simple.FpdbError), fe:
errors+=1
self.printEmailErrorMessage(errors, options.inputFile, hand[0])
self.printEmailErrorMessage(errors, file, hand[0])
#fe.printStackTrace() #todo: get stacktrace
self.db.rollback()
if (options.failOnError):
self.db.commit() #dont remove this, in case hand processing was cancelled this ties up any open ends.
self.cursor.close()
self.db.close()
if (self.settings['failOnError']):
self.db.commit() #dont remove this, in case hand processing was cancelled.
raise
if (options.minPrint!=0):
if ((stored+duplicates+partial+errors)%options.minPrint==0):
if (self.settings['minPrint']!=0):
if ((stored+duplicates+partial+errors)%self.settings['minPrint']==0):
print "stored:", stored, "duplicates:", duplicates, "partial:", partial, "errors:", errors
if (options.handCount!=0):
if ((stored+duplicates+partial+errors)>=options.handCount):
if (not options.quiet):
if (self.settings['handCount']!=0):
if ((stored+duplicates+partial+errors)>=self.settings['handCount']):
if (not self.settings['quiet']):
print "quitting due to reaching the amount of hands to be imported"
print "Total stored:", stored, "duplicates:", duplicates, "partial/damaged:", partial, "errors:", errors, " time:", (time() - starttime)
sys.exit(0)
@ -203,16 +277,21 @@ class Importer:
handsId=0
#todo: this will cause return of an unstored hand number if the last hand was error or partial
self.db.commit()
self.cursor.close()
self.db.close()
self.handsId=handsId
return handsId
#end def import_file_dict
def parseTourneyHistory(self):
print "Tourney history parser stub"
#Find tournament boundaries.
#print self.foabs
def printEmailErrorMessage(self, errors, filename, line):
print "Error No.",errors,", please send the hand causing this to steffen@sycamoretest.info so I can fix it."
print "Filename:",options.inputFile
print "Filename:", filename
print "Here is the first line so you can identify it. Please mention that the error was a ValueError:"
print hand[0]
print self.hand[0]
if __name__ == "__main__":

View File

@ -315,10 +315,12 @@ def filterAnteBlindFold(site,hand):
#removes useless lines as well as trailing spaces
def filterCrap(site, hand, isTourney):
#remove one trailing space at end of line
#remove two trailing spaces at end of line
for i in range (len(hand)):
if (hand[i][-1]==' '):
hand[i]=hand[i][:-1]
if (hand[i][-1]==' '):
hand[i]=hand[i][:-1]
#print "hand after trailing space removal in filterCrap:",hand
#general variable position word filter/string filter