GuiTourneyImport: First pass at a 'Bulk Importer' for Tourney Summaries
This commit is contained in:
Worros
parent
a39b4704e6
commit
1c81df21ce
278
pyfpdb/GuiTourneyImport.py
Executable file
278
pyfpdb/GuiTourneyImport.py
Executable file
|
|
@ -0,0 +1,278 @@
|
|||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#Copyright 2008-2010 Carl Gherardi
|
||||
#This program is free software: you can redistribute it and/or modify
|
||||
#it under the terms of the GNU Affero General Public License as published by
|
||||
#the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
#This program is distributed in the hope that it will be useful,
|
||||
#but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
#GNU General Public License for more details.
|
||||
#
|
||||
#You should have received a copy of the GNU Affero General Public License
|
||||
#along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#In the "official" distribution you can find the license in agpl-3.0.txt.
|
||||
|
||||
import L10n
|
||||
_ = L10n.get_translation()
|
||||
|
||||
# Standard Library modules
|
||||
import os
|
||||
import sys
|
||||
from time import time
|
||||
import traceback
|
||||
import datetime
|
||||
import codecs
|
||||
import re
|
||||
|
||||
# pyGTK modules
|
||||
import pygtk
|
||||
pygtk.require('2.0')
|
||||
import gtk
|
||||
import gobject
|
||||
|
||||
# fpdb/FreePokerTools modules
|
||||
import Configuration
|
||||
from Exceptions import FpdbParseError
|
||||
|
||||
import logging
|
||||
# logging has been set up in fpdb.py or HUD_main.py, use their settings:
|
||||
log = logging.getLogger("importer")
|
||||
|
||||
|
||||
class GuiTourneyImport():
|
||||
|
||||
def load_clicked(self, widget, data=None):
|
||||
print "DEBUG: load_clicked"
|
||||
stored = None
|
||||
dups = None
|
||||
partial = None
|
||||
errs = None
|
||||
ttime = None
|
||||
|
||||
if self.settings['global_lock'].acquire(wait=False, source="GuiTourneyImport"):
|
||||
print "DEBUG: got global lock"
|
||||
# get the dir to import from the chooser
|
||||
selected = self.chooser.get_filenames()
|
||||
print "DEBUG: Files selected: %s" % selected
|
||||
|
||||
sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0]
|
||||
|
||||
for selection in selected:
|
||||
self.importer.addImportFileOrDir(selection, site = sitename)
|
||||
starttime = time()
|
||||
(stored, errs) = self.importer.runImport()
|
||||
|
||||
ttime = time() - starttime
|
||||
if ttime == 0:
|
||||
ttime = 1
|
||||
print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\
|
||||
% (stored, errs, ttime, (stored+0.0) / ttime)
|
||||
self.importer.clearFileList()
|
||||
|
||||
self.settings['global_lock'].release()
|
||||
else:
|
||||
print _("bulk import aborted - global lock not available")
|
||||
|
||||
def get_vbox(self):
|
||||
"""returns the vbox of this thread"""
|
||||
return self.vbox
|
||||
|
||||
def __init__(self, settings, config, sql = None, parent = None):
|
||||
self.settings = settings
|
||||
self.config = config
|
||||
self.parent = parent
|
||||
|
||||
self.importer = SummaryImporter(config, sql, parent)
|
||||
|
||||
self.vbox = gtk.VBox(False, 0)
|
||||
self.vbox.show()
|
||||
|
||||
self.chooser = gtk.FileChooserWidget()
|
||||
self.chooser.set_filename(self.settings['bulkImport-defaultPath'])
|
||||
self.chooser.set_select_multiple(True)
|
||||
self.vbox.add(self.chooser)
|
||||
self.chooser.show()
|
||||
|
||||
# Table widget to hold the settings
|
||||
self.table = gtk.Table(rows=5, columns=5, homogeneous=False)
|
||||
self.vbox.add(self.table)
|
||||
self.table.show()
|
||||
|
||||
# label - tsc
|
||||
self.lab_filter = gtk.Label(_("Site filter:"))
|
||||
self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0,
|
||||
yoptions=gtk.SHRINK)
|
||||
self.lab_filter.show()
|
||||
self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT)
|
||||
self.lab_filter.set_alignment(1.0, 0.5)
|
||||
|
||||
# ComboBox - filter
|
||||
self.cbfilter = gtk.combo_box_new_text()
|
||||
disabled_sites = [] # move disabled sites to bottom of list
|
||||
for w in self.config.hhcs:
|
||||
print "%s = '%s'" %(w, self.config.hhcs[w].summaryImporter)
|
||||
try:
|
||||
# Include sites with a tourney summary importer, and enabled
|
||||
if self.config.supported_sites[w].enabled and self.config.hhcs[w].summaryImporter != '':
|
||||
self.cbfilter.append_text(w)
|
||||
else:
|
||||
disabled_sites.append(w)
|
||||
except: # self.supported_sites[w] may not exist if hud_config is bad
|
||||
disabled_sites.append(w)
|
||||
for w in disabled_sites:
|
||||
if self.config.hhcs[w].summaryImporter != '':
|
||||
self.cbfilter.append_text(w)
|
||||
self.cbfilter.set_active(0)
|
||||
self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1,
|
||||
yoptions=gtk.SHRINK)
|
||||
self.cbfilter.show()
|
||||
|
||||
# button - Import
|
||||
self.load_button = gtk.Button(_('_Bulk Import')) # todo: rename variables to import too
|
||||
self.load_button.connect('clicked', self.load_clicked,
|
||||
_('Import clicked'))
|
||||
self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0,
|
||||
yoptions=gtk.SHRINK)
|
||||
self.load_button.show()
|
||||
|
||||
# label - spacer (keeps rows 3 & 5 apart)
|
||||
self.lab_spacer = gtk.Label()
|
||||
self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0,
|
||||
yoptions=gtk.SHRINK)
|
||||
self.lab_spacer.show()
|
||||
|
||||
class SummaryImporter:
|
||||
def __init__(self, config, sql = None, parent = None):
|
||||
self.config = config
|
||||
self.sql = sql
|
||||
self.parent = parent
|
||||
|
||||
self.filelist = {}
|
||||
self.dirlist = {}
|
||||
|
||||
self.updatedsize = {}
|
||||
self.updatedtime = {}
|
||||
|
||||
def addImportFile(self, filename, site = "default", tsc = "passthrough"):
|
||||
print "DEBUG: addImportFile"
|
||||
if filename in self.filelist or not os.path.exists(filename):
|
||||
print "DEBUG: addImportFile: File exists, or path non-existent"
|
||||
return
|
||||
self.filelist[filename] = [site] + [tsc]
|
||||
print "DEBUG: addImportFile: self.filelist[%s]: %s" %(filename, self.filelist[filename])
|
||||
|
||||
def addImportDirectory(self,dir,monitor=False, site="default", tsc="passthrough"):
|
||||
print "DEBUG: addImportDirectory"
|
||||
if os.path.isdir(dir):
|
||||
if monitor == True:
|
||||
self.monitor = True
|
||||
self.dirlist[site] = [dir] + [tsc]
|
||||
|
||||
for file in os.listdir(dir):
|
||||
self.addImportFile(os.path.join(dir, file), site, tsc)
|
||||
else:
|
||||
log.warning(_("Attempted to add non-directory '%s' as an import directory") % str(dir))
|
||||
|
||||
def addImportFileOrDir(self, inputPath, site = "PokerStars"):
|
||||
print "DEBUG: addImportFileOrDir"
|
||||
tsc = self.config.hhcs[site].summaryImporter
|
||||
if os.path.isdir(inputPath):
|
||||
for subdir in os.walk(inputPath):
|
||||
for file in subdir[2]:
|
||||
self.addImportFile(unicode(os.path.join(subdir[0], file),'utf-8'),
|
||||
site=site, tsc=tsc)
|
||||
else:
|
||||
self.addImportFile(unicode(inputPath,'utf-8'), site=site, tsc=tsc)
|
||||
pass
|
||||
|
||||
def runImport(self):
|
||||
start = datetime.datetime.now()
|
||||
starttime = time()
|
||||
log.info(_("Tourney Summary Import started at %s - %d files to import.") % (start, len(self.filelist)))
|
||||
|
||||
total_errors = 0
|
||||
total_imported = 0
|
||||
for f in self.filelist:
|
||||
(site, tsc) = self.filelist[f]
|
||||
imported, errs = self.importFile(f, tsc, site)
|
||||
total_errors += errs
|
||||
total_imported += imported
|
||||
return (total_imported, total_errors)
|
||||
|
||||
|
||||
def runUpdated(self):
|
||||
pass
|
||||
|
||||
def importFile(self, filename, tsc = "PokerStarsSummary", site = "PokerStars"):
|
||||
mod = __import__(tsc)
|
||||
obj = getattr(mod, tsc, None)
|
||||
if callable(obj):
|
||||
foabs = self.readFile(obj, filename)
|
||||
summaryTexts = re.split(obj.re_SplitTourneys, foabs)
|
||||
print "Found %s summaries" %(len(summaryTexts))
|
||||
errors = 0
|
||||
imported = 0
|
||||
for j, summaryText in enumerate(summaryTexts, start=1):
|
||||
try:
|
||||
conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP")
|
||||
except FpdbParseError, e:
|
||||
errors += 1
|
||||
print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts))
|
||||
imported = j
|
||||
return (imported - errors, errors)
|
||||
|
||||
def clearFileList(self):
|
||||
self.updatedsize = {}
|
||||
self.updatetime = {}
|
||||
self.filelist = {}
|
||||
|
||||
def readFile(self, tsc, filename):
|
||||
codepage = ["utf8"]
|
||||
whole_file = None
|
||||
tsc.codepage
|
||||
|
||||
for kodec in codepage:
|
||||
print "DEBUG: TSC.readFile: trying codepage '%s'" % kodec
|
||||
try:
|
||||
in_fh = codecs.open(filename, 'r', kodec)
|
||||
whole_file = in_fh.read()
|
||||
in_fh.close()
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
return whole_file
|
||||
|
||||
def main(argv=None):
|
||||
"""main can also be called in the python interpreter, by supplying the command line as the argument."""
|
||||
if argv is None:
|
||||
argv = sys.argv[1:]
|
||||
|
||||
(options, argv) = Options.fpdb_options()
|
||||
|
||||
if not options.config:
|
||||
options.config = Configuration.Config(file = "HUD_config.test.xml")
|
||||
|
||||
if options.usage == True:
|
||||
#Print usage examples and exit
|
||||
print _("USAGE:")
|
||||
sys.exit(0)
|
||||
|
||||
if options.hhc == "PokerStarsToFpdb":
|
||||
print _("Need to define a converter")
|
||||
exit(0)
|
||||
|
||||
config = Configuration.Config()
|
||||
db = Database.Database(config)
|
||||
sql = SQL.Sql(db_server = 'sqlite')
|
||||
|
||||
db.recreate_tables()
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue
Block a user