From 1c81df21ce7111ffe8efa27d455a3b2a691ded60 Mon Sep 17 00:00:00 2001 From: Worros Date: Tue, 9 Nov 2010 20:29:13 +0800 Subject: [PATCH] GuiTourneyImport: First pass at a 'Bulk Importer' for Tourney Summaries --- pyfpdb/GuiTourneyImport.py | 278 +++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100755 pyfpdb/GuiTourneyImport.py diff --git a/pyfpdb/GuiTourneyImport.py b/pyfpdb/GuiTourneyImport.py new file mode 100755 index 00000000..ba6d5bb1 --- /dev/null +++ b/pyfpdb/GuiTourneyImport.py @@ -0,0 +1,278 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +#Copyright 2008-2010 Carl Gherardi +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, version 3 of the License. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . +#In the "official" distribution you can find the license in agpl-3.0.txt. + +import L10n +_ = L10n.get_translation() + +# Standard Library modules +import os +import sys +from time import time +import traceback +import datetime +import codecs +import re + +# pyGTK modules +import pygtk +pygtk.require('2.0') +import gtk +import gobject + +# fpdb/FreePokerTools modules +import Configuration +from Exceptions import FpdbParseError + +import logging +# logging has been set up in fpdb.py or HUD_main.py, use their settings: +log = logging.getLogger("importer") + + +class GuiTourneyImport(): + + def load_clicked(self, widget, data=None): + print "DEBUG: load_clicked" + stored = None + dups = None + partial = None + errs = None + ttime = None + + if self.settings['global_lock'].acquire(wait=False, source="GuiTourneyImport"): + print "DEBUG: got global lock" + # get the dir to import from the chooser + selected = self.chooser.get_filenames() + print "DEBUG: Files selected: %s" % selected + + sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0] + + for selection in selected: + self.importer.addImportFileOrDir(selection, site = sitename) + starttime = time() + (stored, errs) = self.importer.runImport() + + ttime = time() - starttime + if ttime == 0: + ttime = 1 + print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\ + % (stored, errs, ttime, (stored+0.0) / ttime) + self.importer.clearFileList() + + self.settings['global_lock'].release() + else: + print _("bulk import aborted - global lock not available") + + def get_vbox(self): + """returns the vbox of this thread""" + return self.vbox + + def __init__(self, settings, config, sql = None, parent = None): + self.settings = settings + self.config = config + self.parent = parent + + self.importer = SummaryImporter(config, sql, parent) + + self.vbox = gtk.VBox(False, 0) + self.vbox.show() + + self.chooser = gtk.FileChooserWidget() + self.chooser.set_filename(self.settings['bulkImport-defaultPath']) + self.chooser.set_select_multiple(True) + self.vbox.add(self.chooser) + self.chooser.show() + +# Table widget to hold the settings + self.table = gtk.Table(rows=5, columns=5, homogeneous=False) + self.vbox.add(self.table) + self.table.show() + +# label - tsc + self.lab_filter = gtk.Label(_("Site filter:")) + self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0, + yoptions=gtk.SHRINK) + self.lab_filter.show() + self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT) + self.lab_filter.set_alignment(1.0, 0.5) + +# ComboBox - filter + self.cbfilter = gtk.combo_box_new_text() + disabled_sites = [] # move disabled sites to bottom of list + for w in self.config.hhcs: + print "%s = '%s'" %(w, self.config.hhcs[w].summaryImporter) + try: + # Include sites with a tourney summary importer, and enabled + if self.config.supported_sites[w].enabled and self.config.hhcs[w].summaryImporter != '': + self.cbfilter.append_text(w) + else: + disabled_sites.append(w) + except: # self.supported_sites[w] may not exist if hud_config is bad + disabled_sites.append(w) + for w in disabled_sites: + if self.config.hhcs[w].summaryImporter != '': + self.cbfilter.append_text(w) + self.cbfilter.set_active(0) + self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1, + yoptions=gtk.SHRINK) + self.cbfilter.show() + +# button - Import + self.load_button = gtk.Button(_('_Bulk Import')) # todo: rename variables to import too + self.load_button.connect('clicked', self.load_clicked, + _('Import clicked')) + self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0, + yoptions=gtk.SHRINK) + self.load_button.show() + +# label - spacer (keeps rows 3 & 5 apart) + self.lab_spacer = gtk.Label() + self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0, + yoptions=gtk.SHRINK) + self.lab_spacer.show() + +class SummaryImporter: + def __init__(self, config, sql = None, parent = None): + self.config = config + self.sql = sql + self.parent = parent + + self.filelist = {} + self.dirlist = {} + + self.updatedsize = {} + self.updatedtime = {} + + def addImportFile(self, filename, site = "default", tsc = "passthrough"): + print "DEBUG: addImportFile" + if filename in self.filelist or not os.path.exists(filename): + print "DEBUG: addImportFile: File exists, or path non-existent" + return + self.filelist[filename] = [site] + [tsc] + print "DEBUG: addImportFile: self.filelist[%s]: %s" %(filename, self.filelist[filename]) + + def addImportDirectory(self,dir,monitor=False, site="default", tsc="passthrough"): + print "DEBUG: addImportDirectory" + if os.path.isdir(dir): + if monitor == True: + self.monitor = True + self.dirlist[site] = [dir] + [tsc] + + for file in os.listdir(dir): + self.addImportFile(os.path.join(dir, file), site, tsc) + else: + log.warning(_("Attempted to add non-directory '%s' as an import directory") % str(dir)) + + def addImportFileOrDir(self, inputPath, site = "PokerStars"): + print "DEBUG: addImportFileOrDir" + tsc = self.config.hhcs[site].summaryImporter + if os.path.isdir(inputPath): + for subdir in os.walk(inputPath): + for file in subdir[2]: + self.addImportFile(unicode(os.path.join(subdir[0], file),'utf-8'), + site=site, tsc=tsc) + else: + self.addImportFile(unicode(inputPath,'utf-8'), site=site, tsc=tsc) + pass + + def runImport(self): + start = datetime.datetime.now() + starttime = time() + log.info(_("Tourney Summary Import started at %s - %d files to import.") % (start, len(self.filelist))) + + total_errors = 0 + total_imported = 0 + for f in self.filelist: + (site, tsc) = self.filelist[f] + imported, errs = self.importFile(f, tsc, site) + total_errors += errs + total_imported += imported + return (total_imported, total_errors) + + + def runUpdated(self): + pass + + def importFile(self, filename, tsc = "PokerStarsSummary", site = "PokerStars"): + mod = __import__(tsc) + obj = getattr(mod, tsc, None) + if callable(obj): + foabs = self.readFile(obj, filename) + summaryTexts = re.split(obj.re_SplitTourneys, foabs) + print "Found %s summaries" %(len(summaryTexts)) + errors = 0 + imported = 0 + for j, summaryText in enumerate(summaryTexts, start=1): + try: + conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP") + except FpdbParseError, e: + errors += 1 + print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts)) + imported = j + return (imported - errors, errors) + + def clearFileList(self): + self.updatedsize = {} + self.updatetime = {} + self.filelist = {} + + def readFile(self, tsc, filename): + codepage = ["utf8"] + whole_file = None + tsc.codepage + + for kodec in codepage: + print "DEBUG: TSC.readFile: trying codepage '%s'" % kodec + try: + in_fh = codecs.open(filename, 'r', kodec) + whole_file = in_fh.read() + in_fh.close() + break + except: + pass + + return whole_file + +def main(argv=None): + """main can also be called in the python interpreter, by supplying the command line as the argument.""" + if argv is None: + argv = sys.argv[1:] + + (options, argv) = Options.fpdb_options() + + if not options.config: + options.config = Configuration.Config(file = "HUD_config.test.xml") + + if options.usage == True: + #Print usage examples and exit + print _("USAGE:") + sys.exit(0) + + if options.hhc == "PokerStarsToFpdb": + print _("Need to define a converter") + exit(0) + + config = Configuration.Config() + db = Database.Database(config) + sql = SQL.Sql(db_server = 'sqlite') + + db.recreate_tables() + + + + +if __name__ == '__main__': + sys.exit(main())