fpdb/pyfpdb/GuiTourneyImport.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

#Copyright 2008-2011 Carl Gherardi
#This program is free software: you can redistribute it and/or modify
#it under the terms of the GNU Affero General Public License as published by
#the Free Software Foundation, version 3 of the License.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU Affero General Public License
#along with this program. If not, see <http://www.gnu.org/licenses/>.
#In the "official" distribution you can find the license in agpl-3.0.txt.

import L10n
_ = L10n.get_translation()

#    Standard Library modules
import os
import sys
from time import time
import traceback
import datetime
import codecs
import re

#    pyGTK modules
import pygtk
pygtk.require('2.0')
import gtk
import gobject

#    fpdb/FreePokerTools modules
import Configuration
import Options
from Exceptions import FpdbParseError

import logging
# logging has been set up in fpdb.py or HUD_main.py, use their settings:
log = logging.getLogger("importer")


class GuiTourneyImport():

    def load_clicked(self, widget, data=None):
        print "DEBUG: load_clicked"
        stored = None
        dups = None
        partial = None
        errs = None
        ttime = None

        if self.settings['global_lock'].acquire(wait=False, source="GuiTourneyImport"):
            print "DEBUG: got global lock"
            #    get the dir to import from the chooser
            selected = self.chooser.get_filenames()
            print "DEBUG: Files selected: %s" % selected

            sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0]

            for selection in selected:
                self.importer.addImportFileOrDir(selection, site = sitename)
            starttime = time()
            (stored, errs) = self.importer.runImport()

            ttime = time() - starttime
            if ttime == 0:
                ttime = 1
            print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\
                     % (stored, errs, ttime, (stored+0.0) / ttime)
            self.importer.clearFileList()

            self.settings['global_lock'].release()
        else:
            print _("bulk import aborted - global lock not available")

    def get_vbox(self):
        """returns the vbox of this thread"""
        return self.vbox

    def __init__(self, settings, config, sql = None, parent = None):
        self.settings = settings
        self.config = config
        self.parent = parent

        self.importer = SummaryImporter(config, sql, parent)

        self.vbox = gtk.VBox(False, 0)
        self.vbox.show()

        self.chooser = gtk.FileChooserWidget()
        self.chooser.set_filename(self.settings['bulkImport-defaultPath'])
        self.chooser.set_select_multiple(True)
        self.vbox.add(self.chooser)
        self.chooser.show()

#    Table widget to hold the settings
        self.table = gtk.Table(rows=5, columns=5, homogeneous=False)
        self.vbox.add(self.table)
        self.table.show()

#    label - tsc 
        self.lab_filter = gtk.Label(_("Site filter:"))
        self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0,
                          yoptions=gtk.SHRINK)
        self.lab_filter.show()
        self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT)
        self.lab_filter.set_alignment(1.0, 0.5)

#    ComboBox - filter
        self.cbfilter = gtk.combo_box_new_text()
        disabled_sites = []                                # move disabled sites to bottom of list
        for w in self.config.hhcs:
            print "%s = '%s'" %(w, self.config.hhcs[w].summaryImporter)
            try:
                # Include sites with a tourney summary importer, and enabled
                if self.config.supported_sites[w].enabled and self.config.hhcs[w].summaryImporter != '':
                    self.cbfilter.append_text(w)
                else:
                    disabled_sites.append(w)
            except: # self.supported_sites[w] may not exist if hud_config is bad
                disabled_sites.append(w)
        for w in disabled_sites:
            if self.config.hhcs[w].summaryImporter != '':
                self.cbfilter.append_text(w)
        self.cbfilter.set_active(0)
        self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1,
                          yoptions=gtk.SHRINK)
        self.cbfilter.show()

#    button - Import
        self.load_button = gtk.Button(_('_Bulk Import'))  # todo: rename variables to import too
        self.load_button.connect('clicked', self.load_clicked,
                                 _('Import clicked'))
        self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0,
                          yoptions=gtk.SHRINK)
        self.load_button.show()

#    label - spacer (keeps rows 3 & 5 apart)
        self.lab_spacer = gtk.Label()
        self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0,
                          yoptions=gtk.SHRINK)
        self.lab_spacer.show()

class SummaryImporter:
    def __init__(self, config, sql = None, parent = None):
        self.config     = config
        self.sql        = sql
        self.parent     = parent

        self.filelist   = {}
        self.dirlist    = {}

        self.updatedsize = {}
        self.updatedtime = {}

    def addImportFile(self, filename, site = "default", tsc = "passthrough"):
        if filename in self.filelist or not os.path.exists(filename):
            print "DEBUG: addImportFile: File exists, or path non-existent"
            return
        self.filelist[filename] = [site] + [tsc]

    def addImportDirectory(self,dir,monitor=False, site="default", tsc="passthrough"):
        if os.path.isdir(dir):
            if monitor == True:
                self.monitor = True
                self.dirlist[site] = [dir] + [tsc]

            for file in os.listdir(dir):
                self.addImportFile(os.path.join(dir, file), site, tsc)
        else:
            log.warning(_("Attempted to add non-directory '%s' as an import directory") % str(dir))

    def addImportFileOrDir(self, inputPath, site = "PokerStars"):
        tsc = self.config.hhcs[site].summaryImporter
        if os.path.isdir(inputPath):
            for subdir in os.walk(inputPath):
                for file in subdir[2]:
                    self.addImportFile(unicode(os.path.join(subdir[0], file),'utf-8'),
                                       site=site, tsc=tsc)
        else:
            self.addImportFile(unicode(inputPath,'utf-8'), site=site, tsc=tsc)
        pass

    def runImport(self):
        start = datetime.datetime.now()
        starttime = time()
        log.info(_("Tourney Summary Import started at %s - %d files to import.") % (start, len(self.filelist)))

        total_errors = 0
        total_imported = 0
        for f in self.filelist:
            (site, tsc) = self.filelist[f]
            imported, errs = self.importFile(f, tsc, site)
            total_errors += errs
            total_imported += imported
        return (total_imported, total_errors)
            

    def runUpdated(self):
        pass

    def importFile(self, filename, tsc = "PokerStarsSummary", site = "PokerStars"):
        mod = __import__(tsc)
        obj = getattr(mod, tsc, None)
        if callable(obj):
            foabs = self.readFile(obj, filename)
            summaryTexts = re.split(obj.re_SplitTourneys, foabs)

            # The summary files tend to have a header or footer
            # Remove the first and/or last entry if it has < 100 characters
            if len(summaryTexts[-1]) <= 100:
                summaryTexts.pop()
                log.warn(_("TourneyImport: Removing text < 100 characters from end of file"))

            if len(summaryTexts[0]) <= 130:
                del summaryTexts[0]
                log.warn(_("TourneyImport: Removing text < 100 characters from start of file"))

            print "Found %s summaries" %(len(summaryTexts))
            errors = 0
            imported = 0
            ####Lock Placeholder####
            for j, summaryText in enumerate(summaryTexts, start=1):
                sc, gsc = {'bk': []}, {'bk': []}
                doinsert = len(summaryTexts)==j
                try:
                    conv = obj(db=self.database, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP")
                    sc, gsc = conv.updateSessionsCache(sc, gsc, None, doinsert)
                except FpdbParseError, e:
                    errors += 1
                print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts))
                imported = j
            ####Lock Placeholder####
        return (imported - errors, errors)

    def clearFileList(self):
        self.updatedsize = {}
        self.updatetime = {}
        self.filelist = {}

    def readFile(self, tsc, filename):
        codepage = ["utf8", "utf16"]
        whole_file = None
        tsc.codepage

        for kodec in codepage:
            try:
                in_fh = codecs.open(filename, 'r', kodec)
                whole_file = in_fh.read()
                in_fh.close()
                break
            except UnicodeDecodeError, e:
                log.warn(_("GTI.readFile: '%s'") % e)
                pass

        return whole_file

def main(argv=None):
    """main can also be called in the python interpreter, by supplying the command line as the argument."""
    import SQL
    if argv is None:
        argv = sys.argv[1:]

    (options, argv) = Options.fpdb_options()

    if options.usage == True:
        #Print usage examples and exit
        print _("USAGE:")
        sys.exit(0)

    if options.hhc == "PokerStarsToFpdb":
        print _("Need to define a converter")
        exit(0)

    config = Configuration.Config("HUD_config.test.xml")
    sql = SQL.Sql(db_server = 'sqlite')

    if options.filename == None:
        print _("Need a filename to import")
        exit(0)

    importer = SummaryImporter(config, sql, None)

    importer.addImportFileOrDir(options.filename, site = options.hhc)
    starttime = time()
    (stored, errs) = importer.runImport()

    ttime = time() - starttime
    if ttime == 0:
        ttime = 1
    print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\
                     % (stored, errs, ttime, (stored+0.0) / ttime)
    importer.clearFileList()

    


if __name__  == '__main__':
    sys.exit(main())