Merge branch 'master' into siteneutral

Conflicts:

	pyfpdb/fpdb_import.py
	pyfpdb/fpdb_save_to_db.py

pyfpdb/Configuration.py

@@ -32,12 +32,12 @@ import shutil
 import xml.dom.minidom
 from xml.dom.minidom import Node
 
-def fix_tf(x):
+def fix_tf(x, default = True):
     if x == "1" or x == 1 or string.lower(x) == "true"  or string.lower(x) == "t":
         return True
     if x == "0" or x == 0 or string.lower(x) == "false" or string.lower(x) == "f":
         return False
-    return False
+    return default
 
 class Layout:
     def __init__(self, node):
@@ -212,18 +212,12 @@ class Import:
         self.interval      = node.getAttribute("interval")
         self.callFpdbHud   = node.getAttribute("callFpdbHud")
         self.hhArchiveBase = node.getAttribute("hhArchiveBase")
-        if node.hasAttribute("saveActions"):
+        self.saveActions = fix_tf(node.getAttribute("saveActions"), True)
-            self.saveActions = fix_tf(node.getAttribute("saveActions"))
+        self.fastStoreHudCache = fix_tf(node.getAttribute("fastStoreHudCache"), True)
-        else:
-            self.saveActions = True
-        if node.hasAttribute("fastStoreHudCache"):
-            self.fastStoreHudCache = fix_tf(node.getAttribute("fastStoreHudCache"))
-        else:
-            self.fastStoreHudCache = False
 
     def __str__(self):
         return "    interval = %s\n    callFpdbHud = %s\n    hhArchiveBase = %s\n    saveActions = %s\n    fastStoreHudCache = %s\n" \
-             % (self.interval, self.callFpdbHud, self.hhArchiveBase, self.saveActions, self.saveActions)
+             % (self.interval, self.callFpdbHud, self.hhArchiveBase, self.saveActions, self.fastStoreHudCache)
 
 class Tv:
     def __init__(self, node):

pyfpdb/FulltiltToFpdb.py

@@ -113,10 +113,19 @@ follow :  whether to tail -f the input"""
 
     def readHandInfo(self, hand):
         m =  self.re_HandInfo.search(hand.handText,re.DOTALL)
-        #print m.groups()
+
+        if(m == None):
+            logging.info("Didn't match re_HandInfo")
+            logging.info(hand.handText)
+            return None
+        
         hand.handid = m.group('HID')
         hand.tablename = m.group('TABLE')
         hand.starttime = time.strptime(m.group('DATETIME'), "%H:%M:%S ET - %Y/%m/%d")
+        hand.maxseats = 8 # assume 8-max until we see otherwise
+        if m.group('TABLEATTRIBUTES'):
+            m2 = re.search("(\d+) max", m.group('TABLEATTRIBUTES'))
+            hand.maxseats = int(m2.group(1))
 # These work, but the info is already in the Hand class - should be used for tourneys though.
 #       m.group('SB')
 #       m.group('BB')

pyfpdb/GuiAutoImport.py

@@ -45,7 +45,7 @@ class GuiAutoImport (threading.Thread):
 
         self.importer = fpdb_import.Importer(self,self.settings, self.config)
         self.importer.setCallHud(True)
-        self.importer.setMinPrint(30)
+        self.importer.setMinPrint(settings['minPrint'])
         self.importer.setQuiet(False)
         self.importer.setFailOnError(False)
         self.importer.setHandCount(0)
@@ -227,13 +227,15 @@ if __name__== "__main__":
 
     parser = OptionParser()
     parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui")
-
+    parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int",
+                    help="How often to print a one-line status report (0 (default) means never)")
     (options, sys.argv) = parser.parse_args()
 
     config = Configuration.Config()
 #    db = fpdb_db.fpdb_db()
 
     settings = {}
+    settings['minPrint'] = options.minPrint
     if os.name == 'nt': settings['os'] = 'windows'
     else:               settings['os'] = 'linuxmac'
 

pyfpdb/GuiBulkImport.py

@@ -60,8 +60,8 @@ class GuiBulkImport():
         self.importer.setFailOnError(self.chk_fail.get_active())
         self.importer.setThreads(int(self.spin_threads.get_text()))
         self.importer.setHandsInDB(self.n_hands_in_db)
-        cb_model = self.cb.get_model()
+        cb_model = self.cb_dropindexes.get_model()
-        cb_index = self.cb.get_active()
+        cb_index = self.cb_dropindexes.get_active()
         if cb_index:
             self.importer.setDropIndexes(cb_model[cb_index][0])
         else:
@@ -159,13 +159,13 @@ class GuiBulkImport():
         self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT)
 
 #    ComboBox - drop indexes
-        self.cb = gtk.combo_box_new_text()
+        self.cb_dropindexes = gtk.combo_box_new_text()
-        self.cb.append_text('auto')
+        self.cb_dropindexes.append_text('auto')
-        self.cb.append_text("don't drop")
+        self.cb_dropindexes.append_text("don't drop")
-        self.cb.append_text('drop')
+        self.cb_dropindexes.append_text('drop')
-        self.cb.set_active(0)
+        self.cb_dropindexes.set_active(0)
-        self.table.attach(self.cb, 4, 5, 1, 2, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK)
+        self.table.attach(self.cb_dropindexes, 4, 5, 1, 2, xpadding = 10, ypadding = 0, yoptions=gtk.SHRINK)
-        self.cb.show()
+        self.cb_dropindexes.show()
 
 #    label - filter
         self.lab_filter = gtk.Label("Site filter:")
@@ -195,31 +195,45 @@ class GuiBulkImport():
 
 #    see how many hands are in the db and adjust accordingly
         tcursor = db.db.cursor()
-        tcursor.execute("Select max(id) from Hands;")
+        tcursor.execute("Select count(1) from Hands;")
         row = tcursor.fetchone()
         tcursor.close()
         self.n_hands_in_db = row[0]
         if self.n_hands_in_db == 0:
-            self.cb.set_active(2)
+            self.cb_dropindexes.set_active(2)
-            self.cb.set_sensitive(False)
+            self.cb_dropindexes.set_sensitive(False)
             self.lab_drop.set_sensitive(False)
 
-if __name__ == '__main__':
+def main(argv=None):
-
+    """main can also be called in the python interpreter, by supplying the command line as the argument.
+>>>import GuiBulkImport
+>>>GuiBulkImport.main("-f ~/data/hands")"""
+    if argv is None:
+        argv = sys.argv[1:]
+    else:
+        argv = argv.split(" ")
 
     def destroy(*args):  # call back for terminating the main eventloop
         gtk.main_quit()
 
     parser = OptionParser()
-    parser.add_option("-f", "--file", dest="filename", help="Input file in quiet mode", metavar="FILE")
+    parser.add_option("-f", "--file", dest="filename", metavar="FILE", default=None,
-    parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True, help="don't start gui")
+                    help="Input file in quiet mode")
-    parser.add_option("-x", "--convert", dest="filtername", help="Conversion filter", default="passthrough")
+    parser.add_option("-q", "--quiet", action="store_false", dest="gui", default=True,
-    (options, sys.argv) = parser.parse_args()
+                    help="don't start gui; deprecated (just give a filename with -f).")
+    parser.add_option("-c", "--convert", dest="filtername", default="passthrough", metavar="FILTER",
+                    help="Conversion filter (*passthrough, FullTiltToFpdb, PokerStarsToFpdb, EverleafToFpdb)")
+    parser.add_option("-x", "--failOnError", action="store_true", default=False,
+                    help="If this option is passed it quits when it encounters any error")
+    parser.add_option("-m", "--minPrint", "--status", dest="minPrint", default="0", type="int",
+                    help="How often to print a one-line status report (0 (default) means never)")
+    (options, sys.argv) = parser.parse_args(args = argv)
 
     config = Configuration.Config()
     db = fpdb_db.fpdb_db()
 
     settings = {}
+    settings['minPrint'] = options.minPrint
     if os.name == 'nt': settings['os'] = 'windows'
     else:               settings['os'] = 'linuxmac'
 
@@ -228,7 +242,10 @@ if __name__ == '__main__':
     settings.update(config.get_import_parameters())
     settings.update(config.get_default_paths())
 
-    if(options.gui == True):
+    if not options.gui:
+        print """-q is deprecated. Just use "-f filename" instead"""
+        # This is because -q on its own causes an error, so -f is necessary and sufficient for cmd line use
+    if not options.filename:
         i = GuiBulkImport(db, settings, config)
         main_window = gtk.Window()
         main_window.connect('destroy', destroy)
@@ -239,8 +256,13 @@ if __name__ == '__main__':
         #Do something useful
         importer = fpdb_import.Importer(False,settings, config) 
         importer.setDropIndexes("auto")
-        importer.setFailOnError(True)
+        importer.setFailOnError(options.failOnError)
-        importer.addImportFile(options.filename, filter=options.filtername)
+        importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), filter=options.filtername)
         importer.setCallHud(False)
         importer.runImport()
         importer.clearFileList()
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+

pyfpdb/fpdb_import.py

@@ -128,6 +128,7 @@ class Importer:
     # Called from GuiBulkImport to add a file or directory.
     def addBulkImportImportFileOrDir(self, inputPath,filter = "passthrough"):
         """Add a file or directory for bulk import"""
+        
         # Bulk import never monitors
         # if directory, add all files in it. Otherwise add single file.
         # TODO: only add sane files?
@@ -137,7 +138,6 @@ class Importer:
                     self.addImportFile(os.path.join(inputPath, subdir[0], file), site="default", filter=filter)
         else:
             self.addImportFile(inputPath, site="default", filter=filter)
-
     #Add a directory of files to filelist
     #Only one import directory per site supported.
     #dirlist is a hash of lists:
@@ -185,7 +185,15 @@ class Importer:
 
     def calculate_auto(self):
         """An heuristic to determine a reasonable value of drop/don't drop"""
-        if len(self.filelist) == 1:            return "don't drop"      
+        if len(self.filelist) == 1:            return "don't drop"
+        if 'handsInDB' not in self.settings:
+            try:
+                tmpcursor = self.fdb.db.cursor()
+                tmpcursor.execute("Select count(1) from Hands;")
+                self.settings['handsInDB'] = tmpcursor.fetchone()[0]
+                tmpcursor.close()
+            except:
+                pass # if this fails we're probably doomed anyway
         if self.settings['handsInDB'] < 5000:  return "drop"
         if len(self.filelist) < 50:            return "don't drop"      
         if self.settings['handsInDB'] > 50000: return "don't drop"
@@ -329,7 +337,6 @@ class Importer:
                     isTourney=fpdb_simple.isTourney(hand[0])
                     if not isTourney:
                         fpdb_simple.filterAnteBlindFold(hand)
-                    hand=fpdb_simple.filterCrap(hand, isTourney)
                     self.hand=hand
 
                     try:

pyfpdb/fpdb_parse_logic.py

@@ -60,6 +60,8 @@ def mainParser(backend, db, cursor, siteID, category, hand, config):
 
     fpdb_simple.isAlreadyInDB(cursor, gametypeID, siteHandNo)
     
+    hand=fpdb_simple.filterCrap(site, hand, isTourney)
+    
     #part 2: classify lines by type (e.g. cards, action, win, sectionchange) and street
     fpdb_simple.classifyLines(hand, category, lineTypes, lineStreets)
         

pyfpdb/fpdb_save_to_db.py

@@ -33,7 +33,6 @@ saveActions = True    # set this to False to avoid storing action data
                     # Cons: no action data is saved, so you need to keep the hand histories
                     #       variance not available on stats page
                     #     : No graphs
- 
 #stores a stud/razz hand into the database
 def ring_stud(config, backend, db, cursor, base, category, site_hand_no, gametype_id, hand_start_time
              ,names, player_ids, start_cashes, antes, card_values, card_suits, winnings, rakes