Merge branch 'master' of git://git.assembla.com/fpdboz

Conflicts:
	pyfpdb/Configuration.py

pyfpdb/Configuration.py

@@ -142,6 +142,7 @@ log.info("DIR SELF = %s" % DIR_SELF)
 log.info("DIR CONFIG = %s" % DIR_CONFIG)
 log.info("DIR DATABASE = %s" % DIR_DATABASE)
 log.info("DIR LOG = %s" % DIR_LOG)
+NEWIMPORT = False
 
 ########################################################################
 def string_to_bool(string, default=True):
@@ -705,7 +706,7 @@ class Config:
         except: hui['agg_bb_mult']    = 1
 
         try:    hui['seats_style']    = self.ui.seats_style
-        except: hui['seats_style']    = 'C'  # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
+        except: hui['seats_style']    = 'A'  # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
 
         try:    hui['seats_cust_nums']    = self.ui.seats_cust_nums
         except: hui['seats_cust_nums']    = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]
@@ -728,7 +729,7 @@ class Config:
         except: hui['h_agg_bb_mult']    = 1
 
         try:    hui['h_seats_style']    = self.ui.h_seats_style
-        except: hui['h_seats_style']    = 'E'  # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
+        except: hui['h_seats_style']    = 'A'  # A / C / E, use A(ll) / C(ustom) / E(xact) seat numbers
 
         try:    hui['h_seats_cust_nums']    = self.ui.h_seats_cust_nums
         except: hui['h_seats_cust_nums']    = ['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]

pyfpdb/Database.py

@@ -21,6 +21,11 @@ Create and manage the database objects.
 
 ########################################################################
 
+# ToDo:  - rebuild indexes / vacuum option
+#        - check speed of get_stats_from_hand() - add log info
+#        - check size of db, seems big? (mysql)
+#        - investigate size of mysql db (200K for just 7K hands? 2GB for 140K hands?)
+
 # postmaster -D /var/lib/pgsql/data
 
 #    Standard Library modules
@@ -70,14 +75,9 @@ class Database:
     indexes = [
                 [ ] # no db with index 0
               , [ ] # no db with index 1
-              , [ # indexes for mysql (list index 2)
+              , [ # indexes for mysql (list index 2) (foreign keys not here, in next data structure)
                 #  {'tab':'Players',         'col':'name',              'drop':0}  unique indexes not dropped
                 #  {'tab':'Hands',           'col':'siteHandNo',        'drop':0}  unique indexes not dropped
-                  {'tab':'Hands',           'col':'gametypeId',        'drop':0} # mct 22/3/09
-                , {'tab':'HandsPlayers',    'col':'handId',            'drop':0} # not needed, handled by fk
-                , {'tab':'HandsPlayers',    'col':'playerId',          'drop':0} # not needed, handled by fk
-                , {'tab':'HandsPlayers',    'col':'tourneyTypeId',     'drop':0}
-                , {'tab':'HandsPlayers',    'col':'tourneysPlayersId', 'drop':0}
                 #, {'tab':'Tourneys',        'col':'siteTourneyNo',     'drop':0}  unique indexes not dropped
                 ]
               , [ # indexes for postgres (list index 3)
@@ -118,6 +118,8 @@ class Database:
                       {'fktab':'Hands',        'fkcol':'gametypeId',    'rtab':'Gametypes',     'rcol':'id', 'drop':1}
                     , {'fktab':'HandsPlayers', 'fkcol':'handId',        'rtab':'Hands',         'rcol':'id', 'drop':1}
                     , {'fktab':'HandsPlayers', 'fkcol':'playerId',      'rtab':'Players',       'rcol':'id', 'drop':1}
+                    , {'fktab':'HandsPlayers', 'fkcol':'tourneyTypeId', 'rtab':'TourneyTypes',  'rcol':'id', 'drop':1}
+                    , {'fktab':'HandsPlayers', 'fkcol':'tourneysPlayersId','rtab':'TourneysPlayers','rcol':'id', 'drop':1}
                     , {'fktab':'HandsActions', 'fkcol':'handsPlayerId', 'rtab':'HandsPlayers',  'rcol':'id', 'drop':1}
                     , {'fktab':'HudCache',     'fkcol':'gametypeId',    'rtab':'Gametypes',     'rcol':'id', 'drop':1}
                     , {'fktab':'HudCache',     'fkcol':'playerId',      'rtab':'Players',       'rcol':'id', 'drop':0}
@@ -431,6 +433,7 @@ class Database:
             err = traceback.extract_tb(sys.exc_info()[2])[-1]
             print "*** Database Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1])
 
+    # is get_stats_from_hand slow?
     def get_stats_from_hand( self, hand, type   # type is "ring" or "tour"
                            , hud_params = {'hud_style':'A', 'agg_bb_mult':1000
                                           ,'seats_style':'A', 'seats_cust_nums':['n/a', 'n/a', (2,2), (3,4), (3,5), (4,6), (5,7), (6,8), (7,9), (8,10), (8,10)]
@@ -1165,9 +1168,9 @@ class Database:
                 print "dropping mysql index ", idx['tab'], idx['col']
                 try:
                     self.get_cursor().execute( "alter table %s drop index %s"
-                                             , (idx['tab'],idx['col']) )
+                                             , (idx['tab'], idx['col']) )
                 except:
-                    pass
+                    print "    drop idx failed: " + str(sys.exc_info())
             elif self.backend == self.PGSQL:
                 print "dropping pg index ", idx['tab'], idx['col']
                 # mod to use tab_col for index name?
@@ -1175,7 +1178,7 @@ class Database:
                     self.get_cursor().execute( "drop index %s_%s_idx"
                                                % (idx['tab'],idx['col']) )
                 except:
-                    pass
+                    print "    drop idx failed: " + str(sys.exc_info())
             else:
                 print "Only MySQL and Postgres supported so far"
                 return -1
@@ -1183,6 +1186,112 @@ class Database:
             self.connection.set_isolation_level(1)   # go back to normal isolation level
     #end def dropAllIndexes
 
+    def createAllForeignKeys(self):
+        """Create foreign keys"""
+
+        try:
+            if self.backend == self.PGSQL:
+                self.connection.set_isolation_level(0)   # allow table/index operations to work
+            c = self.get_cursor()
+        except:
+            print "    set_isolation_level failed: " + str(sys.exc_info())
+
+        for fk in self.foreignKeys[self.backend]:
+            if self.backend == self.MYSQL_INNODB:
+                c.execute("SELECT constraint_name " +
+                          "FROM information_schema.KEY_COLUMN_USAGE " +
+                          #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
+                          "WHERE 1=1 " +
+                          "AND table_name = %s AND column_name = %s " + 
+                          "AND referenced_table_name = %s " +
+                          "AND referenced_column_name = %s ",
+                          (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
+                cons = c.fetchone()
+                #print "afterbulk: cons=", cons
+                if cons:
+                    pass
+                else:
+                    print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
+                    try:
+                        c.execute("alter table " + fk['fktab'] + " add foreign key (" 
+                                  + fk['fkcol'] + ") references " + fk['rtab'] + "(" 
+                                  + fk['rcol'] + ")")
+                    except:
+                        print "    create fk failed: " + str(sys.exc_info())
+            elif self.backend == self.PGSQL:
+                print "creating fk ", fk['fktab'], fk['fkcol'], "->", fk['rtab'], fk['rcol']
+                try:
+                    c.execute("alter table " + fk['fktab'] + " add constraint "
+                              + fk['fktab'] + '_' + fk['fkcol'] + '_fkey'
+                              + " foreign key (" + fk['fkcol']
+                              + ") references " + fk['rtab'] + "(" + fk['rcol'] + ")")
+                except:
+                    print "   create fk failed: " + str(sys.exc_info())
+            else:
+                print "Only MySQL and Postgres supported so far"
+
+        try:
+            if self.backend == self.PGSQL:
+                self.connection.set_isolation_level(1)   # go back to normal isolation level
+        except:
+            print "    set_isolation_level failed: " + str(sys.exc_info())
+    #end def createAllForeignKeys
+
+    def dropAllForeignKeys(self):
+        """Drop all standalone indexes (i.e. not including primary keys or foreign keys)
+           using list of indexes in indexes data structure"""
+        # maybe upgrade to use data dictionary?? (but take care to exclude PK and FK)
+        if self.backend == self.PGSQL:
+            self.connection.set_isolation_level(0)   # allow table/index operations to work
+        c = self.get_cursor()
+
+        for fk in self.foreignKeys[self.backend]:
+            if self.backend == self.MYSQL_INNODB:
+                c.execute("SELECT constraint_name " +
+                          "FROM information_schema.KEY_COLUMN_USAGE " +
+                          #"WHERE REFERENCED_TABLE_SCHEMA = 'fpdb'
+                          "WHERE 1=1 " +
+                          "AND table_name = %s AND column_name = %s " + 
+                          "AND referenced_table_name = %s " +
+                          "AND referenced_column_name = %s ",
+                          (fk['fktab'], fk['fkcol'], fk['rtab'], fk['rcol']) )
+                cons = c.fetchone()
+                #print "preparebulk find fk: cons=", cons
+                if cons:
+                    print "dropping mysql fk", cons[0], fk['fktab'], fk['fkcol']
+                    try:
+                        c.execute("alter table " + fk['fktab'] + " drop foreign key " + cons[0])
+                    except:
+                        print "    drop failed: " + str(sys.exc_info())
+            elif self.backend == self.PGSQL:
+#    DON'T FORGET TO RECREATE THEM!!
+                print "dropping pg fk", fk['fktab'], fk['fkcol']
+                try:
+                    # try to lock table to see if index drop will work:
+                    # hmmm, tested by commenting out rollback in grapher. lock seems to work but 
+                    # then drop still hangs :-(  does work in some tests though??
+                    # will leave code here for now pending further tests/enhancement ...
+                    c.execute( "lock table %s in exclusive mode nowait" % (fk['fktab'],) )
+                    #print "after lock, status:", c.statusmessage
+                    #print "alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol'])
+                    try:
+                        c.execute("alter table %s drop constraint %s_%s_fkey" % (fk['fktab'], fk['fktab'], fk['fkcol']))
+                        print "dropped pg fk pg fk %s_%s_fkey, continuing ..." % (fk['fktab'], fk['fkcol'])
+                    except:
+                        if "does not exist" not in str(sys.exc_value):
+                            print "warning: drop pg fk %s_%s_fkey failed: %s, continuing ..." \
+                                  % (fk['fktab'], fk['fkcol'], str(sys.exc_value).rstrip('\n') )
+                except:
+                    print "warning: constraint %s_%s_fkey not dropped: %s, continuing ..." \
+                          % (fk['fktab'],fk['fkcol'], str(sys.exc_value).rstrip('\n'))
+            else:
+                print "Only MySQL and Postgres supported so far"
+        
+        if self.backend == self.PGSQL:
+            self.connection.set_isolation_level(1)   # go back to normal isolation level
+    #end def dropAllForeignKeys
+
+    
     def fillDefaultData(self):
         c = self.get_cursor() 
         c.execute("INSERT INTO Settings (version) VALUES (118);")
@@ -1209,6 +1318,12 @@ class Database:
 
     #end def fillDefaultData
 
+    def rebuild_indexes(self, start=None):
+        self.dropAllIndexes()
+        self.createAllIndexes()
+        self.dropAllForeignKeys()
+        self.createAllForeignKeys()
+
     def rebuild_hudcache(self, start=None):
         """clears hudcache and rebuilds from the individual handsplayers records"""
 
@@ -1291,7 +1406,7 @@ class Database:
             except:
                 print "Error during analyze:", str(sys.exc_value)
         elif self.backend == self.PGSQL:
-            self.connection.set_isolation_level(0)   # allow vacuum to work
+            self.connection.set_isolation_level(0)   # allow analyze to work
             try:
                 self.get_cursor().execute(self.sql.query['analyze'])
             except:
@@ -1302,6 +1417,25 @@ class Database:
         print "Analyze took %.1f seconds" % (atime,)
     #end def analyzeDB
 
+    def vacuumDB(self):
+        """Do whatever the DB can offer to update index/table statistics"""
+        stime = time()
+        if self.backend == self.MYSQL_INNODB:
+            try:
+                self.get_cursor().execute(self.sql.query['vacuum'])
+            except:
+                print "Error during vacuum:", str(sys.exc_value)
+        elif self.backend == self.PGSQL:
+            self.connection.set_isolation_level(0)   # allow vacuum to work
+            try:
+                self.get_cursor().execute(self.sql.query['vacuum'])
+            except:
+                print "Error during vacuum:", str(sys.exc_value)
+            self.connection.set_isolation_level(1)   # go back to normal isolation level
+        self.commit()
+        atime = time() - stime
+        print "Vacuum took %.1f seconds" % (atime,)
+    #end def analyzeDB
 
 # Start of Hand Writing routines. Idea is to provide a mixture of routines to store Hand data
 # however the calling prog requires. Main aims:
@@ -1383,7 +1517,7 @@ class Database:
 
         q = q.replace('%s', self.sql.query['placeholder'])
 
-        c = self.connection.cursor()
+        c = self.get_cursor()
 
         c.execute(q, (
                 p['tableName'], 
@@ -1569,7 +1703,7 @@ class Database:
 
         #print "DEBUG: inserts: %s" %inserts
         #print "DEBUG: q: %s" % q
-        c = self.connection.cursor()
+        c = self.get_cursor()
         c.executemany(q, inserts)
 
     def storeHudCacheNew(self, gid, pid, hc):

pyfpdb/DerivedStats.py

@@ -143,28 +143,11 @@ class DerivedStats():
 
         self.calcCBets(hand)
 
-        #default_holecards = ["Xx", "Xx", "Xx", "Xx"]
+        for player in hand.players:
-        #if hand.gametype['base'] == "hold":
+            hcs = hand.join_holecards(player[1], asList=True)
-        #    pass
+            hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x']
-        #elif hand.gametype['base'] == "stud":
+            for i, card in enumerate(hcs[:7], 1):
-        #    pass
+                self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card)
-        #else:
-        #    # Flop hopefully...
-        #    pass
-
-        for street in hand.holeStreets:
-            for player in hand.players:
-                for i in range(1,8): self.handsplayers[player[1]]['card%d' % i] = 0
-                #print "DEBUG: hand.holecards[%s]: %s" % (street, hand.holecards[street])
-                if player[1] in hand.holecards[street].keys() and hand.gametype['base'] == "hold":
-                    self.handsplayers[player[1]]['card1'] = Card.encodeCard(hand.holecards[street][player[1]][1][0])
-                    self.handsplayers[player[1]]['card2'] = Card.encodeCard(hand.holecards[street][player[1]][1][1])
-                    try:
-                        self.handsplayers[player[1]]['card3'] = Card.encodeCard(hand.holecards[street][player[1]][1][2])
-                        self.handsplayers[player[1]]['card4'] = Card.encodeCard(hand.holecards[street][player[1]][1][3])
-                    except IndexError:
-                        # Just means no player cards for that street/game - continue
-                        pass 
 
     def assembleHudCache(self, hand):
         pass

pyfpdb/Filters.py

@@ -312,8 +312,8 @@ class Filters(threading.Thread):
                     self.cbAllLimits.set_active(False)
             if not self.limits[limit]:
                 if limit.isdigit():
-                    if self.cbFl is not None:
+                    if self.cbFL is not None:
-                        self.cbFl.set_active(False)
+                        self.cbFL.set_active(False)
                 else:
                     if self.cbNL is not None:
                         self.cbNL.set_active(False)
@@ -329,8 +329,10 @@ class Filters(threading.Thread):
             if self.limits[limit]:
                 for cb in self.cbLimits.values():
                     cb.set_active(False)
-                self.cbNL.set_active(False)
+                if self.cbNL is not None:
-                self.cbFL.set_active(False)
+                    self.cbNL.set_active(False)
+                if self.cbFL is not None:
+                    self.cbFL.set_active(False)
         elif limit == "fl":
             if not self.limits[limit]:
                 # only toggle all fl limits off if they are all currently on

pyfpdb/GuiAutoImport.py

@@ -71,22 +71,24 @@ class GuiAutoImport (threading.Thread):
 
         self.intervalLabel = gtk.Label("Time between imports in seconds:")
         self.intervalLabel.set_alignment(xalign=1.0, yalign=0.5)
-        vbox1.pack_start(self.intervalLabel, True, True, 0)
+        vbox1.pack_start(self.intervalLabel, False, True, 0)
 
         hbox = gtk.HBox(False, 0)
-        vbox2.pack_start(hbox, True, True, 0)
+        vbox2.pack_start(hbox, False, True, 0)
         self.intervalEntry = gtk.Entry()
         self.intervalEntry.set_text(str(self.config.get_import_parameters().get("interval")))
         hbox.pack_start(self.intervalEntry, False, False, 0)
         lbl1 = gtk.Label()
-        hbox.pack_start(lbl1, expand=True, fill=True)
+        hbox.pack_start(lbl1, expand=False, fill=True)
 
         lbl = gtk.Label('')
-        vbox1.pack_start(lbl, expand=True, fill=True)
+        vbox1.pack_start(lbl, expand=False, fill=True)
         lbl = gtk.Label('')
-        vbox2.pack_start(lbl, expand=True, fill=True)
+        vbox2.pack_start(lbl, expand=False, fill=True)
 
         self.addSites(vbox1, vbox2)
+        self.textbuffer = gtk.TextBuffer()
+        self.textview = gtk.TextView(self.textbuffer)
 
         hbox = gtk.HBox(False, 0)
         self.mainVBox.pack_start(hbox, expand=True, padding=3)
@@ -102,13 +104,27 @@ class GuiAutoImport (threading.Thread):
         self.startButton.connect("clicked", self.startClicked, "start clicked")
         hbox.pack_start(self.startButton, expand=False, fill=False)
 
+
         lbl2 = gtk.Label()
         hbox.pack_start(lbl2, expand=True, fill=False)
 
         hbox = gtk.HBox(False, 0)
         hbox.show()
+
         self.mainVBox.pack_start(hbox, expand=True, padding=3)
+
+        scrolledwindow = gtk.ScrolledWindow()
+        scrolledwindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
+        self.mainVBox.pack_end(scrolledwindow, expand=True)
+        scrolledwindow.add(self.textview)
+
         self.mainVBox.show_all()
+        self.addText("AutoImport Ready.")
+
+    def addText(self, text):
+        end_iter = self.textbuffer.get_end_iter()
+        self.textbuffer.insert(end_iter, text)
+        self.textview.scroll_to_mark(self.textbuffer.get_insert(), 0)
 
 
     #end of GuiAutoImport.__init__
@@ -139,8 +155,9 @@ class GuiAutoImport (threading.Thread):
         if self.doAutoImportBool:
             self.startButton.set_label(u' I M P O R T I N G  ')
             self.importer.runUpdated()
-            sys.stdout.write(".")
+            self.addText(".")
-            sys.stdout.flush()
+            #sys.stdout.write(".")
+            #sys.stdout.flush()
             gobject.timeout_add(1000, self.reset_startbutton)
             return True
         return False
@@ -172,7 +189,7 @@ class GuiAutoImport (threading.Thread):
             # - Ideally we want to release the lock if the auto-import is killed by some
             # kind of exception - is this possible?
             if self.settings['global_lock'].acquire(False):   # returns false immediately if lock not acquired
-                print "\nGlobal lock taken ..."
+                self.addText("\nGlobal lock taken ... Auto Import Started.\n")
                 self.doAutoImportBool = True
                 widget.set_label(u'  _Stop Autoimport  ')
                 if self.pipe_to_hud is None:
@@ -190,12 +207,11 @@ class GuiAutoImport (threading.Thread):
                                                             universal_newlines=True)
                     except:
                         err = traceback.extract_tb(sys.exc_info()[2])[-1]
-                        print "*** GuiAutoImport Error opening pipe: " + err[2] + "(" + str(err[1]) + "): " + str(sys.exc_info()[1])
+                        self.addText( "\n*** GuiAutoImport Error opening pipe: " + err[2] + "(" + str(err[1]) + "): " + str(sys.exc_info()[1]))
                     else:
                         for site in self.input_settings:
                             self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1])
-                            print " * Add", site, " import directory", str(self.input_settings[site][0])
+                            self.addText("\n * Add "+ site+ " import directory "+ str(self.input_settings[site][0]))
-                            print "+Import directory - Site: " + site + " dir: " + str(self.input_settings[site][0])
                             self.do_import()
                             interval = int(self.intervalEntry.get_text())
                     if self.importtimer != 0:
@@ -203,14 +219,14 @@ class GuiAutoImport (threading.Thread):
                     self.importtimer = gobject.timeout_add(interval * 1000, self.do_import)
 
             else:
-                print "auto-import aborted - global lock not available"
+                self.addText("\nauto-import aborted - global lock not available")
         else: # toggled off
             gobject.source_remove(self.importtimer)
             self.settings['global_lock'].release()
             self.doAutoImportBool = False # do_import will return this and stop the gobject callback timer
-            print "Stopping autoimport - global lock released."
+            self.addText("\nStopping autoimport - global lock released.")
             if self.pipe_to_hud.poll() is not None:
-                print " * Stop Autoimport: HUD already terminated"
+                self.addText("\n * Stop Autoimport: HUD already terminated")
             else:
                 #print >>self.pipe_to_hud.stdin, "\n"
                 self.pipe_to_hud.communicate('\n') # waits for process to terminate

pyfpdb/HUD_main.py

@@ -112,10 +112,7 @@ class HUD_main(object):
         def idle_func():
 
             gtk.gdk.threads_enter()
-            try: # TODO: seriously need to decrease the scope of this block.. what are we expecting to error?
+            try:
-                 # TODO: The purpose of this try/finally block is to make darn sure that threads_leave()
-                 # TODO: gets called. If there is an exception and threads_leave() doesn't get called we
-                 # TODO: lock up.  REB
                 table.gdkhandle = gtk.gdk.window_foreign_new(table.number)
                 newlabel = gtk.Label("%s - %s" % (table.site, table_name))
                 self.vb.add(newlabel)
@@ -129,9 +126,12 @@ class HUD_main(object):
                     m.update_gui(new_hand_id)
                 self.hud_dict[table_name].update(new_hand_id, self.config)
                 self.hud_dict[table_name].reposition_windows()
+            except:
+                print "*** Exception in HUD_main::idle_func() *** "
+                traceback.print_stack()
             finally:
                 gtk.gdk.threads_leave()
-            return False
+                return False
 
         self.hud_dict[table_name] = Hud.Hud(self, table, max, poker_game, self.config, self.db_connection)
         self.hud_dict[table_name].table_name = table_name
@@ -150,11 +150,11 @@ class HUD_main(object):
             self.hud_dict[table_name].hud_params['h_agg_bb_mult'] = 1
         # sqlcoder: I forget why these are set to true (aren't they ignored from now on?)
         # but I think it's needed:
-        self.hud_params['aggregate_ring'] == True
+        self.hud_params['aggregate_ring'] = True
-        self.hud_params['h_aggregate_ring'] == True
+        self.hud_params['h_aggregate_ring'] = True
         # so maybe the tour ones should be set as well? does this fix the bug I see mentioned?
         self.hud_params['aggregate_tour'] = True
-        self.hud_params['h_aggregate_tour'] == True
+        self.hud_params['h_aggregate_tour'] = True
 
         [aw.update_data(new_hand_id, self.db_connection) for aw in self.hud_dict[table_name].aux_windows]
         gobject.idle_add(idle_func)
@@ -175,7 +175,7 @@ class HUD_main(object):
                 pass
             finally:
                 gtk.gdk.threads_leave()
-            return False
+                return False
 
         gobject.idle_add(idle_func)
 
@@ -222,11 +222,10 @@ class HUD_main(object):
 #        Update an existing HUD
             if temp_key in self.hud_dict:
                 # get stats using hud's specific params and get cards
-                self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days']
-                                                     , self.hud_dict[temp_key].hud_params['h_hud_days'])
-                stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params
-                                                                  ,self.hero_ids[site_id], num_seats)
                 try:
+                    self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days']
+                                                     , self.hud_dict[temp_key].hud_params['h_hud_days'])
+                    stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params, self.hero_ids[site_id])
                     self.hud_dict[temp_key].stat_dict = stat_dict
                 except KeyError:    # HUD instance has been killed off, key is stale
                     sys.stderr.write('hud_dict[%s] was not found\n' % temp_key)
@@ -270,7 +269,7 @@ class HUD_main(object):
                     if hasattr(tablewindow, 'number'):
                         self.create_HUD(new_hand_id, tablewindow, temp_key, max, poker_game, type, stat_dict, cards)
                     else:
-                        sys.stderr.write('Table "%s" no longer exists\n', table_name)
+                        sys.stderr.write('Table "%s" no longer exists\n' % table_name)
 
             self.db_connection.connection.rollback()
 

pyfpdb/Hand.py

@@ -668,6 +668,27 @@ class HoldemOmahaHand(Hand):
         tmp5 = 0
         return (tmp1,tmp2,tmp3,tmp4,tmp5)
 
+    def join_holecards(self, player, asList=False):
+        """With asList = True it returns the set cards for a player including down cards if they aren't know"""
+        # FIXME: This should actually return
+        hcs = [u'0x', u'0x', u'0x', u'0x']
+
+        for street in self.holeStreets:
+            if player in self.holecards[street].keys():
+                hcs[0] = self.holecards[street][player][1][0]
+                hcs[1] = self.holecards[street][player][1][1]
+                try:
+                    hcs[2] = self.holecards[street][player][1][2]
+                    hcs[3] = self.holecards[street][player][1][3]
+                except IndexError:
+                    pass
+
+        if asList == False:
+            return " ".join(hcs)
+        else:
+            return hcs
+
+
     def writeHTMLHand(self):
         from nevow import tags as T
         from nevow import flat
@@ -968,6 +989,16 @@ class DrawHand(Hand):
         # showdownPot INT,                 /* pot size at sd/street7 */
         return (0,0,0,0,0)
 
+    def join_holecards(self, player, asList=False):
+        """With asList = True it returns the set cards for a player including down cards if they aren't know"""
+        # FIXME: This should actually return
+        holecards = [u'0x', u'0x', u'0x', u'0x', u'0x']
+
+        if asList == False:
+            return " ".join(holecards)
+        else:
+            return holecards
+
 
     def writeHand(self, fh=sys.__stdout__):
         # PokerStars format.
@@ -1294,7 +1325,9 @@ Add a complete on [street] by [player] to [amountTo]
         if street == 'SEVENTH' and player != self.hero: return # only write 7th st line for hero, LDO
         return hc + " ".join(self.holecards[street][player][1]) + "] [" + " ".join(self.holecards[street][player][0]) + "]"
 
-    def join_holecards(self, player):
+    def join_holecards(self, player, asList=False):
+        """Function returns a string for the stud writeHand method by default
+           With asList = True it returns the set cards for a player including down cards if they aren't know"""
         holecards = []
         for street in self.holeStreets:
             if self.holecards[street].has_key(player):
@@ -1307,7 +1340,20 @@ Add a complete on [street] by [player] to [amountTo]
                         holecards = holecards + self.holecards[street][player][1]
                 else:
                     holecards = holecards + self.holecards[street][player][0]
-        return " ".join(holecards)
+
+        if asList == False:
+            return " ".join(holecards)
+        else:
+            if player == self.hero or len(holecards) == 7:
+                return holecards
+            elif len(holecards) <= 4:
+                #Non hero folded before showdown, add first two downcards
+                holecards = [u'0x', u'0x'] + holecards
+            else:
+                log.warning("join_holecards: # of holecards should be either < 4, 4 or 7 - 5 and 6 should be impossible for anyone who is not a hero")
+                log.warning("join_holcards: holecards(%s): %s" %(player, holecards))
+            return holecards
+
 
 class Pot(object):
 

pyfpdb/HandHistoryConverter.py

@@ -284,10 +284,8 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py.
             log.info("Unsupported game type: %s" % gametype)
 
         if hand:
-#    uncomment these to calculate some stats
+            if Configuration.NEWIMPORT == False:
-#            print hand
+                hand.writeHand(self.out_fh)
-#            hand.stats.getStats(hand)
-            hand.writeHand(self.out_fh)
             return hand
         else:
             log.info("Unsupported game type: %s" % gametype)

pyfpdb/Hud.py

@@ -482,7 +482,7 @@ class Hud:
             try:
                 # throws "invalid window handle" in WinXP (sometimes?)
                 s.window.destroy()
-            except:
+            except: # TODO: what exception?
                 pass
         self.stat_windows = {}
 #    also kill any aux windows

pyfpdb/Options.py

@@ -35,6 +35,9 @@ def fpdb_options():
     parser.add_option("-r", "--rerunPython",
                       action="store_true",
                       help="Indicates program was restarted with a different path (only allowed once).")
+    parser.add_option("-i", "--infile",
+                      dest="config", default=None,
+                      help="Input file")
     (options, argv) = parser.parse_args()
     return (options, argv)
 

pyfpdb/SQL.py

@@ -461,7 +461,7 @@ class Sql:
                         totalProfit INT,
                         comment text,
                         commentTs DATETIME,
-                        tourneysPlayersId BIGINT UNSIGNED,
+                        tourneysPlayersId BIGINT UNSIGNED, FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id),
                         tourneyTypeId SMALLINT UNSIGNED NOT NULL DEFAULT 1, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
 
                         wonWhenSeenStreet1 FLOAT,
@@ -551,9 +551,7 @@ class Sql:
                         street3Raises TINYINT,
                         street4Raises TINYINT,
 
-                        actionString VARCHAR(15),
+                        actionString VARCHAR(15))
-
-                        FOREIGN KEY (tourneysPlayersId) REFERENCES TourneysPlayers(id))
                         ENGINE=INNODB"""
         elif db_server == 'postgresql':
             self.query['createHandsPlayersTable'] = """CREATE TABLE HandsPlayers (
@@ -3100,8 +3098,10 @@ class Sql:
             analyze table Autorates, GameTypes, Hands, HandsPlayers, HudCache, Players
                         , Settings, Sites, Tourneys, TourneysPlayers, TourneyTypes
             """
-        else:  # assume postgres
+        elif db_server == 'postgresql':
-            self.query['analyze'] = "vacuum analyze"
+            self.query['analyze'] = "analyze"
+        elif db_server == 'sqlite':
+            self.query['analyze'] = "analyze"
 
         if db_server == 'mysql':
             self.query['lockForInsert'] = """
@@ -3109,8 +3109,20 @@ class Sql:
                           , HudCache write, GameTypes write, Sites write, Tourneys write
                           , TourneysPlayers write, TourneyTypes write, Autorates write
                 """
-        else:  # assume postgres
+        elif db_server == 'postgresql':
             self.query['lockForInsert'] = ""
+        elif db_server == 'sqlite':
+            self.query['lockForInsert'] = ""
+
+        if db_server == 'mysql':
+            self.query['vacuum'] = """optimize table Hands, HandsPlayers, HandsActions, Players
+                                                   , HudCache, GameTypes, Sites, Tourneys
+                                                   , TourneysPlayers, TourneyTypes, Autorates
+                                      """
+        elif db_server == 'postgresql':
+            self.query['vacuum'] = """ vacuum """
+        elif db_server == 'sqlite':
+            self.query['vacuum'] = """ vacuum """
 
         self.query['getGametypeFL'] = """SELECT id
                                            FROM Gametypes

pyfpdb/fpdb.py

@@ -159,10 +159,10 @@ class fpdb:
     def add_icon_to_button(self, button):
         iconBox = gtk.HBox(False, 0)        
         image = gtk.Image()
-        image.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_MENU)
+        image.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_SMALL_TOOLBAR)
         gtk.Button.set_relief(button, gtk.RELIEF_NONE)
         settings = gtk.Widget.get_settings(button);
-        (w,h) = gtk.icon_size_lookup_for_settings(settings, gtk.ICON_SIZE_MENU);
+        (w,h) = gtk.icon_size_lookup_for_settings(settings, gtk.ICON_SIZE_SMALL_TOOLBAR);
         gtk.Widget.set_size_request (button, w + 4, h + 4);
         image.show()
         iconBox.pack_start(image, True, False, 0)
@@ -357,6 +357,27 @@ class fpdb:
 
         self.release_global_lock()
 
+    def dia_rebuild_indexes(self, widget, data=None):
+        if self.obtain_global_lock():
+            self.dia_confirm = gtk.MessageDialog(parent=None
+                                                ,flags=0
+                                                ,type=gtk.MESSAGE_WARNING
+                                                ,buttons=(gtk.BUTTONS_YES_NO)
+                                                ,message_format="Confirm rebuilding database indexes")
+            diastring = "Please confirm that you want to rebuild the database indexes."
+            self.dia_confirm.format_secondary_text(diastring)
+
+            response = self.dia_confirm.run()
+            self.dia_confirm.destroy()
+            if response == gtk.RESPONSE_YES:
+                self.db.rebuild_indexes()
+                self.db.vacuumDB()
+                self.db.analyzeDB()
+            elif response == gtk.RESPONSE_NO:
+                print 'User cancelled rebuilding db indexes'
+
+        self.release_global_lock()
+
     def __calendar_dialog(self, widget, entry):
         self.dia_confirm.set_modal(False)
         d = gtk.Window(gtk.WINDOW_TOPLEVEL)
@@ -451,6 +472,7 @@ class fpdb:
                   <menuitem action="createuser"/>
                   <menuitem action="createtabs"/>
                   <menuitem action="rebuildhudcache"/>
+                  <menuitem action="rebuildindexes"/>
                   <menuitem action="stats"/>
                 </menu>
                 <menu action="help">
@@ -492,6 +514,7 @@ class fpdb:
                                  ('createuser', None, 'Create or Delete _User (todo)', None, 'Create or Delete User', self.dia_create_del_user),
                                  ('createtabs', None, 'Create or Recreate _Tables', None, 'Create or Recreate Tables ', self.dia_recreate_tables),
                                  ('rebuildhudcache', None, 'Rebuild HUD Cache', None, 'Rebuild HUD Cache', self.dia_recreate_hudcache),
+                                 ('rebuildindexes', None, 'Rebuild DB Indexes', None, 'Rebuild DB Indexes', self.dia_rebuild_indexes),
                                  ('stats', None, '_Statistics (todo)', None, 'View Database Statistics', self.dia_database_stats),
                                  ('help', None, '_Help'),
                                  ('Abbrev', None, '_Abbrevations (todo)', None, 'List of Abbrevations', self.tab_abbreviations),

pyfpdb/fpdb_import.py

@@ -99,7 +99,7 @@ class Importer:
         for i in xrange(self.settings['threads']):
             self.writerdbs.append( Database.Database(self.config, sql = self.sql) )
 
-        self.NEWIMPORT = False
+        self.NEWIMPORT = Configuration.NEWIMPORT
 
     #Set functions
     def setCallHud(self, value):
@@ -357,6 +357,11 @@ class Importer:
                 if file in self.updatedsize: # we should be able to assume that if we're in size, we're in time as well
                     if stat_info.st_size > self.updatedsize[file] or stat_info.st_mtime > self.updatedtime[file]:
 #                        print "file",counter," updated", os.path.basename(file), stat_info.st_size, self.updatedsize[file], stat_info.st_mtime, self.updatedtime[file]
+                        try:
+                            if not os.path.isdir(file):
+                                self.caller.addText("\n"+file)
+                        except KeyError: # TODO: What error happens here?
+                            pass
                         self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None)
                         self.updatedsize[file] = stat_info.st_size
                         self.updatedtime[file] = time()

pyfpdb/test_PokerStars.py

@@ -5,10 +5,12 @@ import py
 
 import Configuration
 import Database
+import SQL
 import fpdb_import
 
 config = Configuration.Config(file = "HUD_config.test.xml")
 db = Database.Database(config)
+sql = SQL.Sql(db_server = 'sqlite')
 
 settings = {}
 settings.update(config.get_db_parameters())
@@ -95,16 +97,29 @@ def testStudImport():
     (stored, dups, partial, errs, ttime) = importer.runImport()
     importer.clearFileList()
 
+    # Should actually do some testing here
+    assert 1 == 1
+
 def testDrawImport():
-    db.recreate_tables()
+    try:
-    importer = fpdb_import.Importer(False, settings, config)
+        db.recreate_tables()
-    importer.setDropIndexes("don't drop")
+        importer = fpdb_import.Importer(False, settings, config)
-    importer.setFailOnError(True)
+        importer.setDropIndexes("don't drop")
-    importer.setThreads(-1)
+        importer.setFailOnError(True)
-    importer.addBulkImportImportFileOrDir(
+        importer.setThreads(-1)
-            """regression-test-files/cash/Stars/Draw/3-Draw-Limit-USD-0.10-0.20-200911.txt""", site="PokerStars")
+        importer.addBulkImportImportFileOrDir(
-    importer.addBulkImportImportFileOrDir(
+                """regression-test-files/cash/Stars/Draw/3-Draw-Limit-USD-0.10-0.20-200911.txt""", site="PokerStars")
-            """regression-test-files/cash/Stars/Draw/5-Carddraw-USD-0.10-0.20-200911.txt""", site="PokerStars")
+        importer.addBulkImportImportFileOrDir(
-    importer.setCallHud(False)
+                """regression-test-files/cash/Stars/Draw/5-Carddraw-USD-0.10-0.20-200911.txt""", site="PokerStars")
-    (stored, dups, partial, errs, ttime) = importer.runImport()
+        importer.setCallHud(False)
-    importer.clearFileList()
+        (stored, dups, partial, errs, ttime) = importer.runImport()
+        importer.clearFileList()
+    except FpdbError:
+        if Configuration.NEWIMPORT == False:
+            #Old import code doesn't support draw
+            pass
+        else:
+            assert 0 == 1
+
+    # Should actually do some testing here
+    assert 1 == 1