merge from carl

pyfpdb/Configuration.py

@@ -125,6 +125,8 @@ DATABASE_TYPES = (
         DATABASE_TYPE_MYSQL,
         )
 
+NEWIMPORT = False
+
 ########################################################################
 def string_to_bool(string, default=True):
     """converts a string representation of a boolean value to boolean True or False

pyfpdb/DerivedStats.py

@@ -143,28 +143,11 @@ class DerivedStats():
 
         self.calcCBets(hand)
 
-        #default_holecards = ["Xx", "Xx", "Xx", "Xx"]
+        for player in hand.players:
-        #if hand.gametype['base'] == "hold":
+            hcs = hand.join_holecards(player[1], asList=True)
-        #    pass
+            hcs = hcs + [u'0x', u'0x', u'0x', u'0x', u'0x']
-        #elif hand.gametype['base'] == "stud":
+            for i, card in enumerate(hcs[:7], 1):
-        #    pass
+                self.handsplayers[player[1]]['card%s' % i] = Card.encodeCard(card)
-        #else:
-        #    # Flop hopefully...
-        #    pass
-
-        for street in hand.holeStreets:
-            for player in hand.players:
-                for i in range(1,8): self.handsplayers[player[1]]['card%d' % i] = 0
-                #print "DEBUG: hand.holecards[%s]: %s" % (street, hand.holecards[street])
-                if player[1] in hand.holecards[street].keys() and hand.gametype['base'] == "hold":
-                    self.handsplayers[player[1]]['card1'] = Card.encodeCard(hand.holecards[street][player[1]][1][0])
-                    self.handsplayers[player[1]]['card2'] = Card.encodeCard(hand.holecards[street][player[1]][1][1])
-                    try:
-                        self.handsplayers[player[1]]['card3'] = Card.encodeCard(hand.holecards[street][player[1]][1][2])
-                        self.handsplayers[player[1]]['card4'] = Card.encodeCard(hand.holecards[street][player[1]][1][3])
-                    except IndexError:
-                        # Just means no player cards for that street/game - continue
-                        pass 
 
     def assembleHudCache(self, hand):
         pass

pyfpdb/Filters.py

@@ -312,8 +312,8 @@ class Filters(threading.Thread):
                     self.cbAllLimits.set_active(False)
             if not self.limits[limit]:
                 if limit.isdigit():
-                    if self.cbFl is not None:
+                    if self.cbFL is not None:
-                        self.cbFl.set_active(False)
+                        self.cbFL.set_active(False)
                 else:
                     if self.cbNL is not None:
                         self.cbNL.set_active(False)
@@ -329,8 +329,10 @@ class Filters(threading.Thread):
             if self.limits[limit]:
                 for cb in self.cbLimits.values():
                     cb.set_active(False)
-                self.cbNL.set_active(False)
+                if self.cbNL is not None:
-                self.cbFL.set_active(False)
+                    self.cbNL.set_active(False)
+                if self.cbFL is not None:
+                    self.cbFL.set_active(False)
         elif limit == "fl":
             if not self.limits[limit]:
                 # only toggle all fl limits off if they are all currently on

pyfpdb/GuiAutoImport.py

@@ -71,22 +71,24 @@ class GuiAutoImport (threading.Thread):
 
         self.intervalLabel = gtk.Label("Time between imports in seconds:")
         self.intervalLabel.set_alignment(xalign=1.0, yalign=0.5)
-        vbox1.pack_start(self.intervalLabel, True, True, 0)
+        vbox1.pack_start(self.intervalLabel, False, True, 0)
 
         hbox = gtk.HBox(False, 0)
-        vbox2.pack_start(hbox, True, True, 0)
+        vbox2.pack_start(hbox, False, True, 0)
         self.intervalEntry = gtk.Entry()
         self.intervalEntry.set_text(str(self.config.get_import_parameters().get("interval")))
         hbox.pack_start(self.intervalEntry, False, False, 0)
         lbl1 = gtk.Label()
-        hbox.pack_start(lbl1, expand=True, fill=True)
+        hbox.pack_start(lbl1, expand=False, fill=True)
 
         lbl = gtk.Label('')
-        vbox1.pack_start(lbl, expand=True, fill=True)
+        vbox1.pack_start(lbl, expand=False, fill=True)
         lbl = gtk.Label('')
-        vbox2.pack_start(lbl, expand=True, fill=True)
+        vbox2.pack_start(lbl, expand=False, fill=True)
 
         self.addSites(vbox1, vbox2)
+        self.textbuffer = gtk.TextBuffer()
+        self.textview = gtk.TextView(self.textbuffer)
 
         hbox = gtk.HBox(False, 0)
         self.mainVBox.pack_start(hbox, expand=True, padding=3)
@@ -102,13 +104,27 @@ class GuiAutoImport (threading.Thread):
         self.startButton.connect("clicked", self.startClicked, "start clicked")
         hbox.pack_start(self.startButton, expand=False, fill=False)
 
+
         lbl2 = gtk.Label()
         hbox.pack_start(lbl2, expand=True, fill=False)
 
         hbox = gtk.HBox(False, 0)
         hbox.show()
+
         self.mainVBox.pack_start(hbox, expand=True, padding=3)
+
+        scrolledwindow = gtk.ScrolledWindow()
+        scrolledwindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
+        self.mainVBox.pack_end(scrolledwindow, expand=True)
+        scrolledwindow.add(self.textview)
+
         self.mainVBox.show_all()
+        self.addText("AutoImport Ready.")
+
+    def addText(self, text):
+        end_iter = self.textbuffer.get_end_iter()
+        self.textbuffer.insert(end_iter, text)
+        self.textview.scroll_to_mark(self.textbuffer.get_insert(), 0)
 
 
     #end of GuiAutoImport.__init__
@@ -139,8 +155,9 @@ class GuiAutoImport (threading.Thread):
         if self.doAutoImportBool:
             self.startButton.set_label(u' I M P O R T I N G  ')
             self.importer.runUpdated()
-            sys.stdout.write(".")
+            self.addText(".")
-            sys.stdout.flush()
+            #sys.stdout.write(".")
+            #sys.stdout.flush()
             gobject.timeout_add(1000, self.reset_startbutton)
             return True
         return False
@@ -172,7 +189,7 @@ class GuiAutoImport (threading.Thread):
             # - Ideally we want to release the lock if the auto-import is killed by some
             # kind of exception - is this possible?
             if self.settings['global_lock'].acquire(False):   # returns false immediately if lock not acquired
-                print "\nGlobal lock taken ..."
+                self.addText("\nGlobal lock taken ... Auto Import Started.\n")
                 self.doAutoImportBool = True
                 widget.set_label(u'  _Stop Autoimport  ')
                 if self.pipe_to_hud is None:
@@ -190,12 +207,11 @@ class GuiAutoImport (threading.Thread):
                                                             universal_newlines=True)
                     except:
                         err = traceback.extract_tb(sys.exc_info()[2])[-1]
-                        print "*** GuiAutoImport Error opening pipe: " + err[2] + "(" + str(err[1]) + "): " + str(sys.exc_info()[1])
+                        self.addText( "\n*** GuiAutoImport Error opening pipe: " + err[2] + "(" + str(err[1]) + "): " + str(sys.exc_info()[1]))
                     else:
                         for site in self.input_settings:
                             self.importer.addImportDirectory(self.input_settings[site][0], True, site, self.input_settings[site][1])
-                            print " * Add", site, " import directory", str(self.input_settings[site][0])
+                            self.addText("\n * Add "+ site+ " import directory "+ str(self.input_settings[site][0]))
-                            print "+Import directory - Site: " + site + " dir: " + str(self.input_settings[site][0])
                             self.do_import()
                             interval = int(self.intervalEntry.get_text())
                     if self.importtimer != 0:
@@ -203,14 +219,14 @@ class GuiAutoImport (threading.Thread):
                     self.importtimer = gobject.timeout_add(interval * 1000, self.do_import)
 
             else:
-                print "auto-import aborted - global lock not available"
+                self.addText("\nauto-import aborted - global lock not available")
         else: # toggled off
             gobject.source_remove(self.importtimer)
             self.settings['global_lock'].release()
             self.doAutoImportBool = False # do_import will return this and stop the gobject callback timer
-            print "Stopping autoimport - global lock released."
+            self.addText("\nStopping autoimport - global lock released.")
             if self.pipe_to_hud.poll() is not None:
-                print " * Stop Autoimport: HUD already terminated"
+                self.addText("\n * Stop Autoimport: HUD already terminated")
             else:
                 #print >>self.pipe_to_hud.stdin, "\n"
                 self.pipe_to_hud.communicate('\n') # waits for process to terminate

pyfpdb/GuiGraphViewer.py

@@ -31,6 +31,7 @@ try:
     from matplotlib.figure import Figure
     from matplotlib.backends.backend_gtk import FigureCanvasGTK as FigureCanvas
     from matplotlib.backends.backend_gtkagg import NavigationToolbar2GTKAgg as NavigationToolbar
+    from matplotlib.font_manager import FontProperties
     from numpy import arange, cumsum
     from pylab import *
 except ImportError, inst:
@@ -170,7 +171,7 @@ class GuiGraphViewer (threading.Thread):
 
             #Get graph data from DB
             starttime = time()
-            line = self.getRingProfitGraph(playerids, sitenos, limits)
+            (green, blue, red) = self.getRingProfitGraph(playerids, sitenos, limits)
             print "Graph generated in: %s" %(time() - starttime)
 
             self.ax.set_title("Profit graph for ring games")
@@ -179,22 +180,24 @@ class GuiGraphViewer (threading.Thread):
             self.ax.set_xlabel("Hands", fontsize = 12)
             self.ax.set_ylabel("$", fontsize = 12)
             self.ax.grid(color='g', linestyle=':', linewidth=0.2)
-            if line == None or line == []:
+            if green == None or green == []:
 
                 #TODO: Do something useful like alert user
                 print "No hands returned by graph query"
             else:
-    #            text = "All Hands, " + sitename + str(name) + "\nProfit: $" + str(line[-1]) + "\nTotal Hands: " + str(len(line))
+                #text = "Profit: $%.2f\nTotal Hands: %d" %(green[-1], len(green))
-                text = "All Hands, " + "\nProfit: $" + str(line[-1]) + "\nTotal Hands: " + str(len(line))
+                #self.ax.annotate(text,
-
+                #                 xy=(10, -10),
-                self.ax.annotate(text,
+                #                 xycoords='axes points',
-                                 xy=(10, -10),
+                #                 horizontalalignment='left', verticalalignment='top',
-                                 xycoords='axes points',
+                #                 fontsize=10)
-                                 horizontalalignment='left', verticalalignment='top',
-                                 fontsize=10)
 
                 #Draw plot
-                self.ax.plot(line,)
+                self.ax.plot(green, color='green', label='Hands: %d\nProfit: $%.2f' %(len(green), green[-1]))
+                self.ax.plot(blue, color='blue', label='Showdown: $%.2f' %(blue[-1]))
+                self.ax.plot(red, color='red', label='Non-showdown: $%.2f' %(red[-1]))
+                self.ax.legend(loc='best', fancybox=True, shadow=True, prop=FontProperties(size='smaller'))
+
 
                 self.graphBox.add(self.canvas)
                 self.canvas.show()
@@ -270,9 +273,13 @@ class GuiGraphViewer (threading.Thread):
         if winnings == ():
             return None
 
-        y = map(lambda x:float(x[1]), winnings)
+        green = map(lambda x:float(x[1]), winnings)
-        line = cumsum(y)
+        blue  = map(lambda x: float(x[1]) if x[2] == True  else 0.0, winnings)
-        return line/100
+        red   = map(lambda x: float(x[1]) if x[2] == False else 0.0, winnings)
+        greenline = cumsum(green)
+        blueline  = cumsum(blue)
+        redline   = cumsum(red)
+        return (greenline/100, blueline/100, redline/100)
         #end of def getRingProfitGraph
 
     def exportGraph (self, widget, data):

pyfpdb/HUD_main.py

@@ -108,10 +108,7 @@ class HUD_main(object):
         def idle_func():
 
             gtk.gdk.threads_enter()
-            try: # TODO: seriously need to decrease the scope of this block.. what are we expecting to error?
+            try:
-                 # TODO: The purpose of this try/finally block is to make darn sure that threads_leave()
-                 # TODO: gets called. If there is an exception and threads_leave() doesn't get called we
-                 # TODO: lock up.  REB
                 table.gdkhandle = gtk.gdk.window_foreign_new(table.number)
                 newlabel = gtk.Label("%s - %s" % (table.site, table_name))
                 self.vb.add(newlabel)
@@ -125,9 +122,12 @@ class HUD_main(object):
                     m.update_gui(new_hand_id)
                 self.hud_dict[table_name].update(new_hand_id, self.config)
                 self.hud_dict[table_name].reposition_windows()
+            except:
+                print "*** Exception in HUD_main::idle_func() *** "
+                traceback.print_stack()
             finally:
                 gtk.gdk.threads_leave()
-            return False
+                return False
 
         self.hud_dict[table_name] = Hud.Hud(self, table, max, poker_game, self.config, self.db_connection)
         self.hud_dict[table_name].table_name = table_name
@@ -146,11 +146,11 @@ class HUD_main(object):
             self.hud_dict[table_name].hud_params['h_agg_bb_mult'] = 1
         # sqlcoder: I forget why these are set to true (aren't they ignored from now on?)
         # but I think it's needed:
-        self.hud_params['aggregate_ring'] == True
+        self.hud_params['aggregate_ring'] = True
-        self.hud_params['h_aggregate_ring'] == True
+        self.hud_params['h_aggregate_ring'] = True
         # so maybe the tour ones should be set as well? does this fix the bug I see mentioned?
         self.hud_params['aggregate_tour'] = True
-        self.hud_params['h_aggregate_tour'] == True
+        self.hud_params['h_aggregate_tour'] = True
 
         [aw.update_data(new_hand_id, self.db_connection) for aw in self.hud_dict[table_name].aux_windows]
         gobject.idle_add(idle_func)
@@ -171,7 +171,7 @@ class HUD_main(object):
                 pass
             finally:
                 gtk.gdk.threads_leave()
-            return False
+                return False
 
         gobject.idle_add(idle_func)
 
@@ -227,6 +227,9 @@ class HUD_main(object):
                                                                   ,self.hero_ids[site_id], num_seats)
                 t3 = time()
                 try:
+                    self.db_connection.init_hud_stat_vars( self.hud_dict[temp_key].hud_params['hud_days']
+                                                     , self.hud_dict[temp_key].hud_params['h_hud_days'])
+                    stat_dict = self.db_connection.get_stats_from_hand(new_hand_id, type, self.hud_dict[temp_key].hud_params, self.hero_ids[site_id])
                     self.hud_dict[temp_key].stat_dict = stat_dict
                 except KeyError:    # HUD instance has been killed off, key is stale
                     sys.stderr.write('hud_dict[%s] was not found\n' % temp_key)
@@ -271,7 +274,7 @@ class HUD_main(object):
                     if hasattr(tablewindow, 'number'):
                         self.create_HUD(new_hand_id, tablewindow, temp_key, max, poker_game, type, stat_dict, cards)
                     else:
-                        sys.stderr.write('Table "%s" no longer exists\n', table_name)
+                        sys.stderr.write('Table "%s" no longer exists\n' % table_name)
 
             t6 = time()
             log.info("HUD_main.read_stdin: hand read in %4.3f seconds (%4.3f,%4.3f,%4.3f,%4.3f,%4.3f,%4.3f,%4.3f)"

pyfpdb/Hand.py

@@ -668,6 +668,27 @@ class HoldemOmahaHand(Hand):
         tmp5 = 0
         return (tmp1,tmp2,tmp3,tmp4,tmp5)
 
+    def join_holecards(self, player, asList=False):
+        """With asList = True it returns the set cards for a player including down cards if they aren't know"""
+        # FIXME: This should actually return
+        hcs = [u'0x', u'0x', u'0x', u'0x']
+
+        for street in self.holeStreets:
+            if player in self.holecards[street].keys():
+                hcs[0] = self.holecards[street][player][1][0]
+                hcs[1] = self.holecards[street][player][1][1]
+                try:
+                    hcs[2] = self.holecards[street][player][1][2]
+                    hcs[3] = self.holecards[street][player][1][3]
+                except IndexError:
+                    pass
+
+        if asList == False:
+            return " ".join(hcs)
+        else:
+            return hcs
+
+
     def writeHTMLHand(self):
         from nevow import tags as T
         from nevow import flat
@@ -968,6 +989,16 @@ class DrawHand(Hand):
         # showdownPot INT,                 /* pot size at sd/street7 */
         return (0,0,0,0,0)
 
+    def join_holecards(self, player, asList=False):
+        """With asList = True it returns the set cards for a player including down cards if they aren't know"""
+        # FIXME: This should actually return
+        holecards = [u'0x', u'0x', u'0x', u'0x', u'0x']
+
+        if asList == False:
+            return " ".join(holecards)
+        else:
+            return holecards
+
 
     def writeHand(self, fh=sys.__stdout__):
         # PokerStars format.
@@ -1294,7 +1325,9 @@ Add a complete on [street] by [player] to [amountTo]
         if street == 'SEVENTH' and player != self.hero: return # only write 7th st line for hero, LDO
         return hc + " ".join(self.holecards[street][player][1]) + "] [" + " ".join(self.holecards[street][player][0]) + "]"
 
-    def join_holecards(self, player):
+    def join_holecards(self, player, asList=False):
+        """Function returns a string for the stud writeHand method by default
+           With asList = True it returns the set cards for a player including down cards if they aren't know"""
         holecards = []
         for street in self.holeStreets:
             if self.holecards[street].has_key(player):
@@ -1307,7 +1340,20 @@ Add a complete on [street] by [player] to [amountTo]
                         holecards = holecards + self.holecards[street][player][1]
                 else:
                     holecards = holecards + self.holecards[street][player][0]
-        return " ".join(holecards)
+
+        if asList == False:
+            return " ".join(holecards)
+        else:
+            if player == self.hero or len(holecards) == 7:
+                return holecards
+            elif len(holecards) <= 4:
+                #Non hero folded before showdown, add first two downcards
+                holecards = [u'0x', u'0x'] + holecards
+            else:
+                log.warning("join_holecards: # of holecards should be either < 4, 4 or 7 - 5 and 6 should be impossible for anyone who is not a hero")
+                log.warning("join_holcards: holecards(%s): %s" %(player, holecards))
+            return holecards
+
 
 class Pot(object):
 

pyfpdb/HandHistoryConverter.py

@@ -284,10 +284,8 @@ which it expects to find at self.re_TailSplitHands -- see for e.g. Everleaf.py.
             log.info("Unsupported game type: %s" % gametype)
 
         if hand:
-#    uncomment these to calculate some stats
+            if Configuration.NEWIMPORT == False:
-#            print hand
+                hand.writeHand(self.out_fh)
-#            hand.stats.getStats(hand)
-            hand.writeHand(self.out_fh)
             return hand
         else:
             log.info("Unsupported game type: %s" % gametype)

pyfpdb/Hud.py

@@ -482,7 +482,7 @@ class Hud:
             try:
                 # throws "invalid window handle" in WinXP (sometimes?)
                 s.window.destroy()
-            except:
+            except: # TODO: what exception?
                 pass
         self.stat_windows = {}
 #    also kill any aux windows

pyfpdb/Options.py

@@ -35,6 +35,9 @@ def fpdb_options():
     parser.add_option("-r", "--rerunPython",
                       action="store_true",
                       help="Indicates program was restarted with a different path (only allowed once).")
+    parser.add_option("-i", "--infile",
+                      dest="config", default=None,
+                      help="Input file")
     (options, argv) = parser.parse_args()
     return (options, argv)
 

pyfpdb/SQL.py

@@ -2561,7 +2561,7 @@ class Sql:
         #    self.query['playerStatsByPosition'] = """ """
 
         self.query['getRingProfitAllHandsPlayerIdSite'] = """
-            SELECT hp.handId, hp.totalProfit
+            SELECT hp.handId, hp.totalProfit, hp.sawShowdown
             FROM HandsPlayers hp
             INNER JOIN Players pl      ON  (pl.id = hp.playerId)
             INNER JOIN Hands h         ON  (h.id  = hp.handId)

pyfpdb/fpdb_import.py

@@ -99,7 +99,7 @@ class Importer:
         for i in xrange(self.settings['threads']):
             self.writerdbs.append( Database.Database(self.config, sql = self.sql) )
 
-        self.NEWIMPORT = False
+        self.NEWIMPORT = Configuration.NEWIMPORT
 
     #Set functions
     def setCallHud(self, value):
@@ -357,6 +357,11 @@ class Importer:
                 if file in self.updatedsize: # we should be able to assume that if we're in size, we're in time as well
                     if stat_info.st_size > self.updatedsize[file] or stat_info.st_mtime > self.updatedtime[file]:
 #                        print "file",counter," updated", os.path.basename(file), stat_info.st_size, self.updatedsize[file], stat_info.st_mtime, self.updatedtime[file]
+                        try:
+                            if not os.path.isdir(file):
+                                self.caller.addText("\n"+file)
+                        except KeyError: # TODO: What error happens here?
+                            pass
                         self.import_file_dict(self.database, file, self.filelist[file][0], self.filelist[file][1], None)
                         self.updatedsize[file] = stat_info.st_size
                         self.updatedtime[file] = time()

pyfpdb/test_PokerStars.py

@@ -5,10 +5,12 @@ import py
 
 import Configuration
 import Database
+import SQL
 import fpdb_import
 
 config = Configuration.Config(file = "HUD_config.test.xml")
 db = Database.Database(config)
+sql = SQL.Sql(db_server = 'sqlite')
 
 settings = {}
 settings.update(config.get_db_parameters())
@@ -95,16 +97,29 @@ def testStudImport():
     (stored, dups, partial, errs, ttime) = importer.runImport()
     importer.clearFileList()
 
+    # Should actually do some testing here
+    assert 1 == 1
+
 def testDrawImport():
-    db.recreate_tables()
+    try:
-    importer = fpdb_import.Importer(False, settings, config)
+        db.recreate_tables()
-    importer.setDropIndexes("don't drop")
+        importer = fpdb_import.Importer(False, settings, config)
-    importer.setFailOnError(True)
+        importer.setDropIndexes("don't drop")
-    importer.setThreads(-1)
+        importer.setFailOnError(True)
-    importer.addBulkImportImportFileOrDir(
+        importer.setThreads(-1)
-            """regression-test-files/cash/Stars/Draw/3-Draw-Limit-USD-0.10-0.20-200911.txt""", site="PokerStars")
+        importer.addBulkImportImportFileOrDir(
-    importer.addBulkImportImportFileOrDir(
+                """regression-test-files/cash/Stars/Draw/3-Draw-Limit-USD-0.10-0.20-200911.txt""", site="PokerStars")
-            """regression-test-files/cash/Stars/Draw/5-Carddraw-USD-0.10-0.20-200911.txt""", site="PokerStars")
+        importer.addBulkImportImportFileOrDir(
-    importer.setCallHud(False)
+                """regression-test-files/cash/Stars/Draw/5-Carddraw-USD-0.10-0.20-200911.txt""", site="PokerStars")
-    (stored, dups, partial, errs, ttime) = importer.runImport()
+        importer.setCallHud(False)
-    importer.clearFileList()
+        (stored, dups, partial, errs, ttime) = importer.runImport()
+        importer.clearFileList()
+    except FpdbError:
+        if Configuration.NEWIMPORT == False:
+            #Old import code doesn't support draw
+            pass
+        else:
+            assert 0 == 1
+
+    # Should actually do some testing here
+    assert 1 == 1