work in progress!

- command line options
  Try the following
  ./Everleaf --help
  cat everleaf.txt | ./Everleaf -i - -o -
  Try using -i -, and pasting in a hand. Be aware it needs an EOF
<ctrl>-d (unless you specify --follow, which isn't done yet)
- HHC is a thread
- hand gametypes per hand
- lots of stupid little things just to cause trouble
- added logging

Not yet done:
- tail -f mode
- integration into fpdb_import. Two options for grand plan here:
  1)  recently modified files get a HHC filter attached
      when the HHC runs out of pipe it waits
      runUpdated finds modified files and notifies the thread
  2)  the thread follows independently, when it runs out of input it
sleeps and then tries again at intervals
I like both, 1) involves bigger changes to fpdb_import whilst with 2) we
just point the existing fpdb_import code at the filter output (the way
we currently do filtering has to be altered though)

Comments welcome of course.

pyfpdb/EverleafToFpdb.py

@@ -18,9 +18,8 @@
 ########################################################################
 
 import sys
-import Configuration
 from HandHistoryConverter import *
-from time import strftime
+
 
 # Class for converting Everleaf HH format.
 
@@ -29,27 +28,27 @@ class Everleaf(HandHistoryConverter):
     # Static regexes
     re_SplitHands  = re.compile(r"\n\n+")
     re_GameInfo    = re.compile(r".*Blinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<LTYPE>(NL|PL)) (?P<GAME>(Hold\'em|Omaha|7 Card Stud))")
-    re_HandInfo    = re.compile(r".*#(?P<HID>[0-9]+)\n.*\nBlinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<GAMETYPE>.*) - (?P<DATETIME>\d\d\d\d/\d\d/\d\d - \d\d:\d\d:\d\d)\nTable (?P<TABLE>[- a-zA-Z]+)")
+    re_HandInfo    = re.compile(r".*#(?P<HID>[0-9]+)\n.*\nBlinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<GAMETYPE>.*) - (?P<DATETIME>\d\d\d\d/\d\d/\d\d - \d\d:\d\d:\d\d)\nTable (?P<TABLE>[- a-zA-Z]+)", re.MULTILINE)
     re_Button      = re.compile(r"^Seat (?P<BUTTON>\d+) is the button", re.MULTILINE)
     re_PlayerInfo  = re.compile(r"^Seat (?P<SEAT>[0-9]+): (?P<PNAME>.*) \(\s+(\$ (?P<CASH>[.0-9]+) USD|new player|All-in) \)", re.MULTILINE)
     re_Board       = re.compile(r"\[ (?P<CARDS>.+) \]")
-        
+    splitstring    = "\n\n\n"
     
-    def __init__(self, config, file):
+    
-        print "Initialising Everleaf converter class"
+    def __init__(self, in_path = '-', out_path = '-', follow = False):
-        HandHistoryConverter.__init__(self, config, file, sitename="Everleaf") # Call super class init.
+        """\
-        self.sitename = "Everleaf"
+in_path   (default '-' = sys.stdin)
-        self.setFileType("text", "cp1252")
+out_path  (default '-' = sys.stdout)
-        
+follow :  whether to tail -f the input"""
-
+        super(Everleaf, self).__init__(in_path, out_path, sitename="Everleaf", follow=follow) # Call super class init.
-        try:
+        logging.info("Initialising Everleaf converter class")
-            self.ofile     = os.path.join(self.hhdir, file.split(os.path.sep)[-2]+"-"+os.path.basename(file))
+        self.filetype = "text"
-        except:
+        self.codepage = "cp1252"
-            self.ofile     = os.path.join(self.hhdir, "x"+strftime("%d-%m-%y")+os.path.basename(file))
+        self.start()
 
     def compilePlayerRegexs(self):
         player_re = "(?P<PNAME>" + "|".join(map(re.escape, self.players)) + ")"
-        #print "DEBUG player_re: " + player_re
+        logging.debug("player_re: "+ player_re)
         self.re_PostSB          = re.compile(r"^%s: posts small blind \[\$? (?P<SB>[.0-9]+)" % player_re, re.MULTILINE)
         self.re_PostBB          = re.compile(r"^%s: posts big blind \[\$? (?P<BB>[.0-9]+)" % player_re, re.MULTILINE)
         self.re_PostBoth        = re.compile(r"^%s: posts both blinds \[\$? (?P<SBBB>[.0-9]+)" % player_re, re.MULTILINE)
@@ -65,7 +64,7 @@ class Everleaf(HandHistoryConverter):
                 ["ring", "omaha", "pl"]
                ]
 
-    def determineGameType(self):
+    def determineGameType(self, handtext):
         # Cheating with this regex, only support nlhe at the moment
         # Blinds $0.50/$1 PL Omaha - 2008/12/07 - 21:59:48
         # Blinds $0.05/$0.10 NL Hold'em - 2009/02/21 - 11:21:57
@@ -73,8 +72,9 @@ class Everleaf(HandHistoryConverter):
         structure = "" # nl, pl, cn, cp, fl
         game      = ""
 
-        m = self.re_GameInfo.search(self.obs)
+        m = self.re_GameInfo.search(handtext)
         if m == None:
+            logging.debug("Gametype didn't match")
             return None
         if m.group('LTYPE') == "NL":
             structure = "nl"
@@ -95,16 +95,14 @@ class Everleaf(HandHistoryConverter):
         return gametype
 
     def readHandInfo(self, hand):
-        m =  self.re_HandInfo.search(hand.string)
+        m = self.re_HandInfo.search(hand.string)
         if(m == None):
-            print "DEBUG: re_HandInfo.search failed: '%s'" %(hand.string)
+            logging.info("Didn't match re_HandInfo")
-        hand.handid = m.group('HID')
+            logging.info(hand.handtext)
-        hand.tablename = m.group('TABLE')
+            return None
-        hand.max_seats = 6 # assume 6-max unless we have proof it's a larger/smaller game, since everleaf doesn't give seat max info
+        logging.debug("HID %s" % m.group('HID'))
-# These work, but the info is already in the Hand class - should be used for tourneys though.
+        hand.handid =  m.group('HID')
-#		m.group('SB')
+        hand.maxseats = 6     # assume 6-max unless we have proof it's a larger/smaller game, since everleaf doesn't give seat max info
-#		m.group('BB')
-#		m.group('GAMETYPE')
 
 # Believe Everleaf time is GMT/UTC, no transation necessary
 # Stars format (Nov 10 2008): 2008/11/07 12:38:49 CET [2008/11/07 7:38:49 ET]
@@ -114,6 +112,8 @@ class Everleaf(HandHistoryConverter):
 #TODO: Do conversion from GMT to ET
 #TODO: Need some date functions to convert to different timezones (Date::Manip for perl rocked for this)
         hand.starttime = time.strptime(m.group('DATETIME'), "%Y/%m/%d - %H:%M:%S")
+        return
+        #return({'HID': m.group('HID'), 'table':m.group('TABLE'), 'max_seats':6})
 
     def readPlayerStacks(self, hand):
         m = self.re_PlayerInfo.finditer(hand.string)
@@ -139,18 +139,16 @@ class Everleaf(HandHistoryConverter):
             
 
     def readCommunityCards(self, hand, street): # street has been matched by markStreets, so exists in this hand
-        #print "DEBUG " + street + ":"
-        #print hand.streets.group(street) + "\n"
         if street in ('FLOP','TURN','RIVER'):   # a list of streets which get dealt community cards (i.e. all but PREFLOP)
             m = self.re_Board.search(hand.streets.group(street))
             hand.setCommunityCards(street, m.group('CARDS').split(', '))
 
     def readBlinds(self, hand):
-        try:
+        m = self.re_PostSB.search(hand.string)
-            m = self.re_PostSB.search(hand.string)
+        if m is not None:
             hand.addBlind(m.group('PNAME'), 'small blind', m.group('SB'))
-        except Exception, e: # no small blind
+        else:
-            #print e
+            logging.debug("No small blind")
             hand.addBlind(None, None, None)
         for a in self.re_PostBB.finditer(hand.string):
             hand.addBlind(a.group('PNAME'), 'big blind', a.group('BB'))
@@ -187,7 +185,7 @@ class Everleaf(HandHistoryConverter):
             elif action.group('ATYPE') == ' checks':
                 hand.addCheck( street, action.group('PNAME'))
             else:
-                print "DEBUG: unimplemented readAction: %s %s" %(action.group('PNAME'),action.group('ATYPE'),)
+                logging.debug("Unimplemented readAction: %s %s" %(action.group('PNAME'),action.group('ATYPE'),))
 
 
     def readShowdownActions(self, hand):
@@ -210,11 +208,22 @@ class Everleaf(HandHistoryConverter):
 
 
 if __name__ == "__main__":
-    c = Configuration.Config()
+    parser = OptionParser()
-    if len(sys.argv) ==  1:
+    parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/everleaf/Speed_Kuala_full.txt")
-        testfile = "regression-test-files/everleaf/Speed_Kuala_full.txt"
+    parser.add_option("-o", "--output", dest="opath", help="output translation to", default="-")
-    else:
+    parser.add_option("-f", "--follow", dest="follow", help="follow (tail -f) the input", action="store_true", default=False)
-        testfile = sys.argv[1]
+    parser.add_option("-q", "--quiet",
-    e = Everleaf(c, testfile)
+                  action="store_const", const=logging.CRITICAL, dest="verbosity", default=logging.INFO)
-    e.processFile()
+    parser.add_option("-v", "--verbose",
-    print str(e)
+                  action="store_const", const=logging.INFO, dest="verbosity")
+    parser.add_option("--vv",
+                  action="store_const", const=logging.DEBUG, dest="verbosity")
+
+    (options, args) = parser.parse_args()
+
+    LOG_FILENAME = './logging.out'
+    logging.basicConfig(filename=LOG_FILENAME,level=options.verbosity)
+
+    e = Everleaf(in_path = options.ipath, out_path = options.opath, follow = options.follow)
+
+    

pyfpdb/GuiBulkImport.py

@@ -59,7 +59,7 @@ class GuiBulkImport():
         self.importer.setCallHud(False)
         starttime = time()
         (stored, dups, partial, errs, ttime) = self.importer.runImport()
-        print 'GuiBulkImport.import_dir done: Stored: %d Duplicates: %d Partial: %d Errors: %d in %s seconds - %d/sec'\
+        print 'GuiBulkImport.import_dir done: Stored: %d \tDuplicates: %d \tPartial: %d \tErrors: %d in %s seconds - %d/sec'\
              % (stored, dups, partial, errs, ttime, stored / ttime)
         self.importer.clearFileList()
 

pyfpdb/Hand.py

@@ -15,12 +15,12 @@
 #In the "official" distribution you can find the license in
 #agpl-3.0.txt in the docs folder of the package.
 
-import Configuration
+
-import FpdbRegex
 import Hand
 import re
 import sys
 import traceback
+import logging
 import os
 import os.path
 import xml.dom.minidom
@@ -34,10 +34,10 @@ class Hand:
 #    def __init__(self, sitename, gametype, sb, bb, string):
 
     UPS = {'a':'A', 't':'T', 'j':'J', 'q':'Q', 'k':'K', 'S':'s', 'C':'c', 'H':'h', 'D':'d'}
-    def __init__(self, sitename, gametype, string):
+    def __init__(self, sitename, gametype, handtext):
         self.sitename = sitename
         self.gametype = gametype
-        self.string = string
+        self.string = handtext
 
         if gametype[1] == "hold" or self.gametype[1] == "omaha":
             self.streetList = ['PREFLOP','FLOP','TURN','RIVER'] # a list of the observed street names in order
@@ -130,7 +130,7 @@ If a player has None chips he won't be added."""
                     self.actions[street] = []
 
         else:
-            print "empty markStreets match" # better to raise exception and put process hand in a try block
+            logging.error("markstreets didn't match")
 
     def addHoleCards(self, cards, player):
         """\
@@ -151,7 +151,7 @@ player  (string) name of player
 For when a player shows cards for any reason (for showdown or out of choice).
 Card ranks will be uppercased
 """
-        #print "DEBUG: addShownCards", cards,player,holeandboard
+        
         if cards is not None:
             self.shown.add(player)
             self.addHoleCards(cards,player)
@@ -365,7 +365,7 @@ Map the tuple self.gametype onto the pokerstars string describing it
               "cp"  : "Cap Pot Limit"
              }
 
-        print "DEBUG: self.gametype: %s" %(self.gametype)
+        logging.debug("DEBUG: self.gametype: %s" %(self.gametype))
         string = "%s %s" %(gs[self.gametype[1]], ls[self.gametype[2]])
         
         return string

pyfpdb/HandHistoryConverter.py

@@ -15,12 +15,13 @@
 #In the "official" distribution you can find the license in
 #agpl-3.0.txt in the docs folder of the package.
 
-import Configuration
-import FpdbRegex
 import Hand
 import re
 import sys
+import threading
 import traceback
+import logging
+from optparse import OptionParser
 import os
 import os.path
 import xml.dom.minidom
@@ -73,23 +74,26 @@ gettext.install('myapplication')
 
 
 
-class HandHistoryConverter:
+class HandHistoryConverter(threading.Thread):
 #    eval = PokerEval()
-    def __init__(self, config, file, sitename):
+
-        print "HandHistory init called"
+    def __init__(self, in_path = '-', out_path = '-', sitename = None, follow=False):
-        self.c         = config
+        super(HandHistoryConverter, self).__init__()
-        self.sitename  = sitename
+        logging.info("HandHistory init called")
-        self.obs       = ""             # One big string
+        
+        # default filetype and codepage. Subclasses should set these properly.
         self.filetype  = "text"
         self.codepage  = "utf8"
-        self.doc       = None     # For XML based HH files
+        
-        self.file      = file
+        self.in_path = in_path
-        self.hhbase    = self.c.get_import_parameters().get("hhArchiveBase")
+        self.out_path = out_path
-        self.hhbase    = os.path.expanduser(self.hhbase)
+        if self.out_path == '-':
-        self.hhdir     = os.path.join(self.hhbase,sitename)
+            # write to stdout
-        self.gametype  = []
+            self.out_fh = sys.stdout
-        self.ofile     = os.path.join(self.hhdir, os.path.basename(file))
+        else:
-        self.rexx      = FpdbRegex.FpdbRegex()
+            self.out_fh = open(self.out_path, 'a')
+        self.sitename  = sitename
+        self.follow = follow
         self.players   = set()
         self.maxseats  = 10
 
@@ -106,6 +110,85 @@ class HandHistoryConverter:
         #tmp = tmp + "\tsb/bb:      '%s/%s'\n" % (self.gametype[3], self.gametype[4])
         return tmp
 
+    def run(self):
+        if self.follow:
+            for handtext in self.tailHands():
+                self.processHand(handtext)
+        else:
+            handsList = self.allHands()
+            logging.info("Parsing %d hands" % len(handsList))
+            for handtext in handsList:
+                self.processHand(handtext)
+
+    def tailHands(self):
+        """pseudo-code"""
+        while True:
+            ifile.tell()
+            text = ifile.read()
+            if nomoretext:
+                wait or sleep
+            else:
+                ahand = thenexthandinthetext
+                yield(ahand)
+
+    def allHands(self):
+        """Return a list of handtexts in the file at self.in_path"""
+        self.readFile()
+        self.obs = self.obs.strip()
+        self.obs = self.obs.replace('\r\n', '\n')
+        if self.obs == "" or self.obs == None:
+            logging.info("Read no hands.")
+            return
+        return self.obs.split(self.splitstring)
+        
+    def processHand(self, handtext):
+        gametype = self.determineGameType(handtext)
+        if gametype is None:
+            return
+        hand = Hand.Hand(self.sitename, gametype, handtext)
+        self.readHandInfo(hand)
+        self.readPlayerStacks(hand)
+        playersThisHand = set([player[1] for player in hand.players])
+        if playersThisHand <= self.players: # x <= y means 'x is subset of y'
+            # we're ok; the regex should already cover them all.
+            pass
+        else:
+            # we need to recompile the player regexs.
+            self.players = playersThisHand
+            self.compilePlayerRegexs()
+
+        self.markStreets(hand)
+        # Different calls if stud or holdem like
+        if gametype[1] == "hold" or gametype[1] == "omaha":
+            self.readBlinds(hand)
+            self.readButton(hand)
+            self.readHeroCards(hand) # want to generalise to draw games
+        elif gametype[1] == "razz" or gametype[1] == "stud" or gametype[1] == "stud8":
+            self.readAntes(hand)
+            self.readBringIn(hand)
+
+        self.readShowdownActions(hand)
+        
+        # Read actions in street order
+        for street in hand.streetList: # go through them in order
+            #logging.debug(street)
+            if hand.streets.group(street) is not None:
+                if gametype[1] == "hold" or gametype[1] == "omaha":
+                    self.readCommunityCards(hand, street) # read community cards
+                elif gametype[1] == "razz" or gametype[1] == "stud" or gametype[1] == "stud8":
+                    self.readPlayerCards(hand, street)
+
+                self.readAction(hand, street)
+                
+        self.readCollectPot(hand)
+        self.readShownCards(hand)
+        hand.totalPot() # finalise it (total the pot)
+        self.getRake(hand)
+        
+        hand.writeHand(self.out_fh)
+        
+       
+       
     def processFile(self):
         starttime = time.time()
         if not self.sanityCheck():
@@ -268,7 +351,7 @@ class HandHistoryConverter:
 
     def splitFileIntoHands(self):
         hands = []
-        self.obs.strip()
+        self.obs = self.obs.strip()
         list = self.re_SplitHands.split(self.obs)
         list.pop() #Last entry is empty
         for l in list:
@@ -276,13 +359,19 @@ class HandHistoryConverter:
             hands = hands + [Hand.Hand(self.sitename, self.gametype, l)]
         return hands
 
-    def readFile(self, filename):
+    def readFile(self):
-        """Read file"""
+        """Read in_path into self.obs or self.doc"""
-        print "Reading file: '%s'" %(filename)
+        
         if(self.filetype == "text"):
-            infile=codecs.open(filename, "r", self.codepage)
+            if self.in_path == '-':
-            self.obs = infile.read()
+                # read from stdin
-            infile.close()
+                logging.debug("Reading stdin with %s" % self.codepage)
+                in_fh = codecs.getreader('cp1252')(sys.stdin)
+            else:
+                logging.debug("Opening %s with %s" % (self.in_path, self.codepage))
+                in_fh = codecs.open(self.in_path, 'r', self.codepage)
+            self.obs = in_fh.read()
+            in_fh.close()
         elif(self.filetype == "xml"):
             try:
                 doc = xml.dom.minidom.parse(filename)