work in progress!

- command line options
  Try the following
  ./Everleaf --help
  cat everleaf.txt | ./Everleaf -i - -o -
  Try using -i -, and pasting in a hand. Be aware it needs an EOF
<ctrl>-d (unless you specify --follow, which isn't done yet)
- HHC is a thread
- hand gametypes per hand
- lots of stupid little things just to cause trouble
- added logging

Not yet done:
- tail -f mode
- integration into fpdb_import. Two options for grand plan here:
  1)  recently modified files get a HHC filter attached
      when the HHC runs out of pipe it waits
      runUpdated finds modified files and notifies the thread
  2)  the thread follows independently, when it runs out of input it
sleeps and then tries again at intervals
I like both, 1) involves bigger changes to fpdb_import whilst with 2) we
just point the existing fpdb_import code at the filter output (the way
we currently do filtering has to be altered though)

Comments welcome of course.
This commit is contained in:
Matt Turnbull 2009-02-26 00:59:36 +00:00
parent d5b7b17b21
commit 0986e36648
4 changed files with 171 additions and 73 deletions

View File

@ -18,9 +18,8 @@
######################################################################## ########################################################################
import sys import sys
import Configuration
from HandHistoryConverter import * from HandHistoryConverter import *
from time import strftime
# Class for converting Everleaf HH format. # Class for converting Everleaf HH format.
@ -29,27 +28,27 @@ class Everleaf(HandHistoryConverter):
# Static regexes # Static regexes
re_SplitHands = re.compile(r"\n\n+") re_SplitHands = re.compile(r"\n\n+")
re_GameInfo = re.compile(r".*Blinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<LTYPE>(NL|PL)) (?P<GAME>(Hold\'em|Omaha|7 Card Stud))") re_GameInfo = re.compile(r".*Blinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<LTYPE>(NL|PL)) (?P<GAME>(Hold\'em|Omaha|7 Card Stud))")
re_HandInfo = re.compile(r".*#(?P<HID>[0-9]+)\n.*\nBlinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<GAMETYPE>.*) - (?P<DATETIME>\d\d\d\d/\d\d/\d\d - \d\d:\d\d:\d\d)\nTable (?P<TABLE>[- a-zA-Z]+)") re_HandInfo = re.compile(r".*#(?P<HID>[0-9]+)\n.*\nBlinds \$?(?P<SB>[.0-9]+)/\$?(?P<BB>[.0-9]+) (?P<GAMETYPE>.*) - (?P<DATETIME>\d\d\d\d/\d\d/\d\d - \d\d:\d\d:\d\d)\nTable (?P<TABLE>[- a-zA-Z]+)", re.MULTILINE)
re_Button = re.compile(r"^Seat (?P<BUTTON>\d+) is the button", re.MULTILINE) re_Button = re.compile(r"^Seat (?P<BUTTON>\d+) is the button", re.MULTILINE)
re_PlayerInfo = re.compile(r"^Seat (?P<SEAT>[0-9]+): (?P<PNAME>.*) \(\s+(\$ (?P<CASH>[.0-9]+) USD|new player|All-in) \)", re.MULTILINE) re_PlayerInfo = re.compile(r"^Seat (?P<SEAT>[0-9]+): (?P<PNAME>.*) \(\s+(\$ (?P<CASH>[.0-9]+) USD|new player|All-in) \)", re.MULTILINE)
re_Board = re.compile(r"\[ (?P<CARDS>.+) \]") re_Board = re.compile(r"\[ (?P<CARDS>.+) \]")
splitstring = "\n\n\n"
def __init__(self, config, file): def __init__(self, in_path = '-', out_path = '-', follow = False):
print "Initialising Everleaf converter class" """\
HandHistoryConverter.__init__(self, config, file, sitename="Everleaf") # Call super class init. in_path (default '-' = sys.stdin)
self.sitename = "Everleaf" out_path (default '-' = sys.stdout)
self.setFileType("text", "cp1252") follow : whether to tail -f the input"""
super(Everleaf, self).__init__(in_path, out_path, sitename="Everleaf", follow=follow) # Call super class init.
logging.info("Initialising Everleaf converter class")
try: self.filetype = "text"
self.ofile = os.path.join(self.hhdir, file.split(os.path.sep)[-2]+"-"+os.path.basename(file)) self.codepage = "cp1252"
except: self.start()
self.ofile = os.path.join(self.hhdir, "x"+strftime("%d-%m-%y")+os.path.basename(file))
def compilePlayerRegexs(self): def compilePlayerRegexs(self):
player_re = "(?P<PNAME>" + "|".join(map(re.escape, self.players)) + ")" player_re = "(?P<PNAME>" + "|".join(map(re.escape, self.players)) + ")"
#print "DEBUG player_re: " + player_re logging.debug("player_re: "+ player_re)
self.re_PostSB = re.compile(r"^%s: posts small blind \[\$? (?P<SB>[.0-9]+)" % player_re, re.MULTILINE) self.re_PostSB = re.compile(r"^%s: posts small blind \[\$? (?P<SB>[.0-9]+)" % player_re, re.MULTILINE)
self.re_PostBB = re.compile(r"^%s: posts big blind \[\$? (?P<BB>[.0-9]+)" % player_re, re.MULTILINE) self.re_PostBB = re.compile(r"^%s: posts big blind \[\$? (?P<BB>[.0-9]+)" % player_re, re.MULTILINE)
self.re_PostBoth = re.compile(r"^%s: posts both blinds \[\$? (?P<SBBB>[.0-9]+)" % player_re, re.MULTILINE) self.re_PostBoth = re.compile(r"^%s: posts both blinds \[\$? (?P<SBBB>[.0-9]+)" % player_re, re.MULTILINE)
@ -65,7 +64,7 @@ class Everleaf(HandHistoryConverter):
["ring", "omaha", "pl"] ["ring", "omaha", "pl"]
] ]
def determineGameType(self): def determineGameType(self, handtext):
# Cheating with this regex, only support nlhe at the moment # Cheating with this regex, only support nlhe at the moment
# Blinds $0.50/$1 PL Omaha - 2008/12/07 - 21:59:48 # Blinds $0.50/$1 PL Omaha - 2008/12/07 - 21:59:48
# Blinds $0.05/$0.10 NL Hold'em - 2009/02/21 - 11:21:57 # Blinds $0.05/$0.10 NL Hold'em - 2009/02/21 - 11:21:57
@ -73,8 +72,9 @@ class Everleaf(HandHistoryConverter):
structure = "" # nl, pl, cn, cp, fl structure = "" # nl, pl, cn, cp, fl
game = "" game = ""
m = self.re_GameInfo.search(self.obs) m = self.re_GameInfo.search(handtext)
if m == None: if m == None:
logging.debug("Gametype didn't match")
return None return None
if m.group('LTYPE') == "NL": if m.group('LTYPE') == "NL":
structure = "nl" structure = "nl"
@ -95,16 +95,14 @@ class Everleaf(HandHistoryConverter):
return gametype return gametype
def readHandInfo(self, hand): def readHandInfo(self, hand):
m = self.re_HandInfo.search(hand.string) m = self.re_HandInfo.search(hand.string)
if(m == None): if(m == None):
print "DEBUG: re_HandInfo.search failed: '%s'" %(hand.string) logging.info("Didn't match re_HandInfo")
hand.handid = m.group('HID') logging.info(hand.handtext)
hand.tablename = m.group('TABLE') return None
hand.max_seats = 6 # assume 6-max unless we have proof it's a larger/smaller game, since everleaf doesn't give seat max info logging.debug("HID %s" % m.group('HID'))
# These work, but the info is already in the Hand class - should be used for tourneys though. hand.handid = m.group('HID')
# m.group('SB') hand.maxseats = 6 # assume 6-max unless we have proof it's a larger/smaller game, since everleaf doesn't give seat max info
# m.group('BB')
# m.group('GAMETYPE')
# Believe Everleaf time is GMT/UTC, no transation necessary # Believe Everleaf time is GMT/UTC, no transation necessary
# Stars format (Nov 10 2008): 2008/11/07 12:38:49 CET [2008/11/07 7:38:49 ET] # Stars format (Nov 10 2008): 2008/11/07 12:38:49 CET [2008/11/07 7:38:49 ET]
@ -114,6 +112,8 @@ class Everleaf(HandHistoryConverter):
#TODO: Do conversion from GMT to ET #TODO: Do conversion from GMT to ET
#TODO: Need some date functions to convert to different timezones (Date::Manip for perl rocked for this) #TODO: Need some date functions to convert to different timezones (Date::Manip for perl rocked for this)
hand.starttime = time.strptime(m.group('DATETIME'), "%Y/%m/%d - %H:%M:%S") hand.starttime = time.strptime(m.group('DATETIME'), "%Y/%m/%d - %H:%M:%S")
return
#return({'HID': m.group('HID'), 'table':m.group('TABLE'), 'max_seats':6})
def readPlayerStacks(self, hand): def readPlayerStacks(self, hand):
m = self.re_PlayerInfo.finditer(hand.string) m = self.re_PlayerInfo.finditer(hand.string)
@ -139,18 +139,16 @@ class Everleaf(HandHistoryConverter):
def readCommunityCards(self, hand, street): # street has been matched by markStreets, so exists in this hand def readCommunityCards(self, hand, street): # street has been matched by markStreets, so exists in this hand
#print "DEBUG " + street + ":"
#print hand.streets.group(street) + "\n"
if street in ('FLOP','TURN','RIVER'): # a list of streets which get dealt community cards (i.e. all but PREFLOP) if street in ('FLOP','TURN','RIVER'): # a list of streets which get dealt community cards (i.e. all but PREFLOP)
m = self.re_Board.search(hand.streets.group(street)) m = self.re_Board.search(hand.streets.group(street))
hand.setCommunityCards(street, m.group('CARDS').split(', ')) hand.setCommunityCards(street, m.group('CARDS').split(', '))
def readBlinds(self, hand): def readBlinds(self, hand):
try: m = self.re_PostSB.search(hand.string)
m = self.re_PostSB.search(hand.string) if m is not None:
hand.addBlind(m.group('PNAME'), 'small blind', m.group('SB')) hand.addBlind(m.group('PNAME'), 'small blind', m.group('SB'))
except Exception, e: # no small blind else:
#print e logging.debug("No small blind")
hand.addBlind(None, None, None) hand.addBlind(None, None, None)
for a in self.re_PostBB.finditer(hand.string): for a in self.re_PostBB.finditer(hand.string):
hand.addBlind(a.group('PNAME'), 'big blind', a.group('BB')) hand.addBlind(a.group('PNAME'), 'big blind', a.group('BB'))
@ -187,7 +185,7 @@ class Everleaf(HandHistoryConverter):
elif action.group('ATYPE') == ' checks': elif action.group('ATYPE') == ' checks':
hand.addCheck( street, action.group('PNAME')) hand.addCheck( street, action.group('PNAME'))
else: else:
print "DEBUG: unimplemented readAction: %s %s" %(action.group('PNAME'),action.group('ATYPE'),) logging.debug("Unimplemented readAction: %s %s" %(action.group('PNAME'),action.group('ATYPE'),))
def readShowdownActions(self, hand): def readShowdownActions(self, hand):
@ -210,11 +208,22 @@ class Everleaf(HandHistoryConverter):
if __name__ == "__main__": if __name__ == "__main__":
c = Configuration.Config() parser = OptionParser()
if len(sys.argv) == 1: parser.add_option("-i", "--input", dest="ipath", help="parse input hand history", default="regression-test-files/everleaf/Speed_Kuala_full.txt")
testfile = "regression-test-files/everleaf/Speed_Kuala_full.txt" parser.add_option("-o", "--output", dest="opath", help="output translation to", default="-")
else: parser.add_option("-f", "--follow", dest="follow", help="follow (tail -f) the input", action="store_true", default=False)
testfile = sys.argv[1] parser.add_option("-q", "--quiet",
e = Everleaf(c, testfile) action="store_const", const=logging.CRITICAL, dest="verbosity", default=logging.INFO)
e.processFile() parser.add_option("-v", "--verbose",
print str(e) action="store_const", const=logging.INFO, dest="verbosity")
parser.add_option("--vv",
action="store_const", const=logging.DEBUG, dest="verbosity")
(options, args) = parser.parse_args()
LOG_FILENAME = './logging.out'
logging.basicConfig(filename=LOG_FILENAME,level=options.verbosity)
e = Everleaf(in_path = options.ipath, out_path = options.opath, follow = options.follow)

View File

@ -59,7 +59,7 @@ class GuiBulkImport():
self.importer.setCallHud(False) self.importer.setCallHud(False)
starttime = time() starttime = time()
(stored, dups, partial, errs, ttime) = self.importer.runImport() (stored, dups, partial, errs, ttime) = self.importer.runImport()
print 'GuiBulkImport.import_dir done: Stored: %d Duplicates: %d Partial: %d Errors: %d in %s seconds - %d/sec'\ print 'GuiBulkImport.import_dir done: Stored: %d \tDuplicates: %d \tPartial: %d \tErrors: %d in %s seconds - %d/sec'\
% (stored, dups, partial, errs, ttime, stored / ttime) % (stored, dups, partial, errs, ttime, stored / ttime)
self.importer.clearFileList() self.importer.clearFileList()

View File

@ -15,12 +15,12 @@
#In the "official" distribution you can find the license in #In the "official" distribution you can find the license in
#agpl-3.0.txt in the docs folder of the package. #agpl-3.0.txt in the docs folder of the package.
import Configuration
import FpdbRegex
import Hand import Hand
import re import re
import sys import sys
import traceback import traceback
import logging
import os import os
import os.path import os.path
import xml.dom.minidom import xml.dom.minidom
@ -34,10 +34,10 @@ class Hand:
# def __init__(self, sitename, gametype, sb, bb, string): # def __init__(self, sitename, gametype, sb, bb, string):
UPS = {'a':'A', 't':'T', 'j':'J', 'q':'Q', 'k':'K', 'S':'s', 'C':'c', 'H':'h', 'D':'d'} UPS = {'a':'A', 't':'T', 'j':'J', 'q':'Q', 'k':'K', 'S':'s', 'C':'c', 'H':'h', 'D':'d'}
def __init__(self, sitename, gametype, string): def __init__(self, sitename, gametype, handtext):
self.sitename = sitename self.sitename = sitename
self.gametype = gametype self.gametype = gametype
self.string = string self.string = handtext
if gametype[1] == "hold" or self.gametype[1] == "omaha": if gametype[1] == "hold" or self.gametype[1] == "omaha":
self.streetList = ['PREFLOP','FLOP','TURN','RIVER'] # a list of the observed street names in order self.streetList = ['PREFLOP','FLOP','TURN','RIVER'] # a list of the observed street names in order
@ -130,7 +130,7 @@ If a player has None chips he won't be added."""
self.actions[street] = [] self.actions[street] = []
else: else:
print "empty markStreets match" # better to raise exception and put process hand in a try block logging.error("markstreets didn't match")
def addHoleCards(self, cards, player): def addHoleCards(self, cards, player):
"""\ """\
@ -151,7 +151,7 @@ player (string) name of player
For when a player shows cards for any reason (for showdown or out of choice). For when a player shows cards for any reason (for showdown or out of choice).
Card ranks will be uppercased Card ranks will be uppercased
""" """
#print "DEBUG: addShownCards", cards,player,holeandboard
if cards is not None: if cards is not None:
self.shown.add(player) self.shown.add(player)
self.addHoleCards(cards,player) self.addHoleCards(cards,player)
@ -365,7 +365,7 @@ Map the tuple self.gametype onto the pokerstars string describing it
"cp" : "Cap Pot Limit" "cp" : "Cap Pot Limit"
} }
print "DEBUG: self.gametype: %s" %(self.gametype) logging.debug("DEBUG: self.gametype: %s" %(self.gametype))
string = "%s %s" %(gs[self.gametype[1]], ls[self.gametype[2]]) string = "%s %s" %(gs[self.gametype[1]], ls[self.gametype[2]])
return string return string

View File

@ -15,12 +15,13 @@
#In the "official" distribution you can find the license in #In the "official" distribution you can find the license in
#agpl-3.0.txt in the docs folder of the package. #agpl-3.0.txt in the docs folder of the package.
import Configuration
import FpdbRegex
import Hand import Hand
import re import re
import sys import sys
import threading
import traceback import traceback
import logging
from optparse import OptionParser
import os import os
import os.path import os.path
import xml.dom.minidom import xml.dom.minidom
@ -73,23 +74,26 @@ gettext.install('myapplication')
class HandHistoryConverter: class HandHistoryConverter(threading.Thread):
# eval = PokerEval() # eval = PokerEval()
def __init__(self, config, file, sitename):
print "HandHistory init called" def __init__(self, in_path = '-', out_path = '-', sitename = None, follow=False):
self.c = config super(HandHistoryConverter, self).__init__()
self.sitename = sitename logging.info("HandHistory init called")
self.obs = "" # One big string
# default filetype and codepage. Subclasses should set these properly.
self.filetype = "text" self.filetype = "text"
self.codepage = "utf8" self.codepage = "utf8"
self.doc = None # For XML based HH files
self.file = file self.in_path = in_path
self.hhbase = self.c.get_import_parameters().get("hhArchiveBase") self.out_path = out_path
self.hhbase = os.path.expanduser(self.hhbase) if self.out_path == '-':
self.hhdir = os.path.join(self.hhbase,sitename) # write to stdout
self.gametype = [] self.out_fh = sys.stdout
self.ofile = os.path.join(self.hhdir, os.path.basename(file)) else:
self.rexx = FpdbRegex.FpdbRegex() self.out_fh = open(self.out_path, 'a')
self.sitename = sitename
self.follow = follow
self.players = set() self.players = set()
self.maxseats = 10 self.maxseats = 10
@ -106,6 +110,85 @@ class HandHistoryConverter:
#tmp = tmp + "\tsb/bb: '%s/%s'\n" % (self.gametype[3], self.gametype[4]) #tmp = tmp + "\tsb/bb: '%s/%s'\n" % (self.gametype[3], self.gametype[4])
return tmp return tmp
def run(self):
if self.follow:
for handtext in self.tailHands():
self.processHand(handtext)
else:
handsList = self.allHands()
logging.info("Parsing %d hands" % len(handsList))
for handtext in handsList:
self.processHand(handtext)
def tailHands(self):
"""pseudo-code"""
while True:
ifile.tell()
text = ifile.read()
if nomoretext:
wait or sleep
else:
ahand = thenexthandinthetext
yield(ahand)
def allHands(self):
"""Return a list of handtexts in the file at self.in_path"""
self.readFile()
self.obs = self.obs.strip()
self.obs = self.obs.replace('\r\n', '\n')
if self.obs == "" or self.obs == None:
logging.info("Read no hands.")
return
return self.obs.split(self.splitstring)
def processHand(self, handtext):
gametype = self.determineGameType(handtext)
if gametype is None:
return
hand = Hand.Hand(self.sitename, gametype, handtext)
self.readHandInfo(hand)
self.readPlayerStacks(hand)
playersThisHand = set([player[1] for player in hand.players])
if playersThisHand <= self.players: # x <= y means 'x is subset of y'
# we're ok; the regex should already cover them all.
pass
else:
# we need to recompile the player regexs.
self.players = playersThisHand
self.compilePlayerRegexs()
self.markStreets(hand)
# Different calls if stud or holdem like
if gametype[1] == "hold" or gametype[1] == "omaha":
self.readBlinds(hand)
self.readButton(hand)
self.readHeroCards(hand) # want to generalise to draw games
elif gametype[1] == "razz" or gametype[1] == "stud" or gametype[1] == "stud8":
self.readAntes(hand)
self.readBringIn(hand)
self.readShowdownActions(hand)
# Read actions in street order
for street in hand.streetList: # go through them in order
#logging.debug(street)
if hand.streets.group(street) is not None:
if gametype[1] == "hold" or gametype[1] == "omaha":
self.readCommunityCards(hand, street) # read community cards
elif gametype[1] == "razz" or gametype[1] == "stud" or gametype[1] == "stud8":
self.readPlayerCards(hand, street)
self.readAction(hand, street)
self.readCollectPot(hand)
self.readShownCards(hand)
hand.totalPot() # finalise it (total the pot)
self.getRake(hand)
hand.writeHand(self.out_fh)
def processFile(self): def processFile(self):
starttime = time.time() starttime = time.time()
if not self.sanityCheck(): if not self.sanityCheck():
@ -268,7 +351,7 @@ class HandHistoryConverter:
def splitFileIntoHands(self): def splitFileIntoHands(self):
hands = [] hands = []
self.obs.strip() self.obs = self.obs.strip()
list = self.re_SplitHands.split(self.obs) list = self.re_SplitHands.split(self.obs)
list.pop() #Last entry is empty list.pop() #Last entry is empty
for l in list: for l in list:
@ -276,13 +359,19 @@ class HandHistoryConverter:
hands = hands + [Hand.Hand(self.sitename, self.gametype, l)] hands = hands + [Hand.Hand(self.sitename, self.gametype, l)]
return hands return hands
def readFile(self, filename): def readFile(self):
"""Read file""" """Read in_path into self.obs or self.doc"""
print "Reading file: '%s'" %(filename)
if(self.filetype == "text"): if(self.filetype == "text"):
infile=codecs.open(filename, "r", self.codepage) if self.in_path == '-':
self.obs = infile.read() # read from stdin
infile.close() logging.debug("Reading stdin with %s" % self.codepage)
in_fh = codecs.getreader('cp1252')(sys.stdin)
else:
logging.debug("Opening %s with %s" % (self.in_path, self.codepage))
in_fh = codecs.open(self.in_path, 'r', self.codepage)
self.obs = in_fh.read()
in_fh.close()
elif(self.filetype == "xml"): elif(self.filetype == "xml"):
try: try:
doc = xml.dom.minidom.parse(filename) doc = xml.dom.minidom.parse(filename)