From 73e6666cbc77289727a7f7f944994fddc344616d Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 30 Jul 2009 11:40:16 +0800 Subject: [PATCH 1/9] Add auto folder creation to HHC Contributed by grindi on 2+2 http://forumserver.twoplustwo.com/showpost.php?p=12156328&postcount=1666 Creates the sub folders of hhArchiveBase for placing the converted files in. --- pyfpdb/HandHistoryConverter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyfpdb/HandHistoryConverter.py b/pyfpdb/HandHistoryConverter.py index f5d46034..2ded2c50 100644 --- a/pyfpdb/HandHistoryConverter.py +++ b/pyfpdb/HandHistoryConverter.py @@ -57,6 +57,10 @@ class HandHistoryConverter(): self.out_fh = sys.stdout else: # TODO: out_path should be sanity checked. + out_dir = os.path.dirname(self.out_path) + if not os.path.isdir(out_dir): + logging.info("Creatin directory '%s'" % out_dir) + os.makedirs(out_dir) self.out_fh = open(self.out_path, 'w') self.sitename = sitename From 68ac5ff5544e87e899f710f18ba69d51dd452aaa Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 30 Jul 2009 12:13:45 +0800 Subject: [PATCH 2/9] Q&D hack to fix FTP file reads after update Contributed by grindi on 2+2 http://forumserver.twoplustwo.com/showpost.php?p=12156328&postcount=166 Removes u'\xff\xfe' from the end of file if they exist. --- pyfpdb/FulltiltToFpdb.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 4fb0e8b3..3b8026bf 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -118,7 +118,7 @@ follow : whether to tail -f the input""" if not m: return None mg = m.groupdict() - + # translations from captured groups to our info strings limits = { 'No Limit':'nl', 'Pot Limit':'pl', 'Limit':'fl' } games = { # base, category @@ -142,6 +142,24 @@ follow : whether to tail -f the input""" # NB: SB, BB must be interpreted as blinds or bets depending on limit type. return info + #Following function is a hack, we should be dealing with this in readFile (i think correct codepage....) + # Same function as parent class, removing the 2 end characters. - CG + def allHandsAsList(self): + """Return a list of handtexts in the file at self.in_path""" + #TODO : any need for this to be generator? e.g. stars support can email one huge file of all hands in a year. Better to read bit by bit than all at once. + self.readFile() + + # FIXME: it's a hack + if self.obs[:2] == u'\xff\xfe': + self.obs = self.obs[2:].replace('\x00', '') + + self.obs = self.obs.strip() + self.obs = self.obs.replace('\r\n', '\n') + if self.obs == "" or self.obs == None: + logging.info("Read no hands.") + return + return re.split(self.re_SplitHands, self.obs) + def readHandInfo(self, hand): m = self.re_HandInfo.search(hand.handText,re.DOTALL) if(m == None): From 2c7e00f02a7d128123000803286a88a3a78b5da2 Mon Sep 17 00:00:00 2001 From: eblade Date: Thu, 30 Jul 2009 17:29:57 -0400 Subject: [PATCH 3/9] fpdb_import: use stat_info.st_size instead of stat_info.st_mtime to determine hh file updates --- pyfpdb/fpdb_import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index ed87731b..9d76be69 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -271,11 +271,11 @@ class Importer: try: lastupdate = self.updated[file] #rulog.writelines("lastupdate = %d, mtime = %d" % (lastupdate,stat_info.st_mtime)) - if stat_info.st_mtime > lastupdate: + if stat_info.st_size > lastupdate: #stat_info.st_mtime > lastupdate: self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.updated[file] = time() + self.updated[file] = stat_info.st_size #time() except: - self.updated[file] = time() + self.updated[file] = stat_info.st_size #time() # If modified in the last minute run an immediate import. # This codepath only runs first time the file is found. if os.path.isdir(file) or (time() - stat_info.st_mtime) < 60: From 1820e7105476ae1f7491ec98df1b4b2560de6696 Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 30 Jul 2009 18:06:13 -0400 Subject: [PATCH 4/9] Started on supporting expected HH change--rewrote GameType regex. This version should work with the expected HH format for $ tables only. I abstracted the ISO currency codes and currency symbols out of the regex so they only need to be maintained in 1 place. I didn't apply the change to everywhere "\$" appears, in case comments are negative. --- pyfpdb/PokerStarsToFpdb.py | 105 +++++++++---------------------------- 1 file changed, 25 insertions(+), 80 deletions(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 61d4df4c..f30b3df4 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -29,18 +29,31 @@ class PokerStars(HandHistoryConverter): ############################################################ # Class Variables + substitutions = { + 'LEGAL_ISO' : "USD|EUR|GBP|CAD", # legal ISO currency codes + 'LS' : "\$" # legal currency symbols + } + # Static regexes - re_GameInfo = re.compile("""PokerStars\sGame\s\#(?P[0-9]+):\s+ - (Tournament\s\#(?P\d+),\s(?P[\$\+\d\.]+)\s)? - (?PHORSE|8\-Game|HOSE)?\s?\(? - (?PHold\'em|Razz|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball)\s - (?PNo\sLimit|Limit|Pot\sLimit)\)?,?\s - (-\sLevel\s(?P[IVXLC]+)\s)?\(? - (?P\$|)? - (?P[.0-9]+)/\$? - (?P[.0-9]+)\)\s-\s - (?P.*$)""", - re.MULTILINE|re.VERBOSE) + re_GameInfo = re.compile(""" + PokerStars\sGame\s\#(?P[0-9]+):\s+ + (Tournament\s\# # open paren of tournament info + (?P\d+),\s + (?P[%(LS)s\+\d\.]+ # here's how I plan to use LS + \s?(?P%(LEGAL_ISO)s)? + )\s)? # close paren of tournament info + (?PHORSE|8\-Game|HOSE)?\s?\(? + (?PHold\'em|Razz|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball)\s + (?PNo\sLimit|Limit|Pot\sLimit)\)?,?\s + (-\sLevel\s(?P[IVXLC]+)\s)? + \(? # open paren of the stakes + (?P%(LS)s|)? + (?P[.0-9]+)/%(LS)s? + (?P[.0-9]+) + \s?(?P%(LEGAL_ISO)s)? + \)\s-\s # close paren of the stakes + (?P.*$)""" % substitutions, + re.MULTILINE|re.VERBOSE) re_SplitHands = re.compile('\n\n+') re_TailSplitHands = re.compile('(\n\n\n+)') re_HandInfo = re.compile("""^Table\s\'(?P[-\ a-zA-Z\d]+)\'\s @@ -121,6 +134,7 @@ follow : whether to tail -f the input""" return None mg = m.groupdict() + print "mg =", mg # translations from captured groups to fpdb info strings limits = { 'No Limit':'nl', 'Pot Limit':'pl', 'Limit':'fl' } games = { # base, category @@ -270,19 +284,6 @@ follow : whether to tail -f the input""" for a in self.re_PostBoth.finditer(hand.handText): hand.addBlind(a.group('PNAME'), 'both', a.group('SBBB')) -# def readHeroCards(self, hand): -# m = self.re_HeroCards.search(hand.handText) -# if(m == None): -# #Not involved in hand -# hand.involved = False -# else: -# hand.hero = m.group('PNAME') -# # "2c, qh" -> set(["2c","qc"]) -# # Also works with Omaha hands. -# cards = m.group('NEWCARDS') -# cards = set(cards.split(' ')) -# hand.addHoleCards(cards, m.group('PNAME'), shown=False, mucked=False, dealt=True) - def readHeroCards(self, hand): # streets PREFLOP, PREDRAW, and THIRD are special cases beacause # we need to grab hero's cards @@ -318,62 +319,6 @@ follow : whether to tail -f the input""" else: hand.addHoleCards(street, player, open=newcards, closed=oldcards, shown=False, mucked=False, dealt=False) - -# def readDrawCards(self, hand, street): -# logging.debug("readDrawCards") -# m = self.re_HeroCards.finditer(hand.streets[street]) -# if m == None: -# hand.involved = False -# else: -# for player in m: -# hand.hero = player.group('PNAME') # Only really need to do this once -# newcards = player.group('NEWCARDS') -# oldcards = player.group('OLDCARDS') -# if newcards == None: -# newcards = set() -# else: -# newcards = set(newcards.split(' ')) -# if oldcards == None: -# oldcards = set() -# else: -# oldcards = set(oldcards.split(' ')) -# hand.addDrawHoleCards(newcards, oldcards, player.group('PNAME'), street) - - -# def readStudPlayerCards(self, hand, street): -# # See comments of reference implementation in FullTiltToFpdb.py -# logging.debug("readStudPlayerCards") -# m = self.re_HeroCards.finditer(hand.streets[street]) -# for player in m: -# #~ logging.debug(player.groupdict()) -# (pname, oldcards, newcards) = (player.group('PNAME'), player.group('OLDCARDS'), player.group('NEWCARDS')) -# if oldcards: -# oldcards = [c.strip() for c in oldcards.split(' ')] -# if newcards: -# newcards = [c.strip() for c in newcards.split(' ')] -# if street=='ANTES': -# return -# elif street=='THIRD': -# # we'll have observed hero holecards in CARDS and thirdstreet open cards in 'NEWCARDS' -# # hero: [xx][o] -# # others: [o] -# hand.addPlayerCards(player = player.group('PNAME'), street = street, closed = oldcards, open = newcards) -# elif street in ('FOURTH', 'FIFTH', 'SIXTH'): -# # 4th: -# # hero: [xxo] [o] -# # others: [o] [o] -# # 5th: -# # hero: [xxoo] [o] -# # others: [oo] [o] -# # 6th: -# # hero: [xxooo] [o] -# # others: [ooo] [o] -# hand.addPlayerCards(player = player.group('PNAME'), street = street, open = newcards) -# # we may additionally want to check the earlier streets tally with what we have but lets trust it for now. -# elif street=='SEVENTH' and newcards: -# # hero: [xxoooo] [x] -# # others: not reported. -# hand.addPlayerCards(player = player.group('PNAME'), street = street, closed = newcards) def readAction(self, hand, street): m = self.re_Action.finditer(hand.streets[street]) From 3ce0f91b277a6c4511cdfd83a05911e2e00c9b29 Mon Sep 17 00:00:00 2001 From: eblade Date: Thu, 30 Jul 2009 19:53:44 -0400 Subject: [PATCH 5/9] useless change of a comment just to test my rss output --- pyfpdb/fpdb_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index 9d76be69..f9b4cec9 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -351,7 +351,7 @@ class Importer: starttime = time() last_read_hand = 0 loc = 0 - #print "file =", file + # print "file =", file if file == "stdin": inputFile = sys.stdin else: From c400fd9ba7032df39ad93f12884e0bd1da09ed65 Mon Sep 17 00:00:00 2001 From: eblade Date: Thu, 30 Jul 2009 20:56:39 -0400 Subject: [PATCH 6/9] FTtoFpdb: default buyin to $0.00+$0.00 and level to 0, they don't seem to be in the FT hh --- pyfpdb/FulltiltToFpdb.py | 6 ++++++ pyfpdb/Hand.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 3b8026bf..51b62174 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -176,6 +176,12 @@ follow : whether to tail -f the input""" hand.tourNo = m.group('TOURNO') if m.group('PLAY') != None: hand.gametype['currency'] = 'play' + + # TODO: if there's a way to figure these out, we should.. otherwise we have to stuff it with unknowns + if hand.buyin == None: + hand.buyin = "$0.00+$0.00" + if hand.level == None: + hand.level = "0" # These work, but the info is already in the Hand class - should be used for tourneys though. # m.group('SB') diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 4e7d4bdf..50fab822 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -564,7 +564,7 @@ Map the tuple self.gametype onto the pokerstars string describing it def writeGameLine(self): """Return the first HH line for the current hand.""" gs = "PokerStars Game #%s: " % self.handid - + if self.tourNo != None and self.mixed != None: # mixed tournament gs = gs + "Tournament #%s, %s %s (%s) - Level %s (%s) - " % (self.tourNo, self.buyin, self.MS[self.mixed], self.getGameTypeAsString(), self.level, self.getStakesAsString()) elif self.tourNo != None: # all other tournaments From f5ba0b5b7fe9cbc84bac9ff7f4a32c162140826b Mon Sep 17 00:00:00 2001 From: eblade Date: Thu, 30 Jul 2009 21:50:08 -0400 Subject: [PATCH 7/9] Database: insert missing HandToWrite.payin_amounts var EverleafToFpdb: remove DEBUG XXXXXXX line FulltiltToFpdb: pass on converting tournament hands Hand: add Hand.fee var that Database is looking for fpdb_import: clean up file update scan code, queue files that have changed in the last 60 seconds for the next importer pass, instead of locking it up reading them now (not that this does anything since FT no longer updates the mtime until it closes the table, but maybe it works with Stars or others) --- pyfpdb/Database.py | 1 + pyfpdb/EverleafToFpdb.py | 2 +- pyfpdb/FulltiltToFpdb.py | 1 + pyfpdb/Hand.py | 1 + pyfpdb/fpdb_import.py | 22 +++++++--------------- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 8d58da50..9e0007a3 100755 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1620,6 +1620,7 @@ class HandToWrite: self.maxSeats = None self.tableName = None self.seatNos = None + self.payin_amounts = None # tourney import was complaining mightily about this missing except: print "htw.init error: " + str(sys.exc_info()) raise diff --git a/pyfpdb/EverleafToFpdb.py b/pyfpdb/EverleafToFpdb.py index a7685f90..9c63bcf0 100755 --- a/pyfpdb/EverleafToFpdb.py +++ b/pyfpdb/EverleafToFpdb.py @@ -44,7 +44,7 @@ out_path (default '-' = sys.stdout) follow : whether to tail -f the input autostart: whether to run the thread (or you can call start() yourself) debugging: if False, pass on partially supported game types. If true, have a go and error...""" - print "DEBUG: XXXXXXXXXXXXXXX" + #print "DEBUG: XXXXXXXXXXXXXXX" HandHistoryConverter.__init__(self, in_path, out_path, sitename="Everleaf", follow=follow, index=index) logging.info("Initialising Everleaf converter class") self.filetype = "text" diff --git a/pyfpdb/FulltiltToFpdb.py b/pyfpdb/FulltiltToFpdb.py index 51b62174..a68c1cbd 100755 --- a/pyfpdb/FulltiltToFpdb.py +++ b/pyfpdb/FulltiltToFpdb.py @@ -140,6 +140,7 @@ follow : whether to tail -f the input""" if mg['TOURNO'] == None: info['type'] = "ring" else: info['type'] = "tour" # NB: SB, BB must be interpreted as blinds or bets depending on limit type. + if info['type'] == "tour": return None # importer is screwed on tournies, pass on those hands so we don't interrupt other autoimporting return info #Following function is a hack, we should be dealing with this in readFile (i think correct codepage....) diff --git a/pyfpdb/Hand.py b/pyfpdb/Hand.py index 50fab822..0dbb9674 100644 --- a/pyfpdb/Hand.py +++ b/pyfpdb/Hand.py @@ -56,6 +56,7 @@ class Hand(object): self.buttonpos = 0 self.tourNo = None self.buyin = None + self.fee = None # the Database code is looking for this one .. ? self.level = None self.mixed = None self.seating = [] diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index f9b4cec9..76b483ba 100644 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -268,23 +268,15 @@ class Importer: if os.path.exists(file): stat_info = os.stat(file) #rulog.writelines("path exists ") - try: - lastupdate = self.updated[file] - #rulog.writelines("lastupdate = %d, mtime = %d" % (lastupdate,stat_info.st_mtime)) - if stat_info.st_size > lastupdate: #stat_info.st_mtime > lastupdate: + if file in self.updated: + if stat_info.st_size > self.updated[file]: self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - self.updated[file] = stat_info.st_size #time() - except: - self.updated[file] = stat_info.st_size #time() - # If modified in the last minute run an immediate import. - # This codepath only runs first time the file is found. + self.updated[file] = stat_info.st_size + else: if os.path.isdir(file) or (time() - stat_info.st_mtime) < 60: - # TODO attach a HHC thread to the file - # TODO import the output of the HHC thread -- this needs to wait for the HHC to block? - self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) - # TODO we also test if directory, why? - #if os.path.isdir(file): - #self.import_file_dict(file, self.filelist[file][0], self.filelist[file][1]) + self.updated[file] = 0 + else: + self.updated[file] = stat_info.st_size else: self.removeFromFileList[file] = True self.addToDirList = filter(lambda x: self.addImportDirectory(x, True, self.addToDirList[x][0], self.addToDirList[x][1]), self.addToDirList) From 29366d3bb6731c73966c8391a799db2358211d5d Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 30 Jul 2009 23:42:57 -0400 Subject: [PATCH 8/9] Ugly fix to prevent importing from failing due to undef positions array. --- pyfpdb/fpdb_parse_logic.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyfpdb/fpdb_parse_logic.py b/pyfpdb/fpdb_parse_logic.py index fd2d6796..673254d0 100644 --- a/pyfpdb/fpdb_parse_logic.py +++ b/pyfpdb/fpdb_parse_logic.py @@ -167,7 +167,12 @@ def mainParser(settings, siteID, category, hand, config, db = None): except: print "parse: error during commit: " + str(sys.exc_value) - +# HERE's an ugly kludge to keep from failing when positions is undef +# We'll fix this by getting rid of the legacy importer. REB + try: + if positions: pass + except: + positions = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # save data structures in a HandToWrite instance and then insert into database: htw = Database.HandToWrite() htw.set_all( config, settings, base, category, siteTourneyNo, buyin From 5660b7735599316182e1b1586fc83ead199831ed Mon Sep 17 00:00:00 2001 From: Ray Date: Thu, 30 Jul 2009 23:44:59 -0400 Subject: [PATCH 9/9] Delete uneeded intermediate prints. --- pyfpdb/PokerStarsToFpdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index f30b3df4..20ba4319 100755 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -134,7 +134,6 @@ follow : whether to tail -f the input""" return None mg = m.groupdict() - print "mg =", mg # translations from captured groups to fpdb info strings limits = { 'No Limit':'nl', 'Pot Limit':'pl', 'Limit':'fl' } games = { # base, category