From ac4cae5abffa42dc189ade386637d80fde8d72cb Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:14:29 -0600 Subject: [PATCH 01/14] * Updated the methods, variables, and SQL in Database.py and SQL.py to add Hours to the HudCache.styleKey field --- pyfpdb/Database.py | 26 +++++++++++++------------- pyfpdb/SQL.py | 18 +++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 0647baf2..1fb94cbd 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -291,8 +291,8 @@ class Database: # vars for hand ids or dates fetched according to above config: self.hand_1day_ago = 0 # max hand id more than 24 hrs earlier than now - self.date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) - self.h_date_ndays_ago = 'd000000' # date N days ago ('d' + YYMMDD) for hero + self.date_ndays_ago = 'd00000000' # date N days ago ('d' + YYMMDD) + self.h_date_ndays_ago = 'd00000000' # date N days ago ('d' + YYMMDD) for hero self.date_nhands_ago = {} # dates N hands ago per player - not used yet self.saveActions = False if self.import_options['saveActions'] == False else True @@ -703,7 +703,7 @@ class Database: try: # self.date_nhands_ago is used for fetching stats for last n hands (hud_style = 'H') # This option not used yet - needs to be called for each player :-( - self.date_nhands_ago[str(playerid)] = 'd000000' + self.date_nhands_ago[str(playerid)] = 'd00000000' # should use aggregated version of query if appropriate c.execute(self.sql.query['get_date_nhands_ago'], (self.hud_hands, playerid)) @@ -771,11 +771,11 @@ class Database: if hud_style == 'T': stylekey = self.date_ndays_ago elif hud_style == 'A': - stylekey = '0000000' # all stylekey values should be higher than this + stylekey = '000000000' # all stylekey values should be higher than this elif hud_style == 'S': - stylekey = 'zzzzzzz' # all stylekey values should be lower than this + stylekey = 'zzzzzzzzz' # all stylekey values should be lower than this else: - stylekey = '0000000' + stylekey = '000000000' log.info('hud_style: %s' % hud_style) #elif hud_style == 'H': @@ -784,11 +784,11 @@ class Database: if h_hud_style == 'T': h_stylekey = self.h_date_ndays_ago elif h_hud_style == 'A': - h_stylekey = '0000000' # all stylekey values should be higher than this + h_stylekey = '000000000' # all stylekey values should be higher than this elif h_hud_style == 'S': - h_stylekey = 'zzzzzzz' # all stylekey values should be lower than this + h_stylekey = 'zzzzzzzzz' # all stylekey values should be lower than this else: - h_stylekey = '000000' + h_stylekey = '00000000' log.info('h_hud_style: %s' % h_hud_style) #elif h_hud_style == 'H': @@ -1824,11 +1824,11 @@ class Database: """Update cached statistics. If update fails because no record exists, do an insert.""" if self.use_date_in_hudcache: - styleKey = datetime.strftime(starttime, 'd%y%m%d') - #styleKey = "d%02d%02d%02d" % (hand_start_time.year-2000, hand_start_time.month, hand_start_time.day) + styleKey = datetime.strftime(starttime, 'd%y%m%d%h') + #styleKey = "d%02d%02d%02d%02d" % (hand_start_time.year-2000, hand_start_time.month, hand_start_time.day, hand_start_time.hour) else: - # hard-code styleKey as 'A000000' (all-time cache, no key) for now - styleKey = 'A000000' + # hard-code styleKey as 'A00000000' (all-time cache, no key) for now + styleKey = 'A00000000' update_hudcache = self.sql.query['update_hudcache'] update_hudcache = update_hudcache.replace('%s', self.sql.query['placeholder']) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index d45ccd38..560930c3 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -2047,7 +2047,7 @@ class Sql: # gets a date, would need to use handsplayers (not hudcache) to get exact hand Id if db_server == 'mysql': self.query['get_date_nhands_ago'] = """ - select concat( 'd', date_format(max(h.startTime), '%Y%m%d') ) + select concat( 'd', date_format(max(h.startTime), '%Y%m%d%h') ) from (select hp.playerId ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx from HandsPlayers hp @@ -2059,7 +2059,7 @@ class Sql: """ elif db_server == 'postgresql': self.query['get_date_nhands_ago'] = """ - select 'd' || to_char(max(h3.startTime), 'YYMMDD') + select 'd' || to_char(max(h3.startTime), 'YYMMDDHH') from (select hp.playerId ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx from HandsPlayers hp @@ -2071,7 +2071,7 @@ class Sql: """ elif db_server == 'sqlite': # untested guess at query: self.query['get_date_nhands_ago'] = """ - select 'd' || strftime(max(h3.startTime), 'YYMMDD') + select 'd' || strftime(max(h3.startTime), 'YYMMDDHH') from (select hp.playerId ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx from HandsPlayers hp @@ -3290,7 +3290,7 @@ class Sql: else 'E' end AS hc_position - ,date_format(h.startTime, 'd%y%m%d') + ,date_format(h.startTime, 'd%y%m%d%h') ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3379,7 +3379,7 @@ class Sql: ,h.seats ,hc_position - ,date_format(h.startTime, 'd%y%m%d') + ,date_format(h.startTime, 'd%y%m%d%h') """ elif db_server == 'postgresql': self.query['rebuildHudCache'] = """ @@ -3488,7 +3488,7 @@ class Sql: else 'E' end AS hc_position - ,'d' || to_char(h.startTime, 'YYMMDD') + ,'d' || to_char(h.startTime, 'YYMMDDHH') ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3577,7 +3577,7 @@ class Sql: ,h.seats ,hc_position - ,to_char(h.startTime, 'YYMMDD') + ,to_char(h.startTime, 'YYMMDDHH') """ else: # assume sqlite self.query['rebuildHudCache'] = """ @@ -3686,7 +3686,7 @@ class Sql: else 'E' end AS hc_position - ,'d' || substr(strftime('%Y%m%d', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%h', h.startTime),3,7) ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3775,7 +3775,7 @@ class Sql: ,h.seats ,hc_position - ,'d' || substr(strftime('%Y%m%d', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%h', h.startTime),3,7) """ self.query['insert_hudcache'] = """ From ee926ac9ca84e8c07b72673a2d1c6c5f3c720d38 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:18:42 -0600 Subject: [PATCH 02/14] * Modified init_hud_stat_vars() so that it uses the timezone & day_start offset from Filters.py --- pyfpdb/Database.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 1fb94cbd..e5cdfa6b 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -55,6 +55,7 @@ import Card import Charset from Exceptions import * import Configuration +import Filters # Other library modules @@ -689,14 +690,16 @@ class Database: else: if row and row[0]: self.hand_1day_ago = int(row[0]) - - d = timedelta(days=hud_days) + + offset = strptime(Filters.Filters(self, self.config, self.sql).getDates()[0],"%Y-%m-%d %H:%M:%S").tm_hour + + d = timedelta(days=hud_days, hours=offset) now = datetime.utcnow() - d - self.date_ndays_ago = "d%02d%02d%02d" % (now.year - 2000, now.month, now.day) + self.date_ndays_ago = "d%02d%02d%02d%02d" % (now.year - 2000, now.month, now.day, offset) - d = timedelta(days=h_hud_days) + d = timedelta(days=h_hud_days, hours=offset) now = datetime.utcnow() - d - self.h_date_ndays_ago = "d%02d%02d%02d" % (now.year - 2000, now.month, now.day) + self.h_date_ndays_ago = "d%02d%02d%02d%02d" % (now.year - 2000, now.month, now.day, offset) def init_player_hud_stat_vars(self, playerid): # not sure if this is workable, to be continued ... From 8026ba256a1b29f6f7e66724098a0e5e6c8c0d63 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:41:12 -0600 Subject: [PATCH 03/14] It's %H, not %h --- pyfpdb/Database.py | 2 +- pyfpdb/SQL.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index e5cdfa6b..f2ce36b5 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1827,7 +1827,7 @@ class Database: """Update cached statistics. If update fails because no record exists, do an insert.""" if self.use_date_in_hudcache: - styleKey = datetime.strftime(starttime, 'd%y%m%d%h') + styleKey = datetime.strftime(starttime, 'd%y%m%d%H') #styleKey = "d%02d%02d%02d%02d" % (hand_start_time.year-2000, hand_start_time.month, hand_start_time.day, hand_start_time.hour) else: # hard-code styleKey as 'A00000000' (all-time cache, no key) for now diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 560930c3..ffe1ee6d 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -2047,7 +2047,7 @@ class Sql: # gets a date, would need to use handsplayers (not hudcache) to get exact hand Id if db_server == 'mysql': self.query['get_date_nhands_ago'] = """ - select concat( 'd', date_format(max(h.startTime), '%Y%m%d%h') ) + select concat( 'd', date_format(max(h.startTime), '%Y%m%d%H') ) from (select hp.playerId ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx from HandsPlayers hp @@ -3290,7 +3290,7 @@ class Sql: else 'E' end AS hc_position - ,date_format(h.startTime, 'd%y%m%d%h') + ,date_format(h.startTime, 'd%y%m%d%H') ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3379,7 +3379,7 @@ class Sql: ,h.seats ,hc_position - ,date_format(h.startTime, 'd%y%m%d%h') + ,date_format(h.startTime, 'd%y%m%d%H') """ elif db_server == 'postgresql': self.query['rebuildHudCache'] = """ @@ -3686,7 +3686,7 @@ class Sql: else 'E' end AS hc_position - ,'d' || substr(strftime('%Y%m%d%h', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,7) ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3775,7 +3775,7 @@ class Sql: ,h.seats ,hc_position - ,'d' || substr(strftime('%Y%m%d%h', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,7) """ self.query['insert_hudcache'] = """ From b820cd4895ac6039fbe29df22efc3b2972f2d949 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:49:39 -0600 Subject: [PATCH 04/14] * Old Stars HHs don't display max seats. Added regression-test-file showing this and handling in the Stars HHC --- pyfpdb/PokerStarsToFpdb.py | 2 +- .../LHE-10-max-USD-1.00-2.00-No_max_seats.txt | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt diff --git a/pyfpdb/PokerStarsToFpdb.py b/pyfpdb/PokerStarsToFpdb.py index 16c4091e..3817ae3f 100644 --- a/pyfpdb/PokerStarsToFpdb.py +++ b/pyfpdb/PokerStarsToFpdb.py @@ -298,7 +298,7 @@ class PokerStars(HandHistoryConverter): if key == 'BUTTON': hand.buttonpos = info[key] if key == 'MAX': - hand.maxseats = int(info[key]) + if info[key]: hand.maxseats = int(info[key]) if key == 'MIXED': hand.mixed = self.mixes[info[key]] if info[key] is not None else None diff --git a/pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt b/pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt new file mode 100644 index 00000000..173da216 --- /dev/null +++ b/pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt @@ -0,0 +1,42 @@ +PokerStars Game #2428142447: Hold'em Limit ($1/$2 USD) - 2005/08/26 16:12:22 ET +Table 'Teucer II' Seat #5 is the button +Seat 1: Frankson34 ($24 in chips) +Seat 2: webb22 ($64 in chips) +Seat 3: eric_mtx ($26 in chips) +Seat 4: sososolid ($147.75 in chips) +Seat 5: DRILHER ($48.25 in chips) +Seat 6: Naughtychic ($60 in chips) +Seat 7: Terps78 ($71.50 in chips) +Seat 8: ChazDazzle ($69.25 in chips) +Seat 9: alekos ($55 in chips) +Seat 10: BigNards84 ($64.25 in chips) +Naughtychic: posts small blind $0.50 +Terps78: posts big blind $1 +*** HOLE CARDS *** +Dealt to ChazDazzle [8c Kd] +ChazDazzle: folds +alekos: folds +BigNards84: raises $1 to $2 +Frankson34: folds +webb22: folds +eric_mtx: folds +ChazDazzle leaves the table +sososolid: folds +DRILHER: folds +cdhender joins the table at seat #8 +Naughtychic: folds +Terps78: folds +BigNards84 collected $2.50 from pot +BigNards84: doesn't show hand +*** SUMMARY *** +Total pot $2.50 | Rake $0 +Seat 1: Frankson34 folded before Flop (didn't bet) +Seat 2: webb22 folded before Flop (didn't bet) +Seat 3: eric_mtx folded before Flop (didn't bet) +Seat 4: sososolid folded before Flop (didn't bet) +Seat 5: DRILHER (button) folded before Flop (didn't bet) +Seat 6: Naughtychic (small blind) folded before Flop +Seat 7: Terps78 (big blind) folded before Flop +Seat 8: ChazDazzle folded before Flop (didn't bet) +Seat 9: alekos folded before Flop (didn't bet) +Seat 10: BigNards84 collected ($2.50) From 43ec9498e8d08db49cf86bbaade16948b7f5dc31 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:51:13 -0600 Subject: [PATCH 05/14] Just renamed the new regression file --- ...-No_max_seats.txt => LHE-10max-USD-1.00-2.00-No_max_seats.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pyfpdb/regression-test-files/cash/Stars/Flop/{LHE-10-max-USD-1.00-2.00-No_max_seats.txt => LHE-10max-USD-1.00-2.00-No_max_seats.txt} (100%) diff --git a/pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt b/pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10max-USD-1.00-2.00-No_max_seats.txt similarity index 100% rename from pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10-max-USD-1.00-2.00-No_max_seats.txt rename to pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10max-USD-1.00-2.00-No_max_seats.txt From 92c8e689146efd9bab24d8ba74c9eeba07699b5a Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 14:53:53 -0600 Subject: [PATCH 06/14] It's actually an OSError, not an IOError --- pyfpdb/interlocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 pyfpdb/interlocks.py diff --git a/pyfpdb/interlocks.py b/pyfpdb/interlocks.py old mode 100755 new mode 100644 index c2e6abd6..496c5d0b --- a/pyfpdb/interlocks.py +++ b/pyfpdb/interlocks.py @@ -108,7 +108,7 @@ class InterProcessLockFcntl(InterProcessLockBase): self.lockfd = 0 try: os.unlink(self.lock_file_name) - except IOError: + except OSError: # We don't care about the existence of the file too much here. It's the flock() we care about, # And that should just go away magically. pass From ead567c5cc8c5783cdf1236d4312fe6ce22f5539 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 15:13:26 -0600 Subject: [PATCH 07/14] * Added encoding error handling and the ability to identify the worker responsible for splitting a particular file if several were launched via threading --- pyfpdb/SplitHandHistory.py | 76 +++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/pyfpdb/SplitHandHistory.py b/pyfpdb/SplitHandHistory.py index 52c1d340..c996f903 100644 --- a/pyfpdb/SplitHandHistory.py +++ b/pyfpdb/SplitHandHistory.py @@ -1,19 +1,22 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- +#!/usr/bin/env python -#Copyright 2010 Chaz Littlejohn -#This program is free software: you can redistribute it and/or modify -#it under the terms of the GNU Affero General Public License as published by -#the Free Software Foundation, version 3 of the License. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -#GNU General Public License for more details. -# -#You should have received a copy of the GNU Affero General Public License -#along with this program. If not, see . -#In the "official" distribution you can find the license in agpl-3.0.txt. +# Copyright 2010, Chaz Littlejohn +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +######################################################################## import L10n _ = L10n.get_translation() @@ -28,6 +31,7 @@ import Options import Configuration from Exceptions import * from cStringIO import StringIO +import time (options, argv) = Options.fpdb_options() @@ -37,7 +41,7 @@ codepage = ["utf-16", "utf-8", "cp1252"] class SplitHandHistory: - def __init__(self, config, in_path = '-', out_path = None, hands = 100, filter = "PokerStarsToFpdb", archive = False): + def __init__(self, config, in_path = '-', out_path = None, hands = 100, filter = "PokerStarsToFpdb", archive = False, workerid = 0): self.config = config self.in_path = in_path self.out_path = out_path @@ -50,22 +54,25 @@ class SplitHandHistory: self.line_addendum = None self.filedone = False + self.timestamp = str(time.time()) + self.workerid = '%02d' % workerid + #Acquire re_SplitHands for this hh - filter_name = filter.replace("ToFpdb", "") + self.filter_name = filter.replace("ToFpdb", "") mod = __import__(filter) - obj = getattr(mod, filter_name, None) + obj = getattr(mod, self.filter_name, None) self.re_SplitHands = obj.re_SplitHands #Determine line delimiter type if any - if self.re_SplitHands.match('\n\n'): - self.line_delimiter = '\n\n' if self.re_SplitHands.match('\n\n\n'): self.line_delimiter = '\n\n\n' + if self.re_SplitHands.match('\n\n'): + self.line_delimiter = '\n\n' #Add new line addendum for sites which match SplitHand to next line as well - if filter_name == 'OnGame': + if self.filter_name == 'OnGame': self.line_addendum = '*' - if filter_name == 'Carbon': + if self.filter_name == 'Carbon': self.line_addendum = ' %s' % name newfile = file(name, 'w') + os.chmod(name, 0775) return newfile #Archive Hand Splitter @@ -122,8 +131,11 @@ class SplitHandHistory: except FpdbEndOfFile: done = True break + except UnicodeEncodeError: + print _('Absurd character done messed you up') + sys.exit(2) except: - print _("Unexpected error processing file") + print _('Unexpected error processing file') sys.exit(2) n += 1 outfile.close() @@ -174,7 +186,7 @@ class SplitHandHistory: l = infile.readline() l = l.replace('\r\n', '\n') outfile.write(l) - l = infile.readline() + l = infile.readline().encode(self.kodec) while len(l) < 3: l = infile.readline() @@ -182,7 +194,7 @@ class SplitHandHistory: while len(l) > 2: l = l.replace('\r\n', '\n') outfile.write(l) - l = infile.readline() + l = infile.readline().encode(self.kodec) outfile.write(self.line_delimiter) return infile @@ -195,13 +207,19 @@ class SplitHandHistory: def main(argv=None): if argv is None: argv = sys.argv[1:] + + if not options.filename: + options.filename = sys.argv[1] if not options.config: - options.config = Configuration.Config(file = "HUD_config.test.xml") - + options.config = sys.argv[2] + + if sys.argv[3] == "True": + options.archive = True + if options.filename: SplitHH = SplitHandHistory(options.config, options.filename, options.outpath, options.hands, - options.hhc, options.archive) + options.hhc, options.archive, options.workerid) if __name__ == '__main__': sys.exit(main()) From 9bb58ca77950ba56bf5216cf5fd4c3a2ccbf9c14 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 15:16:09 -0600 Subject: [PATCH 08/14] Adds the worker id command line option --- pyfpdb/Options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyfpdb/Options.py b/pyfpdb/Options.py index b8be3d2a..6369b485 100644 --- a/pyfpdb/Options.py +++ b/pyfpdb/Options.py @@ -59,6 +59,8 @@ def fpdb_options(): help=_("File to be split is a PokerStars or Full Tilt Poker archive file")) parser.add_option("-n", "--numhands", dest="hands", default="100", type="int", help=_("How many hands do you want saved to each file. Default is 100")) + parser.add_option("-w", "--workerid", dest="workerid", default="0", type="int", + help=_("Specifies the worker id running the script")) (options, argv) = parser.parse_args() From 9137347acb00957d548ade66d65687d91be15236 Mon Sep 17 00:00:00 2001 From: Chaz Date: Wed, 24 Nov 2010 15:20:48 -0600 Subject: [PATCH 09/14] Just fixed a few bugs --- pyfpdb/IdentifySite.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pyfpdb/IdentifySite.py b/pyfpdb/IdentifySite.py index b4f1a8a7..723991f3 100644 --- a/pyfpdb/IdentifySite.py +++ b/pyfpdb/IdentifySite.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # -*- coding: utf-8 -*- #Copyright 2010 Chaz Littlejohn @@ -28,18 +28,21 @@ import Configuration import Database __ARCHIVE_PRE_HEADER_REGEX='^Hand #(\d+)\s*$|\*{20}\s#\s\d+\s\*+\s+' -re_SplitArchive = re.compile(__ARCHIVE_PRE_HEADER_REGEX) +re_SplitArchive = re.compile(__ARCHIVE_PRE_HEADER_REGEX, re.MULTILINE) class IdentifySite: def __init__(self, config, in_path = '-'): self.in_path = in_path self.config = config - self.db = Database.Database(config) + self.db = Database.Database(self.config) self.sitelist = {} self.filelist = {} self.generateSiteList() - self.walkDirectory(self.in_path, self.sitelist) + if os.path.isdir(self.in_path): + self.walkDirectory(self.in_path, self.sitelist) + else: + self.idSite(self.in_path, self.sitelist) def generateSiteList(self): """Generates a ordered dictionary of site, filter and filter name for each site in hhcs""" @@ -80,7 +83,7 @@ class IdentifySite: for kodec in self.__listof(obj.codepage): try: in_fh = codecs.open(file, 'r', kodec) - whole_file = in_fh.read() + whole_file = in_fh.read(2000) in_fh.close() if info[2] in ('OnGame', 'Winamax'): @@ -94,7 +97,7 @@ class IdentifySite: if re_SplitArchive.search(whole_file): archive = True if m: - self.filelist[file] = [info[0]] + [info[1]] + [kodec] + [archive] + self.filelist[file] = [info[1]] + [kodec] + [archive] break except: pass From 872ea008ddb1205eef9a5fc07c7659c365767041 Mon Sep 17 00:00:00 2001 From: Chaz Date: Thu, 25 Nov 2010 02:35:50 -0600 Subject: [PATCH 10/14] Updated database version to 145 --- pyfpdb/Database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index f2ce36b5..65e7a87d 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -74,7 +74,7 @@ except ImportError: use_numpy = False -DB_VERSION = 144 +DB_VERSION = 145 # Variance created as sqlite has a bunch of undefined aggregate functions. From 21ac5229cba63068408ce975c2db26b431499f2d Mon Sep 17 00:00:00 2001 From: Chaz Date: Thu, 25 Nov 2010 02:36:34 -0600 Subject: [PATCH 11/14] Fixed styleKey char length to 9 --- pyfpdb/SQL.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index ffe1ee6d..6a505eae 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -1064,7 +1064,7 @@ class Sql: activeSeats SMALLINT NOT NULL, position CHAR(1), tourneyTypeId SMALLINT UNSIGNED, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), - styleKey CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + styleKey CHAR(9) NOT NULL, /* 1st char is style (A/T/H/S), other 8 are the key */ HDs INT NOT NULL, wonWhenSeenStreet1 FLOAT, @@ -1165,7 +1165,7 @@ class Sql: activeSeats SMALLINT, position CHAR(1), tourneyTypeId INT, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id), - styleKey CHAR(7) NOT NULL, /* 1st char is style (A/T/H/S), other 6 are the key */ + styleKey CHAR(9) NOT NULL, /* 1st char is style (A/T/H/S), other 8 are the key */ HDs INT, wonWhenSeenStreet1 FLOAT, @@ -3686,7 +3686,7 @@ class Sql: else 'E' end AS hc_position - ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,9) ,count(1) ,sum(wonWhenSeenStreet1) ,sum(wonWhenSeenStreet2) @@ -3775,7 +3775,7 @@ class Sql: ,h.seats ,hc_position - ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,7) + ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,9) """ self.query['insert_hudcache'] = """ From b017fa298d9e8cc0ffd232cb8597439a5fdf8187 Mon Sep 17 00:00:00 2001 From: Worros Date: Thu, 25 Nov 2010 20:11:19 +0800 Subject: [PATCH 12/14] Carbon: Fix regression test parse --- pyfpdb/CarbonToFpdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyfpdb/CarbonToFpdb.py b/pyfpdb/CarbonToFpdb.py index 1a1541ee..a2d2f56f 100644 --- a/pyfpdb/CarbonToFpdb.py +++ b/pyfpdb/CarbonToFpdb.py @@ -133,8 +133,9 @@ or None if we fail to get the info """ self.info = {} mg = m.groupdict() + print mg - limits = { 'No Limit':'nl', 'Limit':'fl' } + limits = { 'No Limit':'nl', 'No Limit ':'nl', 'Limit':'fl' } games = { # base, category 'Holdem' : ('hold','holdem'), 'Holdem Tournament' : ('hold','holdem') } From 31da6ff170997742b7231195edaeae185dac85d0 Mon Sep 17 00:00:00 2001 From: Worros Date: Fri, 26 Nov 2010 12:45:41 +0800 Subject: [PATCH 13/14] Winamax: Pull in tournament support from Forrest Also add tournament directory to regression test file --- pyfpdb/TestHandsPlayers.py | 1 + pyfpdb/WinamaxToFpdb.py | 81 +++++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/pyfpdb/TestHandsPlayers.py b/pyfpdb/TestHandsPlayers.py index ee3be1e9..e0013ecc 100755 --- a/pyfpdb/TestHandsPlayers.py +++ b/pyfpdb/TestHandsPlayers.py @@ -201,6 +201,7 @@ def main(argv=None): walk_testfiles("regression-test-files/cash/iPoker/", compare, importer, iPokerErrors, "iPoker") if sites['Winamax'] == True: walk_testfiles("regression-test-files/cash/Winamax/", compare, importer, WinamaxErrors, "Winamax") + walk_testfiles("regression-test-files/tour/Winamax/", compare, importer, WinamaxErrors, "Winamax") if sites['Win2day'] == True: walk_testfiles("regression-test-files/cash/Win2day/", compare, importer, Win2dayErrors, "Win2day") diff --git a/pyfpdb/WinamaxToFpdb.py b/pyfpdb/WinamaxToFpdb.py index c49f4e09..47721515 100644 --- a/pyfpdb/WinamaxToFpdb.py +++ b/pyfpdb/WinamaxToFpdb.py @@ -82,15 +82,26 @@ class Winamax(HandHistoryConverter): # Winamax Poker - CashGame - HandId: #279823-223-1285031451 - Holdem no limit (0.02€/0.05€) - 2010/09/21 03:10:51 UTC # Table: 'Charenton-le-Pont' 9-max (real money) Seat #5 is the button re_HandInfo = re.compile(u""" - \s*Winamax\sPoker\s-\sCashGame\s-\sHandId:\s\#(?P\d+)-(?P\d+)-(?P\d+).*\s + \s*Winamax\sPoker\s-\s + (?PCashGame)? + (?PTournament\s + (?P.+)?\s + buyIn:\s(?P(?P[%(LS)s\d\,]+)?\s\+?\s(?P[%(LS)s\d\,]+)?\+?(?P[%(LS)s\d\.]+)?\s?(?P%(LEGAL_ISO)s)?|Gratuit|Ticket\suniquement)?\s + (level:\s(?P\d+))? + .*)? + \s-\sHandId:\s\#(?P\d+)-(?P\d+)-(?P\d+).*\s (?PHoldem|Omaha)\s (?Pno\slimit|pot\slimit)\s \( + (((%(LS)s)?(?P[.0-9]+)(%(LS)s)?)/)? ((%(LS)s)?(?P[.0-9]+)(%(LS)s)?)/ ((%(LS)s)?(?P[.0-9]+)(%(LS)s)?) \)\s-\s (?P.*) - Table:\s\'(?P[^']+)\'\s(?P\d+)\-max + Table:\s\'(?P
[^(]+) + (.(?P\d+).\#(?P\d+))?.* + \' + \s(?P\d+)\-max """ % substitutions, re.MULTILINE|re.DOTALL|re.VERBOSE) re_TailSplitHands = re.compile(r'\n\s*\n') @@ -126,8 +137,8 @@ class Winamax(HandHistoryConverter): self.re_PostSB = re.compile('%(PLYR)s posts small blind (%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?' % subst, re.MULTILINE) self.re_PostBB = re.compile('%(PLYR)s posts big blind (%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?' % subst, re.MULTILINE) self.re_DenySB = re.compile('(?P.*) deny SB' % subst, re.MULTILINE) - self.re_Antes = re.compile(r"^%(PLYR)s: posts the ante (%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?" % subst, re.MULTILINE) - self.re_BringIn = re.compile(r"^%(PLYR)s: brings[- ]in( low|) for (%(CUR)s)?(?P[\.0-9]+(%(CUR)s)?)" % subst, re.MULTILINE) + self.re_Antes = re.compile(r"^%(PLYR)s posts ante (%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?" % subst, re.MULTILINE) + self.re_BringIn = re.compile(r"^%(PLYR)s brings[- ]in( low|) for (%(CUR)s)?(?P[\.0-9]+(%(CUR)s)?)" % subst, re.MULTILINE) self.re_PostBoth = re.compile('(?P.*): posts small \& big blind \( (%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?\)' % subst) self.re_PostDead = re.compile('(?P.*) posts dead blind \((%(CUR)s)?(?P[\.0-9]+)(%(CUR)s)?\)' % subst, re.MULTILINE) self.re_HeroCards = re.compile('Dealt\sto\s%(PLYR)s\s\[(?P.*)\]' % subst) @@ -144,6 +155,9 @@ class Winamax(HandHistoryConverter): ["ring", "hold", "fl"], ["ring", "hold", "nl"], ["ring", "hold", "pl"], + ["tour", "hold", "fl"], + ["tour", "hold", "nl"], + ["tour", "hold", "pl"], ] def determineGameType(self, handText): @@ -160,7 +174,11 @@ class Winamax(HandHistoryConverter): mg = m.groupdict() - info['type'] = 'ring' + if mg.get('TOUR'): + info['type'] = 'tour' + elif mg.get('RING'): + info['type'] = 'ring' + info['currency'] = 'EUR' if 'LIMIT' in mg: @@ -202,11 +220,62 @@ class Winamax(HandHistoryConverter): hand.startTime = datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S") # also timezone at end, e.g. " ET" hand.startTime = HandHistoryConverter.changeTimezone(hand.startTime, "CET", "UTC") if key == 'HID1': - hand.handid = "1%.4d%s%s"%(int(info['HID2']),info['HID1'],info['HID3']) # Need to remove non-alphanumerics for MySQL + hand.handid = "1%.9d%s%s"%(int(info['HID2']),info['HID1'],info['HID3']) + if key == 'TOURNO': + hand.tourNo = info[key] if key == 'TABLE': hand.tablename = info[key] + if key == 'BUYIN': + if hand.tourNo!=None: + #print "DEBUG: info['BUYIN']: %s" % info['BUYIN'] + #print "DEBUG: info['BIAMT']: %s" % info['BIAMT'] + #print "DEBUG: info['BIRAKE']: %s" % info['BIRAKE'] + #print "DEBUG: info['BOUNTY']: %s" % info['BOUNTY'] + for k in ['BIAMT','BIRAKE']: + if k in info.keys() and info[k]: + info[k] = info[k].replace(',','.') + + if info[key] == 'Freeroll': + hand.buyin = 0 + hand.fee = 0 + hand.buyinCurrency = "FREE" + else: + if info[key].find("$")!=-1: + hand.buyinCurrency="USD" + elif info[key].find(u"€")!=-1: + hand.buyinCurrency="EUR" + elif info[key].find("FPP")!=-1: + hand.buyinCurrency="PSFP" + else: + #FIXME: handle other currencies, FPP, play money + raise FpdbParseError(_("failed to detect currency")) + + info['BIAMT'] = info['BIAMT'].strip(u'$€FPP') + + if hand.buyinCurrency!="PSFP": + if info['BOUNTY'] != None: + # There is a bounty, Which means we need to switch BOUNTY and BIRAKE values + tmp = info['BOUNTY'] + info['BOUNTY'] = info['BIRAKE'] + info['BIRAKE'] = tmp + info['BOUNTY'] = info['BOUNTY'].strip(u'$€') # Strip here where it isn't 'None' + hand.koBounty = int(100*Decimal(info['BOUNTY'])) + hand.isKO = True + else: + hand.isKO = False + + info['BIRAKE'] = info['BIRAKE'].strip(u'$€') + hand.buyin = int(100*Decimal(info['BIAMT'])) + hand.fee = int(100*Decimal(info['BIRAKE'])) + else: + hand.buyin = int(Decimal(info['BIAMT'])) + hand.fee = 0 + + if key == 'LEVEL': + hand.level = info[key] + # TODO: These hand.buttonpos = 1 hand.maxseats = 10 # Set to None - Hand.py will guessMaxSeats() From 9098c3ba036aa13ce786f4d1bae11cd8a1b9f348 Mon Sep 17 00:00:00 2001 From: Worros Date: Fri, 26 Nov 2010 13:38:39 +0800 Subject: [PATCH 14/14] Regression: Party PLO tourney --- .../Flop/PLO-USD-STT-1.00.201008.Sample.txt | Bin 0 -> 1006 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyfpdb/regression-test-files/tour/PartyPoker/Flop/PLO-USD-STT-1.00.201008.Sample.txt diff --git a/pyfpdb/regression-test-files/tour/PartyPoker/Flop/PLO-USD-STT-1.00.201008.Sample.txt b/pyfpdb/regression-test-files/tour/PartyPoker/Flop/PLO-USD-STT-1.00.201008.Sample.txt new file mode 100644 index 0000000000000000000000000000000000000000..a57df9c2e6cd7adf203fd83a6eb5e4d74dd64786 GIT binary patch literal 1006 zcmah{%Wm5+5bRli{$YUh;23g4y=+If25#cCX&WSJFGdfvL`MW<(h#W_{`)RvtCiD3 z1&9QOJF_$7-tn!##dSK5B9=17U@UK}`QUrLcX8(qziDI`6!_lQeOPi;VkwQ)%>f(T zpsOAfc4fZz_<-+QUU9q+@#A@coAxk~3VEXr8PX)C#?5!k{<$mY+qVq5wiecnx!lw+NVX1h=t7ea=xwwjw`|$o24m6PlDHuFAkT!_bj-@ z6w<&}0_)aVt%#>B)u?v677g?UyP6+FV<>ZcHN}Y)X80-Vnv?GlW8jQ!j=U@w~k4;y=)F7#l z(^}#e-S(+#!=(`ht2kwZ_T~#-*A8S3>};Qqs4E+^y{+|*8e)aW1W2m@QQ`D^rM0jm zL{Ilts6%=e8(o*?EKc4w>Ztp<^Lws0Fh!|RB~sqY4o<6Fe6KWowf+Em$t${>#c`#sbC)eBer$3=fwUe)y3IJB~2wt5Gi&e KT?k70&if5Q=ON$# literal 0 HcmV?d00001