Merge branch 'master' of git://git.assembla.com/fpdboz.git

pyfpdb/CarbonToFpdb.py

@@ -133,8 +133,9 @@ or None if we fail to get the info """
 
         self.info = {}
         mg = m.groupdict()
+        print mg
 
-        limits = { 'No Limit':'nl', 'Limit':'fl' }
+        limits = { 'No Limit':'nl', 'No Limit ':'nl', 'Limit':'fl' }
         games = {              # base, category
                     'Holdem' : ('hold','holdem'),
          'Holdem Tournament' : ('hold','holdem') }

pyfpdb/Database.py

@@ -55,6 +55,7 @@ import Card
 import Charset
 from Exceptions import *
 import Configuration
+import Filters
 
 
 #    Other library modules
@@ -73,7 +74,7 @@ except ImportError:
     use_numpy = False
 
 
-DB_VERSION = 144
+DB_VERSION = 145
 
 
 # Variance created as sqlite has a bunch of undefined aggregate functions.
@@ -291,8 +292,8 @@ class Database:
 
             # vars for hand ids or dates fetched according to above config:
             self.hand_1day_ago = 0             # max hand id more than 24 hrs earlier than now
-            self.date_ndays_ago = 'd000000'    # date N days ago ('d' + YYMMDD)
+            self.date_ndays_ago = 'd00000000'    # date N days ago ('d' + YYMMDD)
-            self.h_date_ndays_ago = 'd000000'  # date N days ago ('d' + YYMMDD) for hero
+            self.h_date_ndays_ago = 'd00000000'  # date N days ago ('d' + YYMMDD) for hero
             self.date_nhands_ago = {}          # dates N hands ago per player - not used yet
 
             self.saveActions = False if self.import_options['saveActions'] == False else True
@@ -690,20 +691,22 @@ class Database:
             if row and row[0]:
                 self.hand_1day_ago = int(row[0])
         
-        d = timedelta(days=hud_days)
+        offset = strptime(Filters.Filters(self, self.config, self.sql).getDates()[0],"%Y-%m-%d %H:%M:%S").tm_hour
-        now = datetime.utcnow() - d
-        self.date_ndays_ago = "d%02d%02d%02d" % (now.year - 2000, now.month, now.day)
         
-        d = timedelta(days=h_hud_days)
+        d = timedelta(days=hud_days, hours=offset)
         now = datetime.utcnow() - d
-        self.h_date_ndays_ago = "d%02d%02d%02d" % (now.year - 2000, now.month, now.day)
+        self.date_ndays_ago = "d%02d%02d%02d%02d" % (now.year - 2000, now.month, now.day, offset)
+
+        d = timedelta(days=h_hud_days, hours=offset)
+        now = datetime.utcnow() - d
+        self.h_date_ndays_ago = "d%02d%02d%02d%02d" % (now.year - 2000, now.month, now.day, offset)
 
     def init_player_hud_stat_vars(self, playerid):
         # not sure if this is workable, to be continued ...
         try:
             # self.date_nhands_ago is used for fetching stats for last n hands (hud_style = 'H')
             # This option not used yet - needs to be called for each player :-(
-            self.date_nhands_ago[str(playerid)] = 'd000000'
+            self.date_nhands_ago[str(playerid)] = 'd00000000'
 
             # should use aggregated version of query if appropriate
             c.execute(self.sql.query['get_date_nhands_ago'], (self.hud_hands, playerid))
@@ -771,11 +774,11 @@ class Database:
         if hud_style == 'T':
             stylekey = self.date_ndays_ago
         elif hud_style == 'A':
-            stylekey = '0000000'  # all stylekey values should be higher than this
+            stylekey = '000000000'  # all stylekey values should be higher than this
         elif hud_style == 'S':
-            stylekey = 'zzzzzzz'  # all stylekey values should be lower than this
+            stylekey = 'zzzzzzzzz'  # all stylekey values should be lower than this
         else:
-            stylekey = '0000000'
+            stylekey = '000000000'
             log.info('hud_style: %s' % hud_style)
 
         #elif hud_style == 'H':
@@ -784,11 +787,11 @@ class Database:
         if h_hud_style == 'T':
             h_stylekey = self.h_date_ndays_ago
         elif h_hud_style == 'A':
-            h_stylekey = '0000000'  # all stylekey values should be higher than this
+            h_stylekey = '000000000'  # all stylekey values should be higher than this
         elif h_hud_style == 'S':
-            h_stylekey = 'zzzzzzz'  # all stylekey values should be lower than this
+            h_stylekey = 'zzzzzzzzz'  # all stylekey values should be lower than this
         else:
-            h_stylekey = '000000'
+            h_stylekey = '00000000'
             log.info('h_hud_style: %s' % h_hud_style)
 
         #elif h_hud_style == 'H':
@@ -1824,11 +1827,11 @@ class Database:
         """Update cached statistics. If update fails because no record exists, do an insert."""
 
         if self.use_date_in_hudcache:
-            styleKey = datetime.strftime(starttime, 'd%y%m%d')
+            styleKey = datetime.strftime(starttime, 'd%y%m%d%H')
-            #styleKey = "d%02d%02d%02d" % (hand_start_time.year-2000, hand_start_time.month, hand_start_time.day)
+            #styleKey = "d%02d%02d%02d%02d" % (hand_start_time.year-2000, hand_start_time.month, hand_start_time.day, hand_start_time.hour)
         else:
-            # hard-code styleKey as 'A000000' (all-time cache, no key) for now
+            # hard-code styleKey as 'A00000000' (all-time cache, no key) for now
-            styleKey = 'A000000'
+            styleKey = 'A00000000'
 
         update_hudcache = self.sql.query['update_hudcache']
         update_hudcache = update_hudcache.replace('%s', self.sql.query['placeholder'])

pyfpdb/IdentifySite.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 #Copyright 2010 Chaz Littlejohn
@@ -28,18 +28,21 @@ import Configuration
 import Database
 
 __ARCHIVE_PRE_HEADER_REGEX='^Hand #(\d+)\s*$|\*{20}\s#\s\d+\s\*+\s+'
-re_SplitArchive = re.compile(__ARCHIVE_PRE_HEADER_REGEX)
+re_SplitArchive = re.compile(__ARCHIVE_PRE_HEADER_REGEX, re.MULTILINE)
 
 
 class IdentifySite:
     def __init__(self, config, in_path = '-'):
         self.in_path = in_path
         self.config = config
-        self.db = Database.Database(config)
+        self.db = Database.Database(self.config)
         self.sitelist = {}
         self.filelist = {}
         self.generateSiteList()
+        if os.path.isdir(self.in_path):
             self.walkDirectory(self.in_path, self.sitelist)
+        else:
+            self.idSite(self.in_path, self.sitelist)
         
     def generateSiteList(self):
         """Generates a ordered dictionary of site, filter and filter name for each site in hhcs"""
@@ -80,7 +83,7 @@ class IdentifySite:
                 for kodec in self.__listof(obj.codepage):
                     try:
                         in_fh = codecs.open(file, 'r', kodec)
-                        whole_file = in_fh.read()
+                        whole_file = in_fh.read(2000)
                         in_fh.close()
                 
                         if info[2] in ('OnGame', 'Winamax'):
@@ -94,7 +97,7 @@ class IdentifySite:
                             if re_SplitArchive.search(whole_file):
                                 archive = True
                         if m:
-                            self.filelist[file] = [info[0]] + [info[1]] + [kodec] + [archive]
+                            self.filelist[file] = [info[1]] + [kodec] + [archive]
                             break
                     except:
                         pass

pyfpdb/Options.py

@@ -59,6 +59,8 @@ def fpdb_options():
                     help=_("File to be split is a PokerStars or Full Tilt Poker archive file"))
     parser.add_option("-n", "--numhands", dest="hands", default="100", type="int",
                     help=_("How many hands do you want saved to each file. Default is 100"))
+    parser.add_option("-w", "--workerid", dest="workerid", default="0", type="int",
+                    help=_("Specifies the worker id running the script"))
 
 
     (options, argv) = parser.parse_args()

pyfpdb/PokerStarsToFpdb.py

@@ -298,7 +298,7 @@ class PokerStars(HandHistoryConverter):
             if key == 'BUTTON':
                 hand.buttonpos = info[key]
             if key == 'MAX':
-                hand.maxseats = int(info[key])
+                if info[key]: hand.maxseats = int(info[key])
 
             if key == 'MIXED':
                 hand.mixed = self.mixes[info[key]] if info[key] is not None else None

pyfpdb/SQL.py

@@ -1064,7 +1064,7 @@ class Sql:
                         activeSeats SMALLINT NOT NULL,
                         position CHAR(1),
                         tourneyTypeId SMALLINT UNSIGNED, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
-                        styleKey CHAR(7) NOT NULL,  /* 1st char is style (A/T/H/S), other 6 are the key */
+                        styleKey CHAR(9) NOT NULL,  /* 1st char is style (A/T/H/S), other 8 are the key */
                         HDs INT NOT NULL,
 
                         wonWhenSeenStreet1 FLOAT,
@@ -1165,7 +1165,7 @@ class Sql:
                         activeSeats SMALLINT,
                         position CHAR(1),
                         tourneyTypeId INT, FOREIGN KEY (tourneyTypeId) REFERENCES TourneyTypes(id),
-                        styleKey CHAR(7) NOT NULL,  /* 1st char is style (A/T/H/S), other 6 are the key */
+                        styleKey CHAR(9) NOT NULL,  /* 1st char is style (A/T/H/S), other 8 are the key */
                         HDs INT,
 
                         wonWhenSeenStreet1 FLOAT,
@@ -2047,7 +2047,7 @@ class Sql:
         # gets a date, would need to use handsplayers (not hudcache) to get exact hand Id
         if db_server == 'mysql':
             self.query['get_date_nhands_ago'] = """
-                select concat( 'd', date_format(max(h.startTime), '%Y%m%d') )
+                select concat( 'd', date_format(max(h.startTime), '%Y%m%d%H') )
                 from (select hp.playerId
                             ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx
                       from HandsPlayers hp
@@ -2059,7 +2059,7 @@ class Sql:
                 """
         elif db_server == 'postgresql':
             self.query['get_date_nhands_ago'] = """
-                select 'd' || to_char(max(h3.startTime), 'YYMMDD')
+                select 'd' || to_char(max(h3.startTime), 'YYMMDDHH')
                 from (select hp.playerId
                             ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx
                       from HandsPlayers hp
@@ -2071,7 +2071,7 @@ class Sql:
                 """
         elif db_server == 'sqlite': # untested guess at query:
             self.query['get_date_nhands_ago'] = """
-                select 'd' || strftime(max(h3.startTime), 'YYMMDD')
+                select 'd' || strftime(max(h3.startTime), 'YYMMDDHH')
                 from (select hp.playerId
                             ,coalesce(greatest(max(hp.handId)-%s,1),1) as maxminusx
                       from HandsPlayers hp
@@ -3290,7 +3290,7 @@ class Sql:
                             else 'E'
                        end                                            AS hc_position
                       <tourney_select_clause>
-                      ,date_format(h.startTime, 'd%y%m%d')
+                      ,date_format(h.startTime, 'd%y%m%d%H')
                       ,count(1)
                       ,sum(wonWhenSeenStreet1)
                       ,sum(wonWhenSeenStreet2)
@@ -3379,7 +3379,7 @@ class Sql:
                         ,h.seats
                         ,hc_position
                         <tourney_group_clause>
-                        ,date_format(h.startTime, 'd%y%m%d')
+                        ,date_format(h.startTime, 'd%y%m%d%H')
 """
         elif db_server == 'postgresql':
             self.query['rebuildHudCache'] = """
@@ -3488,7 +3488,7 @@ class Sql:
                             else 'E'
                        end                                            AS hc_position
                       <tourney_select_clause>
-                      ,'d' || to_char(h.startTime, 'YYMMDD')
+                      ,'d' || to_char(h.startTime, 'YYMMDDHH')
                       ,count(1)
                       ,sum(wonWhenSeenStreet1)
                       ,sum(wonWhenSeenStreet2)
@@ -3577,7 +3577,7 @@ class Sql:
                         ,h.seats
                         ,hc_position
                         <tourney_group_clause>
-                        ,to_char(h.startTime, 'YYMMDD')
+                        ,to_char(h.startTime, 'YYMMDDHH')
 """
         else:   # assume sqlite
             self.query['rebuildHudCache'] = """
@@ -3686,7 +3686,7 @@ class Sql:
                             else 'E'
                        end                                            AS hc_position
                       <tourney_select_clause>
-                      ,'d' || substr(strftime('%Y%m%d', h.startTime),3,7)
+                      ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,9)
                       ,count(1)
                       ,sum(wonWhenSeenStreet1)
                       ,sum(wonWhenSeenStreet2)
@@ -3775,7 +3775,7 @@ class Sql:
                         ,h.seats
                         ,hc_position
                         <tourney_group_clause>
-                        ,'d' || substr(strftime('%Y%m%d', h.startTime),3,7)
+                        ,'d' || substr(strftime('%Y%m%d%H', h.startTime),3,9)
 """
 
         self.query['insert_hudcache'] = """

pyfpdb/SplitHandHistory.py

@@ -1,19 +1,22 @@
-#!/usr/bin/python
+#!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
-#Copyright 2010 Chaz Littlejohn
+#    Copyright 2010, Chaz Littlejohn
-#This program is free software: you can redistribute it and/or modify
-#it under the terms of the GNU Affero General Public License as published by
-#the Free Software Foundation, version 3 of the License.
 #    
-#This program is distributed in the hope that it will be useful,
+#    This program is free software; you can redistribute it and/or modify
-#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    it under the terms of the GNU General Public License as published by
-#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#    the Free Software Foundation; either version 2 of the License, or
-#GNU General Public License for more details.
+#    (at your option) any later version.
 #    
-#You should have received a copy of the GNU Affero General Public License
+#    This program is distributed in the hope that it will be useful,
-#along with this program. If not, see <http://www.gnu.org/licenses/>.
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#In the "official" distribution you can find the license in agpl-3.0.txt.
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#    GNU General Public License for more details.
+#    
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+########################################################################
 
 import L10n
 _ = L10n.get_translation()
@@ -28,6 +31,7 @@ import Options
 import Configuration
 from Exceptions import *
 from cStringIO import StringIO
+import time
 
 (options, argv) = Options.fpdb_options()
 
@@ -37,7 +41,7 @@ codepage = ["utf-16", "utf-8", "cp1252"]
 
 
 class SplitHandHistory:
-    def __init__(self, config, in_path = '-', out_path = None, hands = 100, filter = "PokerStarsToFpdb", archive = False):
+    def __init__(self, config, in_path = '-', out_path = None, hands = 100, filter = "PokerStarsToFpdb", archive = False, workerid = 0):
         self.config = config
         self.in_path = in_path
         self.out_path = out_path
@@ -50,22 +54,25 @@ class SplitHandHistory:
         self.line_addendum = None
         self.filedone = False
         
+        self.timestamp = str(time.time())
+        self.workerid = '%02d' % workerid
+        
         #Acquire re_SplitHands for this hh
-        filter_name = filter.replace("ToFpdb", "")
+        self.filter_name = filter.replace("ToFpdb", "")
         mod = __import__(filter)
-        obj = getattr(mod, filter_name, None)
+        obj = getattr(mod, self.filter_name, None)
         self.re_SplitHands = obj.re_SplitHands
         
         #Determine line delimiter type if any
-        if self.re_SplitHands.match('\n\n'):
-            self.line_delimiter = '\n\n'
         if self.re_SplitHands.match('\n\n\n'):
             self.line_delimiter = '\n\n\n'
+        if self.re_SplitHands.match('\n\n'):
+            self.line_delimiter = '\n\n'
             
         #Add new line addendum for sites which match SplitHand to next line as well
-        if filter_name == 'OnGame':
+        if self.filter_name == 'OnGame':
             self.line_addendum = '*'
-        if filter_name == 'Carbon':
+        if self.filter_name == 'Carbon':
             self.line_addendum = '<game'
             
         #Open the gargantuan file
@@ -75,6 +82,7 @@ class SplitHandHistory:
             except IOError:
                 print _('File not found')
                 sys.exit(2)
+            self.kodec = kodec
         
         #Split with do_hands_per_file if archive and paragraphs if a regular hh
         if self.archive:
@@ -105,9 +113,10 @@ class SplitHandHistory:
             print _('Nope, will not work (fileno=%d)' % fileno)
             sys.exit(2)
         basename = os.path.splitext(os.path.basename(self.in_path))[0]
-        name = os.path.join(self.out_path, basename+'-%06d.txt' % fileno)
+        name = os.path.join(self.out_path, self.filter_name+'-'+basename+'_'+self.workerid+'_'+self.timestamp+'_%06d.txt' % fileno)
         print '-> %s' % name
         newfile = file(name, 'w')
+        os.chmod(name, 0775)
         return newfile
         
     #Archive Hand Splitter
@@ -122,8 +131,11 @@ class SplitHandHistory:
             except FpdbEndOfFile:
                 done = True
                 break
+            except UnicodeEncodeError:
+                print _('Absurd character done messed you up')
+                sys.exit(2)
             except:
-                print _("Unexpected error processing file")
+                print _('Unexpected error processing file')
                 sys.exit(2)
             n += 1
         outfile.close()
@@ -174,7 +186,7 @@ class SplitHandHistory:
         l = infile.readline()
         l = l.replace('\r\n', '\n')
         outfile.write(l)
-        l = infile.readline()
+        l = infile.readline().encode(self.kodec)
     
         while len(l) < 3:
             l = infile.readline()
@@ -182,7 +194,7 @@ class SplitHandHistory:
         while len(l) > 2:
             l = l.replace('\r\n', '\n')
             outfile.write(l)
-            l = infile.readline()
+            l = infile.readline().encode(self.kodec)
         outfile.write(self.line_delimiter)
         return infile
         
@@ -196,12 +208,18 @@ def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
         
+    if not options.filename:
+        options.filename = sys.argv[1]
+
     if not options.config:
-        options.config = Configuration.Config(file = "HUD_config.test.xml")
+        options.config = sys.argv[2]
+        
+    if sys.argv[3] == "True":
+        options.archive = True
         
     if options.filename:
         SplitHH = SplitHandHistory(options.config, options.filename, options.outpath, options.hands,
-                                   options.hhc, options.archive)
+                                   options.hhc, options.archive, options.workerid)
 
 if __name__ == '__main__':
     sys.exit(main())

pyfpdb/interlocks.py

@@ -108,7 +108,7 @@ class InterProcessLockFcntl(InterProcessLockBase):
         self.lockfd = 0
         try:
             os.unlink(self.lock_file_name)
-        except IOError:
+        except OSError:
             # We don't care about the existence of the file too much here.  It's the flock() we care about,
             # And that should just go away magically.
             pass

pyfpdb/regression-test-files/cash/Stars/Flop/LHE-10max-USD-1.00-2.00-No_max_seats.txt

@@ -0,0 +1,42 @@
+PokerStars Game #2428142447:  Hold'em Limit ($1/$2 USD) - 2005/08/26 16:12:22 ET
+Table 'Teucer II' Seat #5 is the button
+Seat 1: Frankson34 ($24 in chips) 
+Seat 2: webb22 ($64 in chips) 
+Seat 3: eric_mtx ($26 in chips) 
+Seat 4: sososolid ($147.75 in chips) 
+Seat 5: DRILHER ($48.25 in chips) 
+Seat 6: Naughtychic ($60 in chips) 
+Seat 7: Terps78 ($71.50 in chips) 
+Seat 8: ChazDazzle ($69.25 in chips) 
+Seat 9: alekos ($55 in chips) 
+Seat 10: BigNards84 ($64.25 in chips) 
+Naughtychic: posts small blind $0.50
+Terps78: posts big blind $1
+*** HOLE CARDS ***
+Dealt to ChazDazzle [8c Kd]
+ChazDazzle: folds 
+alekos: folds 
+BigNards84: raises $1 to $2
+Frankson34: folds 
+webb22: folds 
+eric_mtx: folds 
+ChazDazzle leaves the table
+sososolid: folds 
+DRILHER: folds 
+cdhender joins the table at seat #8 
+Naughtychic: folds 
+Terps78: folds 
+BigNards84 collected $2.50 from pot
+BigNards84: doesn't show hand 
+*** SUMMARY ***
+Total pot $2.50 | Rake $0 
+Seat 1: Frankson34 folded before Flop (didn't bet)
+Seat 2: webb22 folded before Flop (didn't bet)
+Seat 3: eric_mtx folded before Flop (didn't bet)
+Seat 4: sososolid folded before Flop (didn't bet)
+Seat 5: DRILHER (button) folded before Flop (didn't bet)
+Seat 6: Naughtychic (small blind) folded before Flop
+Seat 7: Terps78 (big blind) folded before Flop
+Seat 8: ChazDazzle folded before Flop (didn't bet)
+Seat 9: alekos folded before Flop (didn't bet)
+Seat 10: BigNards84 collected ($2.50)