Merge branch 'master' of git://git.assembla.com/fpdboz

pyfpdb/FullTiltPokerSummary.py

@@ -0,0 +1,138 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+#Copyright 2008-2010 Steffen Schaumburg
+#This program is free software: you can redistribute it and/or modify
+#it under the terms of the GNU Affero General Public License as published by
+#the Free Software Foundation, version 3 of the License.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU Affero General Public License
+#along with this program. If not, see <http://www.gnu.org/licenses/>.
+#In the "official" distribution you can find the license in agpl-3.0.txt.
+
+"""pokerstars-specific summary parsing code"""
+
+from decimal import Decimal
+import datetime
+
+from Exceptions import FpdbParseError
+from HandHistoryConverter import *
+import PokerStarsToFpdb
+from TourneySummary import *
+
+import locale
+lang=locale.getdefaultlocale()[0][0:2]
+if lang=="en":
+    def _(string): return string
+else:
+    import gettext
+    try:
+        trans = gettext.translation("fpdb", localedir="locale", languages=[lang])
+        trans.install()
+    except IOError:
+        def _(string): return string
+
+class FullTiltPokerSummary(TourneySummary):
+    limits = { 'No Limit':'nl', 'Pot Limit':'pl', 'Limit':'fl', 'LIMIT':'fl' }
+    games = {                          # base, category
+                              "Hold'em" : ('hold','holdem'), 
+                                'Omaha' : ('hold','omahahi'),
+                          'Omaha Hi/Lo' : ('hold','omahahilo'),
+                                 'Razz' : ('stud','razz'), 
+                                 'RAZZ' : ('stud','razz'),
+                          '7 Card Stud' : ('stud','studhi'),
+                    '7 Card Stud Hi/Lo' : ('stud','studhilo'),
+                               'Badugi' : ('draw','badugi'),
+              'Triple Draw 2-7 Lowball' : ('draw','27_3draw'),
+                          '5 Card Draw' : ('draw','fivedraw')
+               }
+
+    substitutions = {
+                     'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP",    # legal ISO currency codes
+                            'LS' : "\$|\xe2\x82\xac|"        # legal currency symbols - Euro(cp1252, utf-8)
+                    }
+
+    re_SplitTourneys = re.compile("^Full Tilt Poker Tournament Summary")
+    
+    re_TourNo = re.compile("\#(?P<TOURNO>[0-9]+),")
+
+    re_TourneyInfo = re.compile(u"""
+                        \s.*
+                        (?P<TYPE>Tournament|Sit\s\&\sGo)\s\((?P<TOURNO>[0-9]+)\)(\s+)?
+                        (?P<GAME>Hold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s+
+                        (?P<LIMIT>No\sLimit|Limit|LIMIT|Pot\sLimit)\s+
+                        (Buy-In:\s\$(?P<BUYIN>[.\d]+)(\s\+\s\$(?P<FEE>[.\d]+))?\s+)?
+                        (Buy-In\sChips:\s(?P<CHIPS>\d+)\s+)?
+                        (?P<ENTRIES>[0-9]+)\sEntries\s+
+                        (\$?(?P<ADDED>[.\d]+)\sadded\sto\sthe\sprize\spool\sby\sPokerStars\.com\s+)?
+                        (Total\sPrize\sPool:\s\$?(?P<PRIZEPOOL>[.0-9]+)\s+)?
+                        (Target\sTournament\s.*)?
+                        Tournament\sstarted:\s
+                        (?P<Y>[\d]{4})\/(?P<M>[\d]{2})\/(?P<D>[\d]+)\s+(?P<H>[\d]+):(?P<MIN>[\d]+):(?P<S>[\d]+)\s??(?P<TZ>[A-Z]+)\s
+                               """ % substitutions ,re.VERBOSE|re.MULTILINE|re.DOTALL)
+
+    re_Currency = re.compile(u"""(?P<CURRENCY>[%(LS)s]|FPP)""" % substitutions)
+
+    re_Player = re.compile(u"""(?P<RANK>[\d]+):\s(?P<NAME>[^,\r\n]{2,15})(,(\s)?\$(?P<WINNINGS>[.\d]+))?""")
+
+    re_DateTime = re.compile("\[(?P<Y>[0-9]{4})\/(?P<M>[0-9]{2})\/(?P<D>[0-9]{2})[\- ]+(?P<H>[0-9]+):(?P<MIN>[0-9]+):(?P<S>[0-9]+)")
+
+    codepage = ["utf-16", "cp1252", "utf-8"]
+
+    def parseSummary(self):
+        m = self.re_TourneyInfo.search(self.summaryText)
+        if m == None:
+            tmp = self.summaryText[0:200]
+            log.error(_("parseSummary: Unable to recognise Tourney Info: '%s'") % tmp)
+            log.error(_("parseSummary: Raising FpdbParseError"))
+            raise FpdbParseError(_("Unable to recognise Tourney Info: '%s'") % tmp)
+
+        print "DEBUG: m.groupdict(): %s" % m.groupdict()
+
+        mg = m.groupdict()
+        if 'TOURNO'    in mg: self.tourNo = mg['TOURNO']
+        if 'LIMIT'     in mg: self.gametype['limitType'] = self.limits[mg['LIMIT']]
+        if 'GAME'      in mg: self.gametype['category']  = self.games[mg['GAME']][1]
+        if mg['BUYIN'] != None:
+            self.buyin = int(100*Decimal(mg['BUYIN']))
+        if mg['FEE'] != None:
+            self.fee   = int(100*Decimal(mg['FEE']))
+        if 'PRIZEPOOL' in mg: self.prizepool             = mg['PRIZEPOOL']
+        if 'ENTRIES'   in mg: self.entries               = mg['ENTRIES']
+
+        datetimestr = "%s/%s/%s %s:%s:%s" % (mg['Y'], mg['M'], mg['D'], mg['H'], mg['MIN'], mg['S'])
+        self.startTime = datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S")
+
+        if 'TZ' in mg:
+            self.startTime = HandHistoryConverter.changeTimezone(self.startTime, mg['TZ'], "UTC")
+
+
+        m = self.re_Currency.search(self.summaryText)
+        if m == None:
+            log.error(_("parseSummary: Unable to locate currency"))
+            log.error(_("parseSummary: Raising FpdbParseError"))
+            raise FpdbParseError(_("Unable to locate currency"))
+        #print "DEBUG: m.groupdict(): %s" % m.groupdict()
+
+        mg = m.groupdict()
+        if mg['CURRENCY'] == "$":     self.currency = "USD"
+        elif mg['CURRENCY'] == u"€":  self.currency="EUR"
+        elif mg['CURRENCY'] == "FPP": self.currency="PSFP"
+
+        m = self.re_Player.finditer(self.summaryText)
+        for a in m:
+            mg = a.groupdict()
+            print "DEBUG: a.groupdict(): %s" % mg
+            name = mg['NAME']
+            rank = mg['RANK']
+            winnings = 0
+
+            if 'WINNINGS' in mg and mg['WINNINGS'] != None:
+                winnings = int(100*Decimal(mg['WINNINGS']))
+            self.addPlayer(rank, name, winnings, self.currency, None, None, None)
+

pyfpdb/ImapFetcher.py

@@ -30,6 +30,7 @@ from Exceptions import FpdbParseError
 import SQL
 import Options
 import PokerStarsSummary
+import FullTiltPokerSummary
 
 
 import locale
@@ -44,12 +45,23 @@ else:
     except IOError:
         def _(string): return string
 
-def splitPokerStarsSummaries(emailText):
+def splitPokerStarsSummaries(summaryText):
-    splitSummaries=emailText.split("\nPokerStars Tournament #")[1:]
+    re_SplitTourneys = PokerStarsSummary.PokerStarsSummary.re_SplitTourneys
-    for i in range(len(splitSummaries)):
+    splitSummaries = re.split(re_SplitTourneys, summaryText)
-        splitSummaries[i]="PokerStars Tournament #"+splitSummaries[i]
+
+    if len(splitSummaries) <= 1:
+        print _("DEBUG: re_SplitTourneyss isn't matching")
+
+    return splitSummaries
+
+def splitFullTiltSummaries(summaryText):
+    re_SplitTourneys = FullTiltPokerSummary.FullTiltPokerSummary.re_SplitTourneys
+    splitSummaries = re.split(re_SplitTourneys, summaryText)
+
+    if len(splitSummaries) <= 1:
+        print _("DEBUG: re_SplitTourneyss isn't matching")
+
     return splitSummaries
-#end def emailText
 
 def run(config, db):
         #print "start of IS.run"
@@ -82,7 +94,7 @@ def run(config, db):
         if (len(neededMessages)==0):
             raise error #TODO: show error message
 
-        errors = 0
+        email_bodies = []
         for i, messageData in enumerate(neededMessages, start=1):
             print "Retrieving message %s" % i
             response, bodyData = server.fetch(messageData[1], "(UID BODY[TEXT])")
@@ -90,45 +102,72 @@ def run(config, db):
             if response!="OK":
                 raise error #TODO: show error message
             if messageData[0]=="PS":
-                summaryTexts=(splitPokerStarsSummaries(bodyData))
+                email_bodies.append(bodyData)
-                print "Found %s summaries in email" %(len(summaryTexts))
-                for j, summaryText in enumerate(summaryTexts, start=1):
-                    try:
-                        result=PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summaryText, builtFrom = "IMAP")
-                    except FpdbParseError, e:
-                        errors += 1
-                    print _("Finished importing %s/%s PS summaries") %(j, len(summaryTexts))
-
-        print _("Completed running Imap import, closing server connection")
-        print _("Errors: %s" % errors)
     #finally:
      #   try:
         server.close()
        # finally:
         #    pass
         server.logout()
+        print _("Completed retrieving IMAP messages, closing server connection")
 
-def readFile(filename):
+        errors = 0
-    kodec = "utf8"
+        if len(email_bodies) > 0:
+            errors = importSummaries(db, config, email_bodies, options = None)
+        else:
+            print _("No Tournament summaries found.")
+
+        print _("Errors: %s" % errors)
+
+def readFile(filename, options):
+    codepage = ["utf8"]
+    whole_file = None
+    if options.hhc == "PokerStars":
+        codepage = PokerStarsSummary.PokerStarsSummary.codepage
+    elif options.hhc == "Full Tilt Poker":
+        codepage = FullTiltPokerSummary.FullTiltPokerSummary.codepage
+
+    for kodec in codepage:
+        #print "trying", kodec
+        try:
             in_fh = codecs.open(filename, 'r', kodec)
             whole_file = in_fh.read()
             in_fh.close()
+            break
+        except:
+           pass
+
     return whole_file
 
+def runFake(db, config, options):
+    summaryText = readFile(options.infile, options)
+    importSummaries(db, config,[summaryText], options=options)
 
+def importSummaries(db, config, summaries, options = None):
+    # TODO: At this point we should have:
+    # - list of strings to process
+    # - The sitename OR specialised TourneySummary object
+    # Using options is pretty ugly
+    errors = 0
+    for summaryText in summaries:
+        # And we should def be using a 'Split' from the site object
+        if options == None or options.hhc == "PokerStars":
+            summaryTexts=(splitPokerStarsSummaries(summaryText))
+        elif options.hhc == "Full Tilt Poker":
+            summaryTexts=(splitFullTiltSummaries(summaryText))
 
-def runFake(db, config, infile):
+        print "Found %s summaries in email" %(len(summaryTexts))
-    summaryText = readFile(infile)
+        for j, summaryText in enumerate(summaryTexts, start=1):
-    # This regex should be part of PokerStarsSummary
+            try:
-    re_SplitGames = re.compile("PokerStars Tournament ")
+                if options == None or options.hhc == "PokerStars":
-    summaryList = re.split(re_SplitGames, summaryText)
+                    PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summaryText, builtFrom = "IMAP")
+                elif options.hhc == "Full Tilt Poker":
+                    FullTiltPokerSummary.FullTiltPokerSummary(db=db, config=config, siteName=u"Fulltilt", summaryText=summaryText, builtFrom = "IMAP")
+            except FpdbParseError, e:
+                errors += 1
+            print _("Finished importing %s/%s PS summaries") %(j, len(summaryTexts))
 
-    if len(summaryList) <= 1:
+    return errors
-        print _("DEBUG: re_SplitGames isn't matching")
-
-    for summary in summaryList[1:]:
-        result = PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summary, builtFrom = "file")
-        print _("DEBUG: Processed: %s: tournNo: %s") % (result.tourneyId, result.tourNo)
 
 
 def main(argv=None):
@@ -142,6 +181,10 @@ def main(argv=None):
         print _("USAGE:")
         sys.exit(0)
 
+    if options.hhc == "PokerStarsToFpdb":
+        print _("Need to define a converter")
+        exit(0)
+
     # These options should really come from the OptionsParser
     config = Configuration.Config()
     db = Database.Database(config)
@@ -152,7 +195,7 @@ def main(argv=None):
     settings.update(config.get_default_paths())
     db.recreate_tables()
 
-    runFake(db, config, options.infile)
+    runFake(db, config, options)
 
 if __name__ == '__main__':
     sys.exit(main())

pyfpdb/PokerStarsSummary.py

@@ -57,7 +57,7 @@ class PokerStarsSummary(TourneySummary):
                             'LS' : "\$|\xe2\x82\xac|"        # legal currency symbols - Euro(cp1252, utf-8)
                     }
 
-    re_SplitGames = re.compile("^PokerStars")
+    re_SplitTourneys = re.compile("PokerStars Tournament ")
     
     re_TourNo = re.compile("\#(?P<TOURNO>[0-9]+),")
 
@@ -81,123 +81,14 @@ class PokerStarsSummary(TourneySummary):
 
     re_DateTime = re.compile("\[(?P<Y>[0-9]{4})\/(?P<M>[0-9]{2})\/(?P<D>[0-9]{2})[\- ]+(?P<H>[0-9]+):(?P<MIN>[0-9]+):(?P<S>[0-9]+)")
 
-    re_Entries = re.compile("[0-9]+")
+    codepage = ["utf-8"]
-    re_Prizepool = re.compile("\$[0-9]+\.[0-9]+")
-    re_BuyInFee = re.compile("(?P<BUYIN>[0-9]+\.[0-9]+).*(?P<FEE>[0-9]+\.[0-9]+)")
-    re_FPP = re.compile("(?P<FPP>[0-9]+)\sFPP")
-    #note: the dollar and cent in the below line are currency-agnostic
-    re_Added = re.compile("(?P<DOLLAR>[0-9]+)\.(?P<CENT>[0-9]+)\s(?P<CURRENCY>[A-Z]+)(\sadded\sto\sthe\sprize\spool\sby\sPokerStars)")
-    re_DateTimeET = re.compile("(?P<Y>[0-9]{4})\/(?P<M>[0-9]{2})\/(?P<D>[0-9]{2})[\- ]+(?P<H>[0-9]+):(?P<MIN>[0-9]+):(?P<S>[0-9]+)")
-    re_GameInfo = re.compile(u""".+(?P<LIMIT>No\sLimit|Limit|LIMIT|Pot\sLimit)\s(?P<GAME>Hold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)""")
 
     def parseSummary(self):
-        lines=self.summaryText.splitlines()
-        
-        self.tourNo = self.re_TourNo.findall(lines[0])[0] #ignore game and limit type as thats not recorded
-        
-        result=self.re_GameInfo.search(lines[0])
-        result=result.groupdict()
-        self.gametype['limitType']=self.limits[result['LIMIT']]
-        self.gametype['category']=self.games[result['GAME']][1]
-        
-        if lines[1].find("$")!=-1: #TODO: move this into a method and call that from PokerStarsToFpdb.py:269    if hand.buyinCurrency=="USD" etc.
-            self.currency="USD"
-        elif lines[1].find(u"€")!=-1:
-            self.currency="EUR"
-        elif lines[1].find("FPP")!=-1:
-            self.currency="PSFP"
-        else:
-            raise FpdbParseError(_("didn't recognise buyin currency in:")+lines[1])
-        
-        if self.currency=="USD" or self.currency=="EUR":
-            result=self.re_BuyInFee.search(lines[1])
-            result=result.groupdict()
-            self.buyin=int(100*Decimal(result['BUYIN']))
-            self.fee=int(100*Decimal(result['FEE']))
-        elif self.currency=="PSFP":
-            result=self.re_FPP.search(lines[1])
-            result=result.groupdict()
-            self.buyin=int(Decimal(result['FPP']))
-            self.fee=0
-        
-        currentLine=2
-        self.entries = self.re_Entries.findall(lines[currentLine])[0]
-        currentLine+=1 #note that I chose to make the code keep state (the current line number)
-                       #as that means it'll fail rather than silently skip potentially valuable information
-        #print "after entries lines[currentLine]", lines[currentLine]
-
-        result=self.re_Added.search(lines[currentLine])
-        if result:
-            result=result.groupdict()
-            self.added=100*int(Decimal(result['DOLLAR']))+int(Decimal(result['CENT']))
-            self.addedCurrency=result['CURRENCY']
-            currentLine+=1
-        else:
-            self.added=0
-            self.addedCurrency="NA"
-        #print "after added/entries lines[currentLine]", lines[currentLine]
-
-        result=self.re_Prizepool.findall(lines[currentLine])
-        if result:
-            self.prizepool = result[0]
-            self.prizepool = self.prizepool[1:-3]+self.prizepool[-2:]
-            currentLine+=1
-        #print "after prizepool lines[currentLine]", lines[currentLine]
-        
-        useET=False
-        result=self.re_DateTime.search(lines[currentLine])
-        if not result:
-            print _("in not result starttime")
-            useET=True
-            result=self.re_DateTimeET.search(lines[currentLine])
-        result=result.groupdict()
-        datetimestr = "%s/%s/%s %s:%s:%s" % (result['Y'], result['M'],result['D'],result['H'],result['MIN'],result['S'])
-        self.startTime= datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S") # also timezone at end, e.g. " ET"
-        self.startTime = HandHistoryConverter.changeTimezone(self.startTime, "ET", "UTC")
-        currentLine+=1
-
-        if useET:
-            result=self.re_DateTimeET.search(lines[currentLine])
-        else:
-            result=self.re_DateTime.search(lines[currentLine])
-        if result:
-            result=result.groupdict()
-            datetimestr = "%s/%s/%s %s:%s:%s" % (result['Y'], result['M'],result['D'],result['H'],result['MIN'],result['S'])
-            self.endTime= datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S") # also timezone at end, e.g. " ET"
-            self.endTime = HandHistoryConverter.changeTimezone(self.endTime, "ET", "UTC")
-        currentLine+=1
-        
-        if lines[currentLine].find("Tournament is still in progress")!=-1:
-            currentLine+=1
-        
-        for i in range(currentLine,len(lines)-2): #lines with rank and winnings info
-            if lines[i].find(":")==-1:
-                break
-            result=self.re_Player.search(lines[i])
-            result=result.groupdict()
-            rank=result['RANK']
-            name=result['NAME']
-            winnings=result['WINNINGS']
-
-            if winnings:
-                winnings=int(100*Decimal(winnings))
-            else:
-                winnings=0
-            
-            if result['STILLPLAYING']:
-                #print "stillplaying"
-                rank=None
-                winnings=None
-            
-            self.addPlayer(rank, name, winnings, self.currency, None, None, None)#TODO: currency, ko/addon/rebuy count -> need examples!
-    #end def parseSummary
-
-    def parseSummaryFile(self):
         m = self.re_TourneyInfo.search(self.summaryText)
         if m == None:
             tmp = self.summaryText[0:200]
-            log.error(_("parseSummaryFile: Unable to recognise Tourney Info: '%s'") % tmp)
+            log.error(_("parseSummary: Unable to recognise Tourney Info: '%s'") % tmp)
-            log.error(_("parseSummaryFile: Raising FpdbParseError"))
+            log.error(_("parseSummary: Raising FpdbParseError"))
             raise FpdbParseError(_("Unable to recognise Tourney Info: '%s'") % tmp)
 
         #print "DEBUG: m.groupdict(): %s" % m.groupdict()
@@ -222,8 +113,8 @@ class PokerStarsSummary(TourneySummary):
 
         m = self.re_Currency.search(self.summaryText)
         if m == None:
-            log.error(_("parseSummaryFile: Unable to locate currency"))
+            log.error(_("parseSummary: Unable to locate currency"))
-            log.error(_("parseSummaryFile: Raising FpdbParseError"))
+            log.error(_("parseSummary: Raising FpdbParseError"))
             raise FpdbParseError(_("Unable to locate currency"))
         #print "DEBUG: m.groupdict(): %s" % m.groupdict()
 

pyfpdb/TestHandsPlayers.py

@@ -154,18 +154,41 @@ def main(argv=None):
                     iPokerErrors
                 ]
 
+    sites = {
+                'PokerStars' : True,
+                'Full Tilt Poker' : True,
+                'PartyPoker' : True,
+                'Betfair' : True,
+                'OnGame' : True,
+                'Absolute' : True,
+                'Everleaf' : True,
+                'Carbon' : True,
+                'PKR' : True,
+                'iPoker' : True,
+            }
+
+    if sites['PokerStars'] == True:
         walk_testfiles("regression-test-files/cash/Stars/", compare, importer, PokerStarsErrors, "PokerStars")
         walk_testfiles("regression-test-files/tour/Stars/", compare, importer, PokerStarsErrors, "PokerStars")
+    if sites['Full Tilt Poker'] == True:
         walk_testfiles("regression-test-files/cash/FTP/", compare, importer, FTPErrors, "Full Tilt Poker")
         walk_testfiles("regression-test-files/tour/FTP/", compare, importer, FTPErrors, "Full Tilt Poker")
+    if sites['PartyPoker'] == True:
         walk_testfiles("regression-test-files/cash/PartyPoker/", compare, importer, PartyPokerErrors, "PartyPoker")
         walk_testfiles("regression-test-files/tour/PartyPoker/", compare, importer, PartyPokerErrors, "PartyPoker")
+    if sites['Betfair'] == True:
         walk_testfiles("regression-test-files/cash/Betfair/", compare, importer, BetfairErrors, "Betfair")
+    if sites['OnGame'] == True:
         walk_testfiles("regression-test-files/cash/OnGame/", compare, importer, OnGameErrors, "OnGame")
+    if sites['Absolute'] == True:
         walk_testfiles("regression-test-files/cash/Absolute/", compare, importer, AbsoluteErrors, "Absolute")
+    if sites['Everleaf'] == True:
         walk_testfiles("regression-test-files/cash/Everleaf/", compare, importer, EverleafErrors, "Everleaf")
+    if sites['Carbon'] == True:
         walk_testfiles("regression-test-files/cash/Carbon/", compare, importer, CarbonErrors, "Carbon")
+    if sites['PKR'] == True:
         walk_testfiles("regression-test-files/cash/PKR/", compare, importer, PKRErrors, "PKR")
+    if sites['iPoker'] == True:
         walk_testfiles("regression-test-files/cash/iPoker/", compare, importer, iPokerErrors, "iPoker")
 
     totalerrors = 0

pyfpdb/TourneySummary.py

@@ -125,11 +125,11 @@ class TourneySummary(object):
         self.sym = None
         
         if builtFrom=="IMAP":
+            # Fix line endings?
+            pass
+
         self.parseSummary()
         self.insertOrUpdate()
-        elif builtFrom == "file":
-            self.parseSummaryFile()
-            self.insertOrUpdate()
     #end def __init__
 
     def __str__(self):