From 20bb51145b43389449bafa5d8ad394c41105f8e2 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 16:30:43 +0800 Subject: [PATCH 01/14] Options: add generic usage option --- pyfpdb/Options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyfpdb/Options.py b/pyfpdb/Options.py index 11ef9d43..19139278 100644 --- a/pyfpdb/Options.py +++ b/pyfpdb/Options.py @@ -60,6 +60,9 @@ def fpdb_options(): default = 'EMPTY') parser.add_option("-v", "--version", action = "store_true", help = _("Print version information and exit.")) + parser.add_option("-u", "--usage", action="store_true", dest="usage", default=False, + help=_("Print some useful one liners")) + (options, argv) = parser.parse_args() return (options, argv) From 10a48240dbbc4a98a4d09beead1af999564c03f1 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 16:31:39 +0800 Subject: [PATCH 02/14] PSSummary: Add new parseSummary method --- pyfpdb/PokerStarsSummary.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index 14b9cadc..bd8cbe4e 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -53,6 +53,8 @@ class PokerStarsSummary(TourneySummary): 'Triple Draw 2-7 Lowball' : ('draw','27_3draw'), '5 Card Draw' : ('draw','fivedraw') } + + re_SplitGames = re.compile("^PokerStars") re_TourNo = re.compile("\#[0-9]+,") re_Entries = re.compile("[0-9]+") @@ -167,4 +169,7 @@ class PokerStarsSummary(TourneySummary): self.addPlayer(rank, name, winnings, self.currency, None, None, None)#TODO: currency, ko/addon/rebuy count -> need examples! #end def parseSummary + def parseSummaryFile(self): + lines=self.summaryText.splitlines() + print lines #end class PokerStarsSummary From 04849775d7f33d46422ded5b99fc35677323db7a Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 16:32:46 +0800 Subject: [PATCH 03/14] TS: add new builtFrom option --- pyfpdb/TourneySummary.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyfpdb/TourneySummary.py b/pyfpdb/TourneySummary.py index 105141d5..8fe7ceaa 100644 --- a/pyfpdb/TourneySummary.py +++ b/pyfpdb/TourneySummary.py @@ -127,6 +127,9 @@ class TourneySummary(object): if builtFrom=="IMAP": self.parseSummary() self.insertOrUpdate() + elif builtFrom == "file": + self.parseSummaryFile() + self.insertOrUpdate() #end def __init__ def __str__(self): From 1dda0848d32d7d0f04a2bb13491befe6f39dfc95 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 16:33:16 +0800 Subject: [PATCH 04/14] ImapFetcher: Make it run from command line with -i --- pyfpdb/ImapFetcher.py | 62 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/pyfpdb/ImapFetcher.py b/pyfpdb/ImapFetcher.py index befecd6f..4c0422e1 100755 --- a/pyfpdb/ImapFetcher.py +++ b/pyfpdb/ImapFetcher.py @@ -20,8 +20,17 @@ #see http://tools.ietf.org/html/rfc2060#section-6.4.4 for IMAP4 search criteria from imaplib import IMAP4, IMAP4_SSL +import sys +import codecs +import re + +import Configuration +import Database +import SQL +import Options import PokerStarsSummary + import locale lang=locale.getdefaultlocale()[0][0:2] if lang=="en": @@ -89,4 +98,55 @@ def run(config, db): # finally: # pass server.logout() - \ No newline at end of file + +def readFile(filename): + kodec = "utf8" + in_fh = codecs.open(filename, 'r', kodec) + whole_file = in_fh.read() + in_fh.close() + return whole_file + + + +def runFake(db, config, infile): + summaryText = readFile(infile) + # This regex should be part of PokerStarsSummary + re_SplitGames = re.compile("PokerStars Tournament #") + summaryList = re.split(re_SplitGames, summaryText) + + if len(summaryList) <= 1: + print "DEBUG: re_SplitGames isn't matching" + + for summary in summaryList[1:]: + result = PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summary, builtFrom = "file") + +def splitPokerStarsSummaries(emailText): + splitSummaries=emailText.split("\nPokerStars Tournament #")[1:] + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + (options, argv) = Options.fpdb_options() + + if options.usage == True: + #Print usage examples and exit + print _("USAGE:") + sys.exit(0) + + # These options should really come from the OptionsParser + config = Configuration.Config(file = "HUD_config.test.xml") + db = Database.Database(config) + sql = SQL.Sql(db_server = 'sqlite') + settings = {} + settings.update(config.get_db_parameters()) + settings.update(config.get_import_parameters()) + settings.update(config.get_default_paths()) + db.recreate_tables() + + runFake(db, config, options.infile) + +if __name__ == '__main__': + sys.exit(main()) + From e66b98b3451f79b34390afda2efaf8b84872ab06 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 18:10:25 +0800 Subject: [PATCH 05/14] PSSummary: add function parseSummaryFile This parses the summary file without requiring state. Hope to merge parseSummary and parseSummaryFile at some point soon --- pyfpdb/ImapFetcher.py | 2 +- pyfpdb/PokerStarsSummary.py | 101 ++++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 12 deletions(-) diff --git a/pyfpdb/ImapFetcher.py b/pyfpdb/ImapFetcher.py index 4c0422e1..01d75a54 100755 --- a/pyfpdb/ImapFetcher.py +++ b/pyfpdb/ImapFetcher.py @@ -111,7 +111,7 @@ def readFile(filename): def runFake(db, config, infile): summaryText = readFile(infile) # This regex should be part of PokerStarsSummary - re_SplitGames = re.compile("PokerStars Tournament #") + re_SplitGames = re.compile("PokerStars Tournament ") summaryList = re.split(re_SplitGames, summaryText) if len(summaryList) <= 1: diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index bd8cbe4e..732e8a85 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -37,8 +37,6 @@ else: except IOError: def _(string): return string - - class PokerStarsSummary(TourneySummary): limits = { 'No Limit':'nl', 'Pot Limit':'pl', 'Limit':'fl', 'LIMIT':'fl' } games = { # base, category @@ -54,17 +52,38 @@ class PokerStarsSummary(TourneySummary): '5 Card Draw' : ('draw','fivedraw') } + substitutions = { + 'LEGAL_ISO' : "USD|EUR|GBP|CAD|FPP", # legal ISO currency codes + 'LS' : "\$|\xe2\x82\xac|" # legal currency symbols - Euro(cp1252, utf-8) + } + re_SplitGames = re.compile("^PokerStars") - re_TourNo = re.compile("\#[0-9]+,") + re_TourNo = re.compile("\#(?P[0-9]+),") + + re_TourneyInfo = re.compile(u""" + \#(?P[0-9]+),\s + (?PNo\sLimit|Limit|LIMIT|Pot\sLimit)\s + (?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s + Buy-In:\s\$(?P[.0-9]+)\/\$(?P[.0-9]+)\s + (?P[0-9]+)\splayers\s + Total\sPrize\sPool:\s\$?(?P[.0-9]+)\s + Tournament\sstarted\s-\s + (?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\-\s]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)\s?\(?(?P[A-Z]+)\)\s + """ % substitutions ,re.VERBOSE|re.MULTILINE) + + re_Currency = re.compile(u"""(?P[%(LS)s]|FPP)""" % substitutions) + + re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)?(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?""") + + re_DateTime = re.compile("\[(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)") + re_Entries = re.compile("[0-9]+") re_Prizepool = re.compile("\$[0-9]+\.[0-9]+") - re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)?(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?""") re_BuyInFee = re.compile("(?P[0-9]+\.[0-9]+).*(?P[0-9]+\.[0-9]+)") re_FPP = re.compile("(?P[0-9]+)\sFPP") #note: the dollar and cent in the below line are currency-agnostic re_Added = re.compile("(?P[0-9]+)\.(?P[0-9]+)\s(?P[A-Z]+)(\sadded\sto\sthe\sprize\spool\sby\sPokerStars)") - re_DateTime = re.compile("\[(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)") re_DateTimeET = re.compile("(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)") re_GameInfo = re.compile(u""".+(?PNo\sLimit|Limit|LIMIT|Pot\sLimit)\s(?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)""") @@ -103,7 +122,7 @@ class PokerStarsSummary(TourneySummary): currentLine+=1 #note that I chose to make the code keep state (the current line number) #as that means it'll fail rather than silently skip potentially valuable information #print "after entries lines[currentLine]", lines[currentLine] - + result=self.re_Added.search(lines[currentLine]) if result: result=result.groupdict() @@ -114,7 +133,7 @@ class PokerStarsSummary(TourneySummary): self.added=0 self.addedCurrency="NA" #print "after added/entries lines[currentLine]", lines[currentLine] - + result=self.re_Prizepool.findall(lines[currentLine]) if result: self.prizepool = result[0] @@ -133,7 +152,7 @@ class PokerStarsSummary(TourneySummary): self.startTime= datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S") # also timezone at end, e.g. " ET" self.startTime = HandHistoryConverter.changeTimezone(self.startTime, "ET", "UTC") currentLine+=1 - + if useET: result=self.re_DateTimeET.search(lines[currentLine]) else: @@ -156,7 +175,7 @@ class PokerStarsSummary(TourneySummary): rank=result['RANK'] name=result['NAME'] winnings=result['WINNINGS'] - + if winnings: winnings=int(100*Decimal(winnings)) else: @@ -169,7 +188,67 @@ class PokerStarsSummary(TourneySummary): self.addPlayer(rank, name, winnings, self.currency, None, None, None)#TODO: currency, ko/addon/rebuy count -> need examples! #end def parseSummary + def parseSummaryFile(self): - lines=self.summaryText.splitlines() - print lines + print self.summaryText + m = self.re_TourneyInfo.search(self.summaryText) + if m: + print "DEBUG: m.groupdict(): %s" % m.groupdict() + else: + tmp = self.summaryText[0:100] + log.error(_("parseSummaryFile: Unable to recognise Tourney Info: '%s'") % tmp) + log.error(_("parseSummaryFile: Raising FpdbParseError")) + raise FpdbParseError(_("Unable to recognise Tourney Info: '%s'") % tmp) + + mg = m.groupdict() + if 'TOURNO' in mg: self.tourNo = mg['TOURNO'] + if 'LIMIT' in mg: self.gametype['limitType'] = self.limits[mg['LIMIT']] + if 'GAME' in mg: self.gametype['category'] = self.games[mg['GAME']][1] + if 'BUYIN' in mg: self.buyin = int(100*Decimal(mg['BUYIN'])) + if 'FEE' in mg: self.fee = int(100*Decimal(mg['FEE'])) + if 'PRIZEPOOL' in mg: self.prizepool = mg['PRIZEPOOL'] + if 'ENTRIES' in mg: self.entries = mg['ENTRIES'] + + datetimestr = "%s/%s/%s %s:%s:%s" % (mg['Y'], mg['M'], mg['D'], mg['H'], mg['MIN'], mg['S']) + self.startTime = datetime.datetime.strptime(datetimestr, "%Y/%m/%d %H:%M:%S") + + if 'TZ' in mg: + self.startTime = HandHistoryConverter.changeTimezone(self.startTime, mg['TZ'], "UTC") + + + m = self.re_Currency.search(self.summaryText) + if m: + print "DEBUG: m.groupdict(): %s" % m.groupdict() + else: + log.error(_("parseSummaryFile: Unable to locate currency")) + log.error(_("parseSummaryFile: Raising FpdbParseError")) + raise FpdbParseError(_("Unable to locate currency")) + + mg = m.groupdict() + if mg['CURRENCY'] == "$": self.currency = "USD" + elif mg['CURRENCY'] == u"€": self.currency="EUR" + elif mg['CURRENCY'] == "FPP": self.currency="PSFP" + + m = self.re_Player.finditer(self.summaryText) + for a in m: + mg = a.groupdict() + print "DEBUG: a.groupdict(): %s" % mg + name = mg['NAME'] + rank = mg['RANK'] + winnings = 0 + + if 'WINNINGS' in mg and mg['WINNINGS'] != None: + winnings = int(100*Decimal(mg['WINNINGS'])) + + if 'STILLPLAYING' in mg: + #print "stillplaying" + rank=None + winnings=None + + #TODO: currency, ko/addon/rebuy count -> need examples! + self.addPlayer(rank, name, winnings, self.currency, None, None, None) + + print self + sys.exit(0) + #end class PokerStarsSummary From 3fd2b2f8383cc11e0cea55348d6508fb3345f791 Mon Sep 17 00:00:00 2001 From: gimick Date: Sat, 4 Sep 2010 12:30:43 +0100 Subject: [PATCH 06/14] Import: Show progress bar during bulk import --- pyfpdb/GuiBulkImport.py | 10 ++++---- pyfpdb/fpdb.pyw | 2 +- pyfpdb/fpdb_import.py | 53 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/pyfpdb/GuiBulkImport.py b/pyfpdb/GuiBulkImport.py index a90f566c..e311c68e 100755 --- a/pyfpdb/GuiBulkImport.py +++ b/pyfpdb/GuiBulkImport.py @@ -142,10 +142,12 @@ class GuiBulkImport(): """returns the vbox of this thread""" return self.vbox - def __init__(self, settings, config, sql = None): + def __init__(self, settings, config, parent, sql = None): self.settings = settings self.config = config - self.importer = fpdb_import.Importer(self, self.settings, config, sql) + self.parent = parent + + self.importer = fpdb_import.Importer(self, self.settings, config,parent, sql) self.vbox = gtk.VBox(False, 0) self.vbox.show() @@ -385,7 +387,7 @@ def main(argv=None): print _('-q is deprecated. Just use "-f filename" instead') # This is because -q on its own causes an error, so -f is necessary and sufficient for cmd line use if not options.filename: - i = GuiBulkImport(settings, config) + i = GuiBulkImport(settings, config, None) main_window = gtk.Window() main_window.connect('destroy', destroy) main_window.add(i.vbox) @@ -393,7 +395,7 @@ def main(argv=None): gtk.main() else: #Do something useful - importer = fpdb_import.Importer(False,settings, config) + importer = fpdb_import.Importer(False,settings, config, self.parent) # importer.setDropIndexes("auto") importer.setDropIndexes(_("don't drop")) importer.setFailOnError(options.failOnError) diff --git a/pyfpdb/fpdb.pyw b/pyfpdb/fpdb.pyw index 385d07ef..656284ad 100755 --- a/pyfpdb/fpdb.pyw +++ b/pyfpdb/fpdb.pyw @@ -996,7 +996,7 @@ class fpdb: def tab_bulk_import(self, widget, data=None): """opens a tab for bulk importing""" - new_import_thread = GuiBulkImport.GuiBulkImport(self.settings, self.config, self.sql) + new_import_thread = GuiBulkImport.GuiBulkImport(self.settings, self.config, self.window, self.sql) self.threads.append(new_import_thread) bulk_tab=new_import_thread.get_vbox() self.add_and_display_tab(bulk_tab, _("Bulk Import")) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index a0f7373a..e9ce1c31 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -70,12 +70,13 @@ else: psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) class Importer: - def __init__(self, caller, settings, config, sql = None): + def __init__(self, caller, settings, config, parent, sql = None): """Constructor""" self.settings = settings self.caller = caller self.config = config self.sql = sql + self.parent = parent #log = Configuration.get_logger("logging.conf", "importer", log_dir=self.config.dir_log) self.filelist = {} @@ -301,7 +302,16 @@ class Importer: totpartial = 0 toterrors = 0 tottime = 0 + progresscount = 0 + progressgoal = len(self.filelist) + + ProgressDialog = ProgressBar(self.parent) + for file in self.filelist: + + progresscount += 1 + ProgressDialog.progress_update(progresscount,progressgoal) + (stored, duplicates, partial, errors, ttime) = self.import_file_dict(db, file ,self.filelist[file][0], self.filelist[file][1], q) totstored += stored @@ -313,6 +323,8 @@ class Importer: print _("sending finish message queue length ="), q.qsize() db.send_finish_msg(q) + del ProgressDialog + return (totstored, totdups, totpartial, toterrors) # end def importFiles @@ -527,6 +539,45 @@ class Importer: logfile.write(str(s) + "\n") logfile.write("\n") logfile.close() + + +class ProgressBar: + + def __del__(self): + self.progress.destroy() + + def progress_update(self, fraction, sum): + + progresspercent = float(fraction) / (float(sum) + 1.0) + x = sometext.center(100) + + self.pbar.set_fraction(progresspercent) + self.pbar.set_text(str(fraction) + " / " + str(sum)) + + def __init__(self, parent): + + self.progress = gtk.Window(gtk.WINDOW_TOPLEVEL) + + self.progress.set_resizable(False) + self.progress.set_modal(True) + self.progress.set_transient_for(parent) + self.progress.set_decorated(False) + + vbox = gtk.VBox(False, 5) + vbox.set_border_width(10) + self.progress.add(vbox) + vbox.show() + + align = gtk.Alignment(0.5, 0.5, 0, 0) + vbox.pack_start(align, True, True, 2) + align.show() + + self.pbar = gtk.ProgressBar() + align.add(self.pbar) + self.pbar.show() + + self.progress.show() + if __name__ == "__main__": print _("CLI for fpdb_import is now available as CliFpdb.py") From 016b0ed48f15b8b92b2e15b997a296ce1fe7a7a1 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 20:06:24 +0800 Subject: [PATCH 07/14] Tourney Graphing and Summary import update It now gets through my 200 tourney torture test. --- pyfpdb/GuiTourneyGraphViewer.py | 2 +- pyfpdb/ImapFetcher.py | 3 ++- pyfpdb/PokerStarsSummary.py | 38 ++++++++++++++++++--------------- pyfpdb/SQL.py | 2 +- pyfpdb/TourneySummary.py | 4 ++-- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/pyfpdb/GuiTourneyGraphViewer.py b/pyfpdb/GuiTourneyGraphViewer.py index e213e1f4..e8a3d41e 100644 --- a/pyfpdb/GuiTourneyGraphViewer.py +++ b/pyfpdb/GuiTourneyGraphViewer.py @@ -268,7 +268,7 @@ class GuiTourneyGraphViewer (threading.Thread): if len(winnings) == 0: return None - green = map(lambda x:float(x[0]), winnings) + green = map(lambda x:float(x[1]), winnings) #blue = map(lambda x: float(x[1]) if x[2] == True else 0.0, winnings) #red = map(lambda x: float(x[1]) if x[2] == False else 0.0, winnings) greenline = cumsum(green) diff --git a/pyfpdb/ImapFetcher.py b/pyfpdb/ImapFetcher.py index 01d75a54..b3b5785f 100755 --- a/pyfpdb/ImapFetcher.py +++ b/pyfpdb/ImapFetcher.py @@ -119,6 +119,7 @@ def runFake(db, config, infile): for summary in summaryList[1:]: result = PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summary, builtFrom = "file") + print "DEBUG: Processed: %s: tournNo: %s" % (result.tourneyId, result.tourNo) def splitPokerStarsSummaries(emailText): splitSummaries=emailText.split("\nPokerStars Tournament #")[1:] @@ -136,7 +137,7 @@ def main(argv=None): sys.exit(0) # These options should really come from the OptionsParser - config = Configuration.Config(file = "HUD_config.test.xml") + config = Configuration.Config() db = Database.Database(config) sql = SQL.Sql(db_server = 'sqlite') settings = {} diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index 732e8a85..36ceae48 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -65,12 +65,15 @@ class PokerStarsSummary(TourneySummary): \#(?P[0-9]+),\s (?PNo\sLimit|Limit|LIMIT|Pot\sLimit)\s (?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s - Buy-In:\s\$(?P[.0-9]+)\/\$(?P[.0-9]+)\s + (?P[ a-zA-Z]+\s)? + (Buy-In:\s\$(?P[.0-9]+)(\/\$(?P[.0-9]+))?\s)? (?P[0-9]+)\splayers\s - Total\sPrize\sPool:\s\$?(?P[.0-9]+)\s + (\$?(?P[.\d]+)\sadded\sto\sthe\sprize\spool\sby\sPokerStars\.com\s)? + (Total\sPrize\sPool:\s\$?(?P[.0-9]+)\s)? + (Target\sTournament\s.*)? Tournament\sstarted\s-\s (?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\-\s]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)\s?\(?(?P[A-Z]+)\)\s - """ % substitutions ,re.VERBOSE|re.MULTILINE) + """ % substitutions ,re.VERBOSE|re.MULTILINE|re.DOTALL) re_Currency = re.compile(u"""(?P[%(LS)s]|FPP)""" % substitutions) @@ -190,22 +193,23 @@ class PokerStarsSummary(TourneySummary): #end def parseSummary def parseSummaryFile(self): - print self.summaryText m = self.re_TourneyInfo.search(self.summaryText) - if m: - print "DEBUG: m.groupdict(): %s" % m.groupdict() - else: - tmp = self.summaryText[0:100] + if m == None: + tmp = self.summaryText[0:200] log.error(_("parseSummaryFile: Unable to recognise Tourney Info: '%s'") % tmp) log.error(_("parseSummaryFile: Raising FpdbParseError")) raise FpdbParseError(_("Unable to recognise Tourney Info: '%s'") % tmp) + #print "DEBUG: m.groupdict(): %s" % m.groupdict() + mg = m.groupdict() if 'TOURNO' in mg: self.tourNo = mg['TOURNO'] if 'LIMIT' in mg: self.gametype['limitType'] = self.limits[mg['LIMIT']] if 'GAME' in mg: self.gametype['category'] = self.games[mg['GAME']][1] - if 'BUYIN' in mg: self.buyin = int(100*Decimal(mg['BUYIN'])) - if 'FEE' in mg: self.fee = int(100*Decimal(mg['FEE'])) + if mg['BUYIN'] != None: + self.buyin = int(100*Decimal(mg['BUYIN'])) + if mg['FEE'] != None: + self.fee = int(100*Decimal(mg['FEE'])) if 'PRIZEPOOL' in mg: self.prizepool = mg['PRIZEPOOL'] if 'ENTRIES' in mg: self.entries = mg['ENTRIES'] @@ -217,12 +221,11 @@ class PokerStarsSummary(TourneySummary): m = self.re_Currency.search(self.summaryText) - if m: - print "DEBUG: m.groupdict(): %s" % m.groupdict() - else: + if m == None: log.error(_("parseSummaryFile: Unable to locate currency")) log.error(_("parseSummaryFile: Raising FpdbParseError")) raise FpdbParseError(_("Unable to locate currency")) + #print "DEBUG: m.groupdict(): %s" % m.groupdict() mg = m.groupdict() if mg['CURRENCY'] == "$": self.currency = "USD" @@ -232,7 +235,7 @@ class PokerStarsSummary(TourneySummary): m = self.re_Player.finditer(self.summaryText) for a in m: mg = a.groupdict() - print "DEBUG: a.groupdict(): %s" % mg + #print "DEBUG: a.groupdict(): %s" % mg name = mg['NAME'] rank = mg['RANK'] winnings = 0 @@ -240,15 +243,16 @@ class PokerStarsSummary(TourneySummary): if 'WINNINGS' in mg and mg['WINNINGS'] != None: winnings = int(100*Decimal(mg['WINNINGS'])) - if 'STILLPLAYING' in mg: + if 'STILLPLAYING' in mg and mg['STILLPLAYING'] != None: #print "stillplaying" rank=None winnings=None #TODO: currency, ko/addon/rebuy count -> need examples! + #print "DEBUG: addPlayer(%s, %s, %s, %s, None, None, None)" %(rank, name, winnings, self.currency) + #print "DEBUG: self.buyin: %s self.fee %s" %(self.buyin, self.fee) self.addPlayer(rank, name, winnings, self.currency, None, None, None) - print self - sys.exit(0) + #print self #end class PokerStarsSummary diff --git a/pyfpdb/SQL.py b/pyfpdb/SQL.py index 87857908..fff3f07e 100644 --- a/pyfpdb/SQL.py +++ b/pyfpdb/SQL.py @@ -3050,7 +3050,7 @@ class Sql: # Tourney Results query #################################### self.query['tourneyResults'] = """ - SELECT (tp.winnings - tt.buyIn - tt.fee) as profit, tp.koCount, tp.rebuyCount, tp.addOnCount, tt.buyIn, tt.fee + SELECT tp.tourneyId, (tp.winnings - tt.buyIn - tt.fee) as profit, tp.koCount, tp.rebuyCount, tp.addOnCount, tt.buyIn, tt.fee, t.siteTourneyNo FROM TourneysPlayers tp INNER JOIN Players pl ON (pl.id = tp.playerId) INNER JOIN Tourneys t ON (t.id = tp.tourneyId) diff --git a/pyfpdb/TourneySummary.py b/pyfpdb/TourneySummary.py index 8fe7ceaa..f78d96de 100644 --- a/pyfpdb/TourneySummary.py +++ b/pyfpdb/TourneySummary.py @@ -74,8 +74,8 @@ class TourneySummary(object): self.endTime = None self.tourNo = None self.currency = None - self.buyin = None - self.fee = None + self.buyin = 0 + self.fee = 0 self.hero = None self.maxseats = 0 self.entries = 0 From c4203c87b1e8dcb3c2caf86b63c6e17b357f7511 Mon Sep 17 00:00:00 2001 From: gimick Date: Sat, 4 Sep 2010 13:08:58 +0100 Subject: [PATCH 08/14] import: fix error in progress bar --- pyfpdb/fpdb_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyfpdb/fpdb_import.py b/pyfpdb/fpdb_import.py index e9ce1c31..630c0d73 100755 --- a/pyfpdb/fpdb_import.py +++ b/pyfpdb/fpdb_import.py @@ -549,7 +549,6 @@ class ProgressBar: def progress_update(self, fraction, sum): progresspercent = float(fraction) / (float(sum) + 1.0) - x = sometext.center(100) self.pbar.set_fraction(progresspercent) self.pbar.set_text(str(fraction) + " / " + str(sum)) From b982658446987e957781eb08211409092118992d Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 20:36:31 +0800 Subject: [PATCH 09/14] PSSummary: identify step tickets --- pyfpdb/PokerStarsSummary.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index 36ceae48..dc6d8292 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -77,7 +77,7 @@ class PokerStarsSummary(TourneySummary): re_Currency = re.compile(u"""(?P[%(LS)s]|FPP)""" % substitutions) - re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)?(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?""") + re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?((?PTournament\sTicket)\s\(WSOP\sStep\s(?P\d)\))?""") re_DateTime = re.compile("\[(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)") @@ -248,6 +248,10 @@ class PokerStarsSummary(TourneySummary): rank=None winnings=None + if 'TICKET' and mg['TICKET'] != None: + print "DEBUG: TODO! fix Step ticket values" + print "\tWinning = Level %s" % mg['LEVEL'] + #TODO: currency, ko/addon/rebuy count -> need examples! #print "DEBUG: addPlayer(%s, %s, %s, %s, None, None, None)" %(rank, name, winnings, self.currency) #print "DEBUG: self.buyin: %s self.fee %s" %(self.buyin, self.fee) From ff8147390a6f9c095aa38ad37b783d525e830585 Mon Sep 17 00:00:00 2001 From: Worros Date: Sat, 4 Sep 2010 22:18:23 +0800 Subject: [PATCH 10/14] OnGame: Betfair Stud H/L test file --- ...tudHL-USD-0.25-0.50-201008.All.streets.txt | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 pyfpdb/regression-test-files/cash/OnGame/Stud/7-StudHL-USD-0.25-0.50-201008.All.streets.txt diff --git a/pyfpdb/regression-test-files/cash/OnGame/Stud/7-StudHL-USD-0.25-0.50-201008.All.streets.txt b/pyfpdb/regression-test-files/cash/OnGame/Stud/7-StudHL-USD-0.25-0.50-201008.All.streets.txt new file mode 100644 index 00000000..df701803 --- /dev/null +++ b/pyfpdb/regression-test-files/cash/OnGame/Stud/7-StudHL-USD-0.25-0.50-201008.All.streets.txt @@ -0,0 +1,76 @@ +***** History for hand R5-78227816-62 ***** +Start hand: Sat Aug 28 20:33:38 GMT+0300 2010 +Table: Taegu [78227816] (LIMIT SEVEN_CARD_STUD_HI_LO $0.25/$0.50, ante: $0.05, Real money) +User: XMAN1 +Players in round: 8 +Seat 1: XMAN1 ($25) +Seat 3: boneos56 ($27.78) +Seat 5: grumset2007 ($3.48) +Seat 6: mylonas77 ($40.73) +Seat 7: YMAN1 ($14.70) +Seat 8: Lee Clayton ($14.79) +Seat 9: guggi_cool ($8.34) +Seat 10: HangEv ($29.08) +boneos56 posts ante $0.05 +grumset2007 posts ante $0.05 +YMAN1 posts ante $0.05 +Lee Clayton posts ante $0.05 +guggi_cool posts ante $0.05 +HangEv posts ante $0.05 +mylonas77 posts ante $0.05 +XMAN1 posts ante $0.05 +--- +Dealing pocket cards +Dealing to XMAN1: [8d, 8s, 4d] +Dealing to boneos56: [-, -, Kd] +Dealing to grumset2007: [-, -, Ks] +Dealing to mylonas77: [-, -, 5c] +Dealing to YMAN1: [-, -, As] +Dealing to Lee Clayton: [-, -, Js] +Dealing to guggi_cool: [-, -, 9s] +Dealing to HangEv: [-, -, 6s] +XMAN1 small bring in $0.12 +boneos56 folds +grumset2007 folds +mylonas77 calls $0.12 +YMAN1 folds +Lee Clayton folds +guggi_cool folds +HangEv folds +--- +Dealing 4th street +Dealing to XMAN1: [3h] +Dealing to mylonas77: [2h] +mylonas77 bets $0.25 +XMAN1 calls $0.25 +--- +Dealing 5th street +Dealing to XMAN1: [8c] +Dealing to mylonas77: [Kh] +mylonas77 bets $0.50 +XMAN1 raises $1 to $1.37 +mylonas77 calls $0.50 +--- +Dealing 6th street +Dealing to XMAN1: [4c] +Dealing to mylonas77: [Qh] +XMAN1 bets $0.50 +mylonas77 calls $0.50 +--- +Dealing river +Dealing to XMAN1: [5h] +XMAN1 bets $0.50 +mylonas77 calls $0.50 +--- +Summary: +Main pot: $5.14 won by XMAN1 ($4.89) +Rake taken: $0.25 +Seat 1: XMAN1 ($27.47), net: +$2.47, [8d, 8s, 4d, 3h, 8c, 4c, 5h] (FULL_HOUSE EIGHT, FOUR) +Seat 3: boneos56 ($27.73), net: -$0.05 +Seat 5: grumset2007 ($3.43), net: -$0.05 +Seat 6: mylonas77 ($38.31), net: -$2.42 +Seat 7: YMAN1 ($14.65), net: -$0.05 +Seat 8: Lee Clayton ($14.74), net: -$0.05 +Seat 9: guggi_cool ($8.29), net: -$0.05 +Seat 10: HangEv ($29.03), net: -$0.05 +***** End of hand R5-78227816-62 ***** From 3bd8f3d3520520ec9aee2fde175be9942fca8271 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 5 Sep 2010 00:55:55 +0800 Subject: [PATCH 11/14] PSSummary: *mutters abaout \r\n* --- pyfpdb/PokerStarsSummary.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyfpdb/PokerStarsSummary.py b/pyfpdb/PokerStarsSummary.py index dc6d8292..0066d0de 100644 --- a/pyfpdb/PokerStarsSummary.py +++ b/pyfpdb/PokerStarsSummary.py @@ -64,12 +64,12 @@ class PokerStarsSummary(TourneySummary): re_TourneyInfo = re.compile(u""" \#(?P[0-9]+),\s (?PNo\sLimit|Limit|LIMIT|Pot\sLimit)\s - (?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s - (?P[ a-zA-Z]+\s)? - (Buy-In:\s\$(?P[.0-9]+)(\/\$(?P[.0-9]+))?\s)? - (?P[0-9]+)\splayers\s - (\$?(?P[.\d]+)\sadded\sto\sthe\sprize\spool\sby\sPokerStars\.com\s)? - (Total\sPrize\sPool:\s\$?(?P[.0-9]+)\s)? + (?PHold\'em|Razz|RAZZ|7\sCard\sStud|7\sCard\sStud\sHi/Lo|Omaha|Omaha\sHi/Lo|Badugi|Triple\sDraw\s2\-7\sLowball|5\sCard\sDraw)\s+ + (?P[ a-zA-Z]+\s+)? + (Buy-In:\s\$(?P[.0-9]+)(\/\$(?P[.0-9]+))?\s+)? + (?P[0-9]+)\splayers\s+ + (\$?(?P[.\d]+)\sadded\sto\sthe\sprize\spool\sby\sPokerStars\.com\s+)? + (Total\sPrize\sPool:\s\$?(?P[.0-9]+)\s+)? (Target\sTournament\s.*)? Tournament\sstarted\s-\s (?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\-\s]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)\s?\(?(?P[A-Z]+)\)\s @@ -77,7 +77,7 @@ class PokerStarsSummary(TourneySummary): re_Currency = re.compile(u"""(?P[%(LS)s]|FPP)""" % substitutions) - re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?((?PTournament\sTicket)\s\(WSOP\sStep\s(?P\d)\))?""") + re_Player = re.compile(u"""(?P[0-9]+):\s(?P.*)\s\(.*\),(\s)(\$(?P[0-9]+\.[0-9]+))?(?Pstill\splaying)?((?PTournament\sTicket)\s\(WSOP\sStep\s(?P\d)\))?\s+?""") re_DateTime = re.compile("\[(?P[0-9]{4})\/(?P[0-9]{2})\/(?P[0-9]{2})[\- ]+(?P[0-9]+):(?P[0-9]+):(?P[0-9]+)") From 7ac652c12ec036f60684dc70f4395ab4bdad6004 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 5 Sep 2010 01:12:31 +0800 Subject: [PATCH 12/14] THP: Fix Importer call --- pyfpdb/TestHandsPlayers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/TestHandsPlayers.py b/pyfpdb/TestHandsPlayers.py index 944499ed..60091567 100755 --- a/pyfpdb/TestHandsPlayers.py +++ b/pyfpdb/TestHandsPlayers.py @@ -129,7 +129,7 @@ def main(argv=None): settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) db.recreate_tables() - importer = fpdb_import.Importer(False, settings, config) + importer = fpdb_import.Importer(False, settings, config, None) importer.setDropIndexes("don't drop") importer.setFailOnError(True) importer.setThreads(-1) From edefd3ffb95bca82015546a8572534513ebe54e5 Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 5 Sep 2010 01:14:08 +0800 Subject: [PATCH 13/14] ImapFetcher: Give some more meaningful output --- pyfpdb/ImapFetcher.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pyfpdb/ImapFetcher.py b/pyfpdb/ImapFetcher.py index b3b5785f..304fce2d 100755 --- a/pyfpdb/ImapFetcher.py +++ b/pyfpdb/ImapFetcher.py @@ -26,6 +26,7 @@ import re import Configuration import Database +from Exceptions import FpdbParseError import SQL import Options import PokerStarsSummary @@ -72,26 +73,34 @@ def run(config, db): response, searchData = server.search(None, "SUBJECT", "PokerStars Tournament History Request") for messageNumber in searchData[0].split(" "): response, headerData = server.fetch(messageNumber, "(BODY[HEADER.FIELDS (SUBJECT)])") - #print "response to fetch subject:",response if response!="OK": raise error #TODO: show error message neededMessages.append(("PS", messageNumber)) - + + print "ImapFetcher: Found %s messages to fetch" %(len(neededMessages)) + if (len(neededMessages)==0): raise error #TODO: show error message - for messageData in neededMessages: + + errors = 0 + for i, messageData in enumerate(neededMessages, start=1): + print "Retrieving message %s" % i response, bodyData = server.fetch(messageData[1], "(UID BODY[TEXT])") bodyData=bodyData[0][1] if response!="OK": raise error #TODO: show error message if messageData[0]=="PS": summaryTexts=(splitPokerStarsSummaries(bodyData)) - for summaryText in summaryTexts: - result=PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summaryText, builtFrom = "IMAP") - #print "finished importing a PS summary with result:",result - #TODO: count results and output to shell like hand importer does - - print _("completed running Imap import, closing server connection") + print "Found %s summaries in email" %(len(summaryTexts)) + for j, summaryText in enumerate(summaryTexts, start=1): + try: + result=PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summaryText, builtFrom = "IMAP") + except FpdbParseError, e: + errors += 1 + print "Finished importing %s/%s PS summaries" %(j, len(summaryTexts)) + + print _("Completed running Imap import, closing server connection") + print _("Errors: %s" % errors) #finally: # try: server.close() @@ -121,9 +130,6 @@ def runFake(db, config, infile): result = PokerStarsSummary.PokerStarsSummary(db=db, config=config, siteName=u"PokerStars", summaryText=summary, builtFrom = "file") print "DEBUG: Processed: %s: tournNo: %s" % (result.tourneyId, result.tourNo) -def splitPokerStarsSummaries(emailText): - splitSummaries=emailText.split("\nPokerStars Tournament #")[1:] - def main(argv=None): if argv is None: From 025b491c40ba6a2123d5b766f01a7e97812936bb Mon Sep 17 00:00:00 2001 From: Worros Date: Sun, 5 Sep 2010 01:14:57 +0800 Subject: [PATCH 14/14] OnGame: tart prep for Stud parsing --- pyfpdb/OnGameToFpdb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyfpdb/OnGameToFpdb.py b/pyfpdb/OnGameToFpdb.py index d54972ac..ecdc6cda 100755 --- a/pyfpdb/OnGameToFpdb.py +++ b/pyfpdb/OnGameToFpdb.py @@ -66,7 +66,7 @@ class OnGame(HandHistoryConverter): # 'Razz' : ('stud','razz'), # 'RAZZ' : ('stud','razz'), # '7 Card Stud' : ('stud','studhi'), - # '7 Card Stud Hi/Lo' : ('stud','studhilo'), + 'SEVEN_CARD_STUD_HI_LO' : ('stud','studhilo'), # 'Badugi' : ('draw','badugi'), # 'Triple Draw 2-7 Lowball' : ('draw','27_3draw'), # '5 Card Draw' : ('draw','fivedraw') @@ -117,8 +117,6 @@ class OnGame(HandHistoryConverter): (?P[0-9]{4}) """, re.MULTILINE|re.VERBOSE) - # self.rexx.button_re = re.compile('#SUMMARY\nDealer: (?P.*)\n') - #Seat 1: .Lucchess ($4.17 in chips) #Seat 1: phantomaas ($27.11) #Seat 5: mleo17 ($9.37) @@ -165,6 +163,7 @@ class OnGame(HandHistoryConverter): return [ ["ring", "hold", "fl"], ["ring", "hold", "nl"], + ["ring", "stud", "fl"], ] def determineGameType(self, handText):