From 6dcec4800563f282502e607869df62e7b9381bb1 Mon Sep 17 00:00:00 2001 From: Mika Bostrom Date: Sun, 24 Jan 2010 19:59:49 +0200 Subject: [PATCH 01/10] Use common encoding routine everywhere The string/locale manipulation in Database.py was open-coded and did not use Charset.to_utf8() like the rest of the code. --- pyfpdb/Database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 7194b230..2d91ff4c 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -38,7 +38,6 @@ from decimal import Decimal import string import re import Queue -import codecs # pyGTK modules @@ -52,7 +51,6 @@ import Charset from Exceptions import * log = Configuration.get_logger("logging.conf") -encoder = codecs.lookup('utf-8') class Database: @@ -1560,7 +1558,7 @@ class Database: def insertPlayer(self, name, site_id): result = None - (_name, _len) = encoder.encode(unicode(name)) + _name = Charset.to_utf8(name) c = self.get_cursor() q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s" q = q.replace('%s', self.sql.query['placeholder']) From 33277ce68b2aab6bb8fa35ddef2d6ab046d8655a Mon Sep 17 00:00:00 2001 From: Mika Bostrom Date: Sun, 24 Jan 2010 21:09:30 +0200 Subject: [PATCH 02/10] Add new encoder This encoder is used to handle input from HH conversion, which needs to end up as UTF-8 in the database. Switch the open-coded routine from Database.py to this common routine so all encodings now take place in the same file. --- pyfpdb/Charset.py | 10 ++++++++++ pyfpdb/Database.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pyfpdb/Charset.py b/pyfpdb/Charset.py index f8043876..16c236b6 100644 --- a/pyfpdb/Charset.py +++ b/pyfpdb/Charset.py @@ -40,6 +40,16 @@ def to_utf8(s): print 'Could not convert: "%s"' % s raise +def to_db_utf8(s): + if not_needed: return s + + try: + (_out, _len) = encoder_to_utf.encode(unicode(s)) + return _out + except UnicodeDecodeError: + print 'Could not convert: "%s"' % s + raise + def to_gui(s): if not_needed: return s diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index 2d91ff4c..c340079c 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -1558,7 +1558,7 @@ class Database: def insertPlayer(self, name, site_id): result = None - _name = Charset.to_utf8(name) + _name = Charset.to_db_utf8(name) c = self.get_cursor() q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s" q = q.replace('%s', self.sql.query['placeholder']) From 85c9070ec8bc418237af83eb9adad8194e7cbcaa Mon Sep 17 00:00:00 2001 From: Mika Bostrom Date: Sun, 24 Jan 2010 22:17:03 +0200 Subject: [PATCH 03/10] Write charmap-related errors directly to stderr This change is needed to skip a nasty behaviour: if the string triggered a decoding error, it will trigger one *AGAIN* if the string is printed to console. By writing directly to sys.stderr we skip the locale/conversion issues and get the troublesome string directly in a file where it is stored as a raw sequence of octets. --- pyfpdb/Charset.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyfpdb/Charset.py b/pyfpdb/Charset.py index 16c236b6..86d33418 100644 --- a/pyfpdb/Charset.py +++ b/pyfpdb/Charset.py @@ -15,6 +15,9 @@ #In the "official" distribution you can find the license in #agpl-3.0.txt in the docs folder of the package. +# Error logging +import sys + # String manipulation import codecs @@ -37,7 +40,7 @@ def to_utf8(s): _out = unicode(s, Configuration.LOCALE_ENCODING).encode('utf-8') return _out except UnicodeDecodeError: - print 'Could not convert: "%s"' % s + sys.stderr.write('Could not convert: "%s"\n' % s) raise def to_db_utf8(s): @@ -47,7 +50,7 @@ def to_db_utf8(s): (_out, _len) = encoder_to_utf.encode(unicode(s)) return _out except UnicodeDecodeError: - print 'Could not convert: "%s"' % s + sys.stderr.write('Could not convert: "%s"\n' % s) raise def to_gui(s): @@ -57,6 +60,6 @@ def to_gui(s): (_out, _len) = encoder_to_sys.encode(s) return _out except UnicodeDecodeError: - print 'Could not convert: "%s"' % s + sys.stderr.write('Could not convert: "%s"\n' % s) raise From 7f04ed88f4522f7788017f9fb20c7b1d5dff3a1c Mon Sep 17 00:00:00 2001 From: Mika Bostrom Date: Tue, 26 Jan 2010 08:01:46 +0200 Subject: [PATCH 04/10] Use proper encoding name When system is unicode, the second item in locale.getdefaultlocale() is "UTF8", not "utf-8". --- pyfpdb/Charset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyfpdb/Charset.py b/pyfpdb/Charset.py index 86d33418..d49b6219 100644 --- a/pyfpdb/Charset.py +++ b/pyfpdb/Charset.py @@ -29,7 +29,7 @@ encoder_to_sys = codecs.lookup(Configuration.LOCALE_ENCODING) # I'm saving a few cycles with this one not_needed = False -if Configuration.LOCALE_ENCODING == 'utf-8': +if Configuration.LOCALE_ENCODING == 'UTF8': not_needed = True def to_utf8(s): From 241dc1e717e276d675ce71c65a7e7921e1edba86 Mon Sep 17 00:00:00 2001 From: Gerko de Roo Date: Sat, 23 Jan 2010 16:24:00 +0100 Subject: [PATCH 05/10] Fixed limit and freeroll support for Partypoker tourney's --- pyfpdb/PartyPokerToFpdb.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/pyfpdb/PartyPokerToFpdb.py b/pyfpdb/PartyPokerToFpdb.py index 4cf64547..3d5a959b 100755 --- a/pyfpdb/PartyPokerToFpdb.py +++ b/pyfpdb/PartyPokerToFpdb.py @@ -55,22 +55,22 @@ class PartyPoker(HandHistoryConverter): # $5 USD NL Texas Hold'em - Saturday, July 25, 07:53:52 EDT 2009 # NL Texas Hold'em $1 USD Buy-in Trny:45685440 Level:8 Blinds-Antes(600/1 200 -50) - Sunday, May 17, 11:25:07 MSKS 2009 re_GameInfoRing = re.compile(""" - (?P\$|)\s*(?P[0-9,]+)\s*(?:USD)?\s* - (?P(NL|PL|))\s+ + (?P\$|)\s*(?P[.,0-9]+)([.,0-9/$]+)?\s*(?:USD)?\s* + (?P(NL|PL|))\s* (?P(Texas\ Hold\'em|Omaha)) \s*\-\s* (?P.+) """, re.VERBOSE) re_GameInfoTrny = re.compile(""" - (?P(NL|PL|))\s+ + (?P(NL|PL|))\s* (?P(Texas\ Hold\'em|Omaha))\s+ - (?P\$?[.0-9]+)\s*(?PUSD)?\s*Buy-in\s+ + (?:(?P\$?[.,0-9]+)\s*(?PUSD)?\s*Buy-in\s+)? Trny:\s?(?P\d+)\s+ Level:\s*(?P\d+)\s+ - Blinds(?:-Antes)?\( - (?P[.0-9 ]+)\s* - /(?P[.0-9 ]+) - (?:\s*-\s*(?P[.0-9 ]+)\$?)? + ((Blinds|Stakes)(?:-Antes)?)\( + (?P[.,0-9 ]+)\s* + /(?P[.,0-9 ]+) + (?:\s*-\s*(?P[.,0-9 ]+)\$?)? \) \s*\-\s* (?P.+) @@ -85,10 +85,9 @@ class PartyPoker(HandHistoryConverter): re.VERBOSE) re_HandInfo = re.compile(""" - ^Table\s+ - (?P[a-zA-Z0-9 ]+)\s+ + ^Table\s+(?P[$a-zA-Z0-9 ]+)\s+ (?: \#|\(|)(?P\d+)\)?\s+ - (?:[^ ]+\s+\#(?P\d+).+)? # table number for mtt + (?:[a-zA-Z0-9 ]+\s+\#(?P\d+).+)? (\(No\sDP\)\s)? \((?PReal|Play)\s+Money\)\s+ # FIXME: check if play money is correct Seat\s+(?P