From 3b1e67e78bcb0051fef3ade940466dfd403d8214 Mon Sep 17 00:00:00 2001 From: Mika Bostrom Date: Thu, 24 Dec 2009 09:52:47 +0200 Subject: [PATCH] Store names as UTF-8 The names should be always in UTF-8 encoding. At least for PostgreSQL the encdoding of the database comes from the time of running 'initdb' (which is different from 'createdb') and if the encoding was selected or set to something else at that time, the following error will occur: File ".../pyfpdb/Database.py", line 1630, in self.pcache = LambdaDict(lambda key:self.insertPlayer(key, siteid)) File ".../pyfpdb/Database.py", line 1661, in insertPlayer c.execute (q, (site_id, _name)) File "/usr/lib/python2.5/encodings/iso8859_15.py", line 12, in encode return codecs.charmap_encode(input,errors,encoding_table) UnicodeEncodeError: 'charmap' codec can't encode character u'\u2122' in position 10: character maps to This happens because 'name' is a regular string as opposed to a valid unicode object. By forcing the string to unicode and encoding it in UTF-8 the error goes away. In my case the database encoding was ISO-8859-15 (latin9) but any other "wrong" encoding would trigger the same problem. This is a relatively common problem in python. --- pyfpdb/Database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyfpdb/Database.py b/pyfpdb/Database.py index fcb0c066..fb70dbab 100644 --- a/pyfpdb/Database.py +++ b/pyfpdb/Database.py @@ -38,6 +38,7 @@ from decimal import Decimal import string import re import Queue +import codecs # pyGTK modules @@ -50,6 +51,7 @@ import Tourney from Exceptions import * log = Configuration.get_logger("logging.conf") +encoder = codecs.lookup('utf-8') class Database: @@ -1641,6 +1643,7 @@ class Database: def insertPlayer(self, name, site_id): result = None + (_name, _len) = encoder.encode(unicode(name)) c = self.get_cursor() q = "SELECT name, id FROM Players WHERE siteid=%s and name=%s" q = q.replace('%s', self.sql.query['placeholder']) @@ -1654,12 +1657,12 @@ class Database: #print "DEBUG: name: %s site: %s" %(name, site_id) - c.execute (q, (site_id, name)) + c.execute (q, (site_id, _name)) tmp = c.fetchone() if (tmp == None): #new player c.execute ("INSERT INTO Players (name, siteId) VALUES (%s, %s)".replace('%s',self.sql.query['placeholder']) - ,(name, site_id)) + ,(_name, site_id)) #Get last id might be faster here. #c.execute ("SELECT id FROM Players WHERE name=%s", (name,)) result = self.get_last_insert_id(c)