e17058953c
Kangaderoo has a patch which potentially fixes storage and display issues for users who do not have their database text storage as utf8. Functions and variables added to Charset to disect the patch
96 lines
2.9 KiB
Python
96 lines
2.9 KiB
Python
#!/usr/bin/python
|
|
|
|
#Copyright 2010 Mika Bostrom
|
|
#This program is free software: you can redistribute it and/or modify
|
|
#it under the terms of the GNU Affero General Public License as published by
|
|
#the Free Software Foundation, version 3 of the License.
|
|
#
|
|
#This program is distributed in the hope that it will be useful,
|
|
#but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
#GNU General Public License for more details.
|
|
#
|
|
#You should have received a copy of the GNU Affero General Public License
|
|
#along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#In the "official" distribution you can find the license in
|
|
#agpl-3.0.txt in the docs folder of the package.
|
|
|
|
# Error logging
|
|
import sys
|
|
|
|
# String manipulation
|
|
import codecs
|
|
|
|
# Settings
|
|
import Configuration
|
|
|
|
encoder_to_utf = codecs.lookup('utf-8')
|
|
encoder_to_sys = codecs.lookup(Configuration.LOCALE_ENCODING)
|
|
coder_hex = codecs.lookup('hex_codec')
|
|
|
|
hex_coding = False #FIXME: Should only be on if db is not UTF8 - test in Database.py?
|
|
# I'm saving a few cycles with this one
|
|
not_needed1, not_needed2, not_needed3 = False, False, False
|
|
if Configuration.LOCALE_ENCODING == 'UTF8':
|
|
not_needed1, not_needed2, not_needed3 = True, True, True
|
|
|
|
def to_utf8(s):
|
|
if not_needed1: return s
|
|
|
|
try:
|
|
#(_out, _len) = encoder_to_utf.encode(s)
|
|
_out = unicode(s, Configuration.LOCALE_ENCODING).encode('utf-8')
|
|
return _out
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Could not convert: "%s"\n' % s)
|
|
raise
|
|
except UnicodeEncodeError:
|
|
sys.stderr.write('Could not encode: "%s"\n' % s)
|
|
raise
|
|
except TypeError: # TypeError is raised when we give unicode() an already encoded string
|
|
return s
|
|
|
|
def to_db_utf8(s):
|
|
if not_needed2: return s
|
|
|
|
try:
|
|
(_out, _len) = encoder_to_utf.encode(unicode(s))
|
|
return _out
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Could not convert: "%s"\n' % s)
|
|
raise
|
|
except UnicodeEncodeError:
|
|
sys.stderr.write('Could not encode: "%s"\n' % s)
|
|
raise
|
|
|
|
def to_gui(s):
|
|
if not_needed3: return s
|
|
|
|
try:
|
|
# we usually don't want to use 'replace' but this is only for displaying
|
|
# in the gui so it doesn't matter if names are missing an accent or two
|
|
(_out, _len) = encoder_to_sys.encode(s, 'replace')
|
|
return _out
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Could not convert: "%s"\n' % s)
|
|
raise
|
|
except UnicodeEncodeError:
|
|
sys.stderr.write('Could not encode: "%s"\n' % s)
|
|
raise
|
|
|
|
def to_hex(s):
|
|
try:
|
|
out = coder_hex.encode(s)[0]
|
|
return out
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Could not convert: "%s"\n' % s)
|
|
return s
|
|
|
|
def from_hex(s):
|
|
try:
|
|
out = coder_hex.decode(s)[0]
|
|
return out
|
|
except UnicodeDecodeError:
|
|
sys.stderr.write('Could not convert: "%s"\n' % s)
|
|
return s
|