Roundup 1.2.1, Python 2.5 and sqlite:
UnicodeEncodeError for German umlauts

I have installed Roundup 1.2.1 on Windows with Python
2.5 and am using the sqlite backend (with the sqlite
that is now included in the Python 2.5 distribution).

When trying to save a message that contains German
umlauts (äöü), an exception occurs.

I get the following traceback:

Traceback (most recent call last):
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\client.py",
line 730, in renderContext
    result = pt.render(self, None, None, **args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\templating.py",
line 323, in render
    getEngine().getContext(c), output, tal=1,
strictinsert=0)()
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 192, in __call__
    self.interpret(self.program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 666, in do_useMacro
    self.interpret(macro)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 411, in do_optTag_tal
    self.do_optTag(stuff)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 396, in do_optTag
    return self.no_tag(start, program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 391, in no_tag
    self.interpret(program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 689, in do_defineSlot
    self.interpret(slot)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 632, in do_condition
    self.interpret(block)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 632, in do_condition
    self.interpret(block)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 608, in do_loop_tal
    self.interpret(block)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 411, in do_optTag_tal
    self.do_optTag(stuff)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 396, in do_optTag
    return self.no_tag(start, program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 391, in no_tag
    self.interpret(program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 544, in do_insertTranslation
    self.interpretWithStream(stuff[1], tmpstream)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 216, in interpretWithStream
    self.interpret(program)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 516, in do_i18nVariable
    value = self.engine.evaluate(expression)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\PageTemplates\TALES.py",
line 227, in evaluate
    return expression(self)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\PageTemplates\Expressions.py",
line 194, in __call__
    return self._eval(econtext)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\PageTemplates\Expressions.py",
line 182, in _eval
    ob = self._subexprs[-1](econtext)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\PageTemplates\Expressions.py",
line 138, in _eval
    ob = restrictedTraverse(ob, path, getSecurityManager())
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\PageTemplates\Expressions.py",
line 317, in restrictedTraverse
    o = guarded_getattr(object, name, M)
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\templating.py",
line 830, in __getattr__
    return self[attr]
  File
"C:\Programme\Python25\lib\site-packages\roundup\cgi\templating.py",
line 807, in __getitem__
    value = self._klass.get(self._nodeid, items[0], None)
  File
"C:\Programme\Python25\lib\site-packages\roundup\backends\rdbms_common.py",
line 2626, in get
    return Class.get(self, nodeid, propname, default)
  File
"C:\Programme\Python25\lib\site-packages\roundup\backends\rdbms_common.py",
line 1460, in get
    d = self.db.getnode(self.classname, nodeid)
  File
"C:\Programme\Python25\lib\site-packages\roundup\backends\rdbms_common.py",
line 963, in getnode
    value =
self.sql_to_hyperdb_value[props[name].__class__](value)
UnicodeEncodeError: 'ascii' codec can't encode
characters in position 0-2: ordinal not in range(128)

The problem does not occur with Roundup 1.2.1, Python
2.4.4 and pysqlite 1.1.8.

I also wanted to try Roundup 1.2.1, Python 2.4.4 and
pysqlite 2.3.2, but this combination gave me the
following error during tracker initialisation:

Traceback (most recent call last):
  File "<string>", line 1, in ?
  File
"C:\Python24\Lib\site-packages\roundup\scripts\roundup_admin.py",
line 38, in run
    sys.exit(tool.main())
  File
"C:\Python24\Lib\site-packages\roundup\admin.py", line
1491, in main
    ret = self.run_command(args)
  File
"C:\Python24\Lib\site-packages\roundup\admin.py", line
1369, in run_command
    return self.do_initialise(self.tracker_home, args)
  File
"C:\Python24\Lib\site-packages\roundup\admin.py", line
514, in do_initialise
    tracker.init(password.Password(adminpw))
  File
"C:\Python24\Lib\site-packages\roundup\instance.py",
line 154, in init
    db = self.open('admin')
  File
"C:\Python24\Lib\site-packages\roundup\instance.py",
line 84, in open
    vars = {
  File
"C:\Python24\Lib\site-packages\roundup\backends\rdbms_common.py",
line 129, in __init__
    self.open_connection()
  File
"C:\Python24\Lib\site-packages\roundup\backends\back_sqlite.py",
line 114, in open_connection
    (self.conn, self.cursor) = self.sql_open_connection()
  File
"C:\Python24\Lib\site-packages\roundup\backends\back_sqlite.py",
line 105, in sql_open_connection
    conn = sqlite.connect(db=db)
TypeError: function takes at least 1 argument (0 given)

Logged In: YES 
user_id=6405

The error you're getting is not related to unicode, and has been fixed in CVS. Expect roundup 1.3.0 to be released very soon.

If you need the code sooner, you can grab back_sqlite.py from CVS via the CVS browser on sf.net

Logged In: YES 
user_id=1195809

Hi Richard,

Actually this bug report contains two problems. Sorry about
that, maybe these should have been two separate bug reports.

Bug 1: The UnicodeEncodeError for German umlauts
Bug 2: The inability to use Python 2.4.4 with pysqlite 2.3.2

I have just tested Roundup 1.3.0. It turns out that Bug 2 is
fixed, whereas Bug 1 isn't.

I.e. German Umlauts still give me the same traceback with
the UnicodeEncodeError if I use
- Python 2.5 with the integrated pysqlite or
- Python 2.4.4 with pysqlite 2.3.2
but not if I use
- Python 2.4.4 with pysqlite 1.1.8.

Thanks and best regards,
Christoph

Logged In: YES 
user_id=6405

Sorry, my bad. I thought I'd tested this under 2.5 but hadn't. It's definitely 
fixed now.

Logged In: YES 
user_id=1195809

Hi Richard,

With 1.3.1, I can now enter Umlauts in messages! :-)

However, I cannot use them in the search box. :-(

Searching for "äöü" (in message bodies) gives me the
traceback below.

Thanks and best regards,
Christoph

Traceback (most recent call last):
  File
"C:\Python25\Lib\site-packages\roundup\cgi\client.py", line
732, in renderContext
    result = pt.render(self, None, None, **args)
  File
"C:\Python25\Lib\site-packages\roundup\cgi\templating.py",
line 323, in render
    getEngine().getContext(c), output, tal=1, strictinsert=0)()
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 192, in __call__
    self.interpret(self.program)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 666, in do_useMacro
    self.interpret(macro)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 411, in do_optTag_tal
    self.do_optTag(stuff)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 396, in do_optTag
    return self.no_tag(start, program)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 391, in no_tag
    self.interpret(program)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 236, in interpret
    handlers[opcode](self, args)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 265, in do_startEndTag
    self.do_startTag(stuff, self.endsep, self.endlen)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 293, in do_startTag
    ok, name, s = attrAction(self, item)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 372, in attrAction_tal
    translated = self.translate(msgid or value, value, {})
  File
"C:\Python25\lib\site-packages\roundup\cgi\TAL\TALInterpreter.py",
line 618, in translate
    msgid, i18ndict, default=default)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TranslationService.py",
line 77, in translate
    target_language=target_language)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TranslationService.py",
line 32, in translate
    _msg = self.gettext(msgid)
  File
"C:\Python25\lib\site-packages\roundup\cgi\TranslationService.py",
line 38, in gettext
    return self.ugettext(msgid).encode(self.OUTPUT_ENCODING)
  File "C:\Python25\lib\gettext.py", line 404, in ugettext
    return unicode(message)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in
position 0: ordinal not in range(128)

Sorry, I can't reproduce this. Exactly what are the steps I need to take to reproduce it? Please start with a new demo instance. Also, exactly what Python and backend are you using?

Logged In: NO 

Hi Richard,

This is strange because I can reproduce the problem easily by following these steps on a Windows XP system:

1.) Install Python 2.5
2.) Install pywin32 build 210
3.) Extract the Roundup 1.3.1 tarball
4.) Run demo.py
5.) Search for "äöü" (in the search box in the upper-right corner)

(My system is in German, I am not sure if that makes a difference.)

Best regards,
Christoph

can't be reproduced and hasn't been reported again.
Please submit new bug if this happens with recent version.