diff -r 3adff0fb0207 CHANGES.txt --- a/CHANGES.txt Fri Mar 22 15:53:27 2013 +0100 +++ b/CHANGES.txt Sat Apr 20 21:56:57 2013 -0400 @@ -6,7 +6,24 @@ 2013-??-??: 1.4.22 Features: - +- Support for tx_Source property on database handle. Can be used by + detectors to find out the source of a change in an auditor to block + changes arriving by unauthenticated mechanisms (e.g. plain email + where headers can be faked). The property db.tx_Source has the + following values: + * None - Default value set to None. May be valid if it's a script + that is created by the user. Otherwise it's an error and indicates + that some code path is not properly setting the tx_Source property. + * "cli" - this string value is set when using roundup-admin and + supplied scripts. + * "web" - this string value is set when using any web based + technique: html interface, xmlrpc .... + * "email" - this string value is set when using an unauthenticated + email based technique. + * "email-sig-openpgp" - this string value is set when email with a + valid pgp signature is used. (*NOTE* the testing for this mode + is incomplete. If you have a pgp infrastructure you should test + and verify that this is properly set.) - Introducing Template Loader API (anatoly techtonik) - Experimental support for Jinja2, try 'jinja2' for template_engine in config (anatoly techtonik) diff -r 3adff0fb0207 roundup/admin.py --- a/roundup/admin.py Fri Mar 22 15:53:27 2013 +0100 +++ b/roundup/admin.py Sat Apr 20 21:56:57 2013 -0400 @@ -1475,6 +1475,8 @@ if not self.db: self.db = tracker.open('admin') + self.db.tx_Source = 'cli' + # do the command ret = 0 try: diff -r 3adff0fb0207 roundup/cgi/client.py --- a/roundup/cgi/client.py Fri Mar 22 15:53:27 2013 +0100 +++ b/roundup/cgi/client.py Sat Apr 20 21:56:57 2013 -0400 @@ -792,12 +792,15 @@ # open the database or only set the user if not hasattr(self, 'db'): self.db = self.instance.open(username) + self.db.tx_Source = "web" else: if self.instance.optimize: self.db.setCurrentUser(username) + self.db.tx_Source = "web" else: self.db.close() self.db = self.instance.open(username) + self.db.tx_Source = "web" # The old session API refers to the closed database; # we can no longer use it. self.session_api = Session(self) diff -r 3adff0fb0207 roundup/instance.py --- a/roundup/instance.py Fri Mar 22 15:53:27 2013 +0100 +++ b/roundup/instance.py Sat Apr 20 21:56:57 2013 -0400 @@ -121,6 +121,8 @@ extension(self) detectors = self.get_extensions('detectors') db = env['db'] + db.tx_Source = None + # apply the detectors for detector in detectors: detector(db) @@ -150,6 +152,7 @@ % (classname, propname, linkto)) db.post_init() + # FIXME is this needed? db.tx_Source = None self.db_open = 1 return db diff -r 3adff0fb0207 roundup/mailgw.py --- a/roundup/mailgw.py Fri Mar 22 15:53:27 2013 +0100 +++ b/roundup/mailgw.py Sat Apr 20 21:56:57 2013 -0400 @@ -1010,6 +1010,9 @@ "be PGP encrypted.") if self.message.pgp_signed(): self.message.verify_signature(author_address) + # signature has been verified + self.db.tx_Source = "email-sig-openpgp" + elif self.message.pgp_encrypted(): # Replace message with the contents of the decrypted # message for content extraction @@ -1019,8 +1022,26 @@ encr_only = self.config.PGP_REQUIRE_INCOMING == 'encrypted' encr_only = encr_only or not pgp_role() self.crypt = True - self.message = self.message.decrypt(author_address, - may_be_unsigned = encr_only) + try: + # see if the message has a valid signature + message = self.message.decrypt(author_address, + may_be_unsigned = False) + # only set if MailUsageError is not raised + # indicating that we have a valid signature + self.db.tx_Source = "email-sig-openpgp" + except MailUsageError: + # if there is no signature or an error in the message + # we get here. Try decrypting it again if we don't + # need signatures. + if encr_only: + message = self.message.decrypt(author_address, + may_be_unsigned = encr_only) + else: + # something failed with the message decryption/sig + # chain. Pass the error up. + raise + # store the decrypted message + self.message = message elif pgp_role(): raise MailUsageError, _(""" This tracker has been configured to require all email be PGP signed or @@ -1537,6 +1558,9 @@ ''' # get database handle for handling one email self.db = self.instance.open ('admin') + + self.db.tx_Source = "email" + try: return self._handle_message(message) finally: diff -r 3adff0fb0207 scripts/add-issue --- a/scripts/add-issue Fri Mar 22 15:53:27 2013 +0100 +++ b/scripts/add-issue Sat Apr 20 21:56:57 2013 -0400 @@ -28,6 +28,7 @@ # open the tracker tracker = instance.open(tracker_home) db = tracker.open('admin') +db.tx_Source = "cli" uid = db.user.lookup('admin') try: # try to open the tracker as the current user diff -r 3adff0fb0207 scripts/copy-user.py --- a/scripts/copy-user.py Fri Mar 22 15:53:27 2013 +0100 +++ b/scripts/copy-user.py Sat Apr 20 21:56:57 2013 -0400 @@ -42,6 +42,9 @@ db1 = instance1.open('admin') db2 = instance2.open('admin') + db1.tx_Source = "cli" + db2.tx_Source = "cli" + userlist = db1.user.list() for userid in userids: try: diff -r 3adff0fb0207 scripts/spam-remover --- a/scripts/spam-remover Fri Mar 22 15:53:27 2013 +0100 +++ b/scripts/spam-remover Sat Apr 20 21:56:57 2013 -0400 @@ -93,6 +93,8 @@ cmd.show_help() tracker = instance.open(opt.instance) db = tracker.open('admin') + db.tx_Source = "cli" + users = dict.fromkeys (db.user.lookup(u) for u in opt.users) files_to_remove = {} for fn in opt.filenames: diff -r 3adff0fb0207 test/memorydb.py --- a/test/memorydb.py Fri Mar 22 15:53:27 2013 +0100 +++ b/test/memorydb.py Sat Apr 20 21:56:57 2013 -0400 @@ -50,6 +50,10 @@ execfile(os.path.join(dirname, fn), vars) vars['init'](db) + vars = {} + execfile("test/tx_Source_detector.py", vars) + vars['init'](db) + ''' status = Class(db, "status", name=String()) status.setkey("name") @@ -194,6 +198,7 @@ self.newnodes = {} # keep track of the new nodes by class self.destroyednodes = {}# keep track of the destroyed nodes by class self.transactions = [] + self.tx_Source = None def filename(self, classname, nodeid, property=None, create=0): shutil.copyfile(__file__, __file__+'.dummy') diff -r 3adff0fb0207 test/test_cgi.py --- a/test/test_cgi.py Fri Mar 22 15:53:27 2013 +0100 +++ b/test/test_cgi.py Sat Apr 20 21:56:57 2013 -0400 @@ -75,11 +75,22 @@ # open the database self.db = self.instance.open('admin') + self.db.tx_Source = "web" self.db.user.create(username='Chef', address='chef@bork.bork.bork', realname='Bork, Chef', roles='User') self.db.user.create(username='mary', address='mary@test.test', roles='User', realname='Contrary, Mary') + self.db.issue.addprop(tx_Source=hyperdb.String()) + self.db.msg.addprop(tx_Source=hyperdb.String()) + + vars = dict(globals()) + vars['db'] = self.db + vars = {} + execfile("test/tx_Source_detector.py", vars) + vars['init'](self.db) + + test = self.instance.backend.Class(self.db, "test", string=hyperdb.String(), number=hyperdb.Number(), boolean=hyperdb.Boolean(), link=hyperdb.Link('test'), @@ -207,6 +218,7 @@ self.assertEqual(self.db.issue.get(issue,'status'),'1') self.assertEqual(self.db.status.lookup('1'),'2') self.assertEqual(self.db.status.lookup('2'),'1') + self.assertEqual(self.db.issue.get('1','tx_Source'),'web') form = cgi.FieldStorage() cl = client.Client(self.instance, None, {'PATH_INFO':'/'}, form) cl.classname = 'issue' @@ -226,6 +238,7 @@ self.assertEqual(self.db.issue.get(issue,'keyword'),['1']) self.assertEqual(self.db.keyword.lookup('1'),'2') self.assertEqual(self.db.keyword.lookup('2'),'1') + self.assertEqual(self.db.issue.get(issue,'tx_Source'),'web') form = cgi.FieldStorage() cl = client.Client(self.instance, None, {'PATH_INFO':'/'}, form) cl.classname = 'issue' @@ -271,11 +284,13 @@ nodeid = self.db.issue.create(status='unread') self.assertEqual(self.parseForm({'status': 'unread'}, 'issue', nodeid), ({('issue', nodeid): {}}, [])) + self.assertEqual(self.db.issue.get(nodeid,'tx_Source'),'web') def testUnsetLink(self): nodeid = self.db.issue.create(status='unread') self.assertEqual(self.parseForm({'status': '-1'}, 'issue', nodeid), ({('issue', nodeid): {'status': None}}, [])) + self.assertEqual(self.db.issue.get(nodeid,'tx_Source'),'web') def testInvalidLinkValue(self): # XXX This is not the current behaviour - should we enforce this? diff -r 3adff0fb0207 test/test_mailgw.py --- a/test/test_mailgw.py Fri Mar 22 15:53:27 2013 +0100 +++ b/test/test_mailgw.py Sat Apr 20 21:56:57 2013 -0400 @@ -129,6 +129,8 @@ return res +from roundup.hyperdb import String + class MailgwTestAbstractBase(unittest.TestCase, DiffHelper): count = 0 schema = 'classic' @@ -139,6 +141,11 @@ # and open the database / "instance" self.db = memorydb.create('admin') + self.db.tx_Source = "email" + + self.db.issue.addprop(tx_Source=String()) + self.db.msg.addprop(tx_Source=String()) + self.instance = Tracker() self.instance.db = self.db self.instance.config = self.db.config @@ -206,6 +213,7 @@ ''') assert not os.path.exists(SENDMAILDEBUG) self.assertEqual(self.db.issue.get(nodeid, 'title'), 'Testing...') + self.assertEqual(self.db.issue.get(nodeid, 'tx_Source'), 'email') class MailgwTestCase(MailgwTestAbstractBase): @@ -320,6 +328,11 @@ l = self.db.issue.get(nodeid, 'nosy') l.sort() self.assertEqual(l, [self.chef_id, self.richard_id]) + + # check that the message has the right source code + l = self.db.msg.get('1', 'tx_Source') + self.assertEqual(l, 'email') + return nodeid def testNewIssue(self): @@ -413,6 +426,7 @@ nosy: Chef, mary, richard status: unread title: Testing... +tx_Source: email _______________________________________________________________________ Roundup issue tracker @@ -455,6 +469,7 @@ nosy: Chef, mary, richard status: unread title: Testing... +tx_Source: email _______________________________________________________________________ Roundup issue tracker @@ -499,6 +514,7 @@ nosy: Chef, mary, richard status: unread title: Testing... +tx_Source: email _______________________________________________________________________ Roundup issue tracker @@ -1195,7 +1211,6 @@ X-Roundup-Issue-Status: chatting Content-Transfer-Encoding: quoted-printable - richard added the comment: This is a followup @@ -1230,6 +1245,10 @@ self.assertEqual(l, [self.chef_id, self.richard_id, self.mary_id, self.john_id]) + # check that the message has the right tx_Source + l = self.db.msg.get('2', 'tx_Source') + self.assertEqual(l, 'email') + self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, mary@test.test @@ -1247,7 +1266,6 @@ X-Roundup-Issue-Status: chatting Content-Transfer-Encoding: quoted-printable - richard added the comment: This is a followup @@ -1265,6 +1283,7 @@ ''') def testNosyGeneration(self): + self.db.tx_Source = "email" self.db.issue.create(title='test') # create a nosy message @@ -1274,6 +1293,10 @@ l = self.db.issue.create(title='test', messages=[msg], nosy=[self.chef_id, self.mary_id, self.john_id]) + + # check that message has right tx_Source + self.assertEqual(self.db.msg.get('1', 'tx_Source'), 'email') + self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, mary@test.test @@ -1300,6 +1323,7 @@ nosy: Chef, john, mary, richard status: unread title: test +tx_Source: email _______________________________________________________________________ Roundup issue tracker @@ -1373,6 +1397,10 @@ This is a followup ''') + + l = self.db.msg.get('2', 'tx_Source') + self.assertEqual(l, 'email') + self.compareMessages(self._get_mail(), '''FROM: roundup-admin@your.tracker.email.domain.example TO: chef@bork.bork.bork, john@test.test, mary@test.test @@ -3390,6 +3418,10 @@ m = self.db.issue.get(nodeid, 'messages')[0] self.assertEqual(self.db.msg.get(m, 'content'), 'This is a pgp signed message.') + # check that the message has the right source code + l = self.db.msg.get(m, 'tx_Source') + self.assertEqual(l, 'email-sig-openpgp') + def testPGPSignedMessageFail(self): # require both, signing and encryption @@ -3441,6 +3473,9 @@ m = self.db.issue.get(nodeid, 'messages')[0] self.assertEqual(self.db.msg.get(m, 'content'), 'This is the text to be encrypted') + # check that the message has the right source code + l = self.db.msg.get(m, 'tx_Source') + self.assertEqual(l, 'email') def testPGPEncryptedUnsignedMessageFromNonPGPUser(self): msg = self.encrypted_msg.replace('John Doe ', @@ -3450,6 +3485,10 @@ self.assertEqual(self.db.msg.get(m, 'content'), 'This is the text to be encrypted') self.assertEqual(self.db.msg.get(m, 'author'), self.mary_id) + # check that the message has the right source code + l = self.db.msg.get(m, 'tx_Source') + self.assertEqual(l, 'email') + # check that a bounce-message that is triggered *after* # decrypting is properly encrypted: @@ -3533,7 +3572,9 @@ m = self.db.issue.get(nodeid, 'messages')[0] self.assertEqual(self.db.msg.get(m, 'content'), 'This is the text of a signed and encrypted email.') - + # check that the message has the right source code + l = self.db.msg.get(m, 'tx_Source') + self.assertEqual(l, 'email-sig-openpgp') def test_suite(): suite = unittest.TestSuite() diff -r 3adff0fb0207 test/test_userauditor.py --- a/test/test_userauditor.py Fri Mar 22 15:53:27 2013 +0100 +++ b/test/test_userauditor.py Sat Apr 20 21:56:57 2013 -0400 @@ -6,6 +6,7 @@ self.dirname = '_test_user_auditor' self.instance = setupTracker(self.dirname) self.db = self.instance.open('admin') + self.db.tx_Source = "cli" try: import pytz diff -r 3adff0fb0207 test/test_xmlrpc.py --- a/test/test_xmlrpc.py Fri Mar 22 15:53:27 2013 +0100 +++ b/test/test_xmlrpc.py Sat Apr 20 21:56:57 2013 -0400 @@ -10,6 +10,7 @@ from roundup import init, instance, password, hyperdb, date from roundup.xmlrpc import RoundupInstance from roundup.backends import list_backends +from roundup.hyperdb import String import db_test_base @@ -26,6 +27,8 @@ # open the database self.db = self.instance.open('admin') + + # Get user id (user4 maybe). Used later to get data from db. self.joeid = 'user' + self.db.user.create(username='joe', password=password.Password('random'), address='random@home.org', realname='Joe Random', roles='User') @@ -33,9 +36,27 @@ self.db.commit() self.db.close() self.db = self.instance.open('joe') + self.db.tx_Source = 'web' + + # for some reason the db changes aren't preserved across the + # commit/close above. To have them seen we have to set them up + # in the database open of the joe user. + + self.db.issue.addprop(tx_Source=hyperdb.String()) + self.db.msg.addprop(tx_Source=hyperdb.String()) + + vars = dict(globals()) + vars['db'] = self.db + vars = {} + execfile("test/tx_Source_detector.py", vars) + vars['init'](self.db) + + print "setUpCalled: dirname %s"%self.dirname + self.server = RoundupInstance(self.db, self.instance.actions, None) def tearDown(self): + print "tearDowm called" self.db.close() try: shutil.rmtree(self.dirname) @@ -62,10 +83,17 @@ self.assertRaises(Unauthorised, self.server.set, 'user1', 'realname=Joe Doe') def testCreate(self): + print "testCreate called: dirtree %s"%self.dirname results = self.server.create('issue', 'title=foo') issueid = 'issue' + results results = self.server.display(issueid, 'title') + + print self.db.issue.getprops() + + self.assertEqual(results['title'], 'foo') + self.assertEqual(self.db.issue.get('1', "tx_Source"), 'web') + def testFileCreate(self): results = self.server.create('file', 'content=hello\r\nthere') @@ -116,6 +144,8 @@ self.db.setCurrentUser('joe') def testAuthFilter(self): + print "testAuthFilter called: dirtree %s"%self.dirname + # this checks if we properly check for search permissions self.db.security.permissions = {} self.db.security.addRole(name='User') @@ -183,6 +213,12 @@ self.db.close() self.db = self.instance.open('chef') + self.db.tx_Source = 'web' + # These have to be set on every database open for some odd reason. + # I assume it is a test harness issue of some sort. + self.db.issue.addprop(tx_Source=hyperdb.String()) + self.db.msg.addprop(tx_Source=hyperdb.String()) + self.server = RoundupInstance(self.db, self.instance.actions, None) # Filter on keyword works for role 'Project': diff -r 3adff0fb0207 test/tx_Source_detector.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/tx_Source_detector.py Sat Apr 20 21:56:57 2013 -0400 @@ -0,0 +1,71 @@ +# +# Example output when the web interface changes item 3 and the email +# (non pgp) interface changes item 4: +# +# tx_SourceCheckAudit(3) pre db.tx_Source: cgi +# tx_SourceCheckAudit(4) pre db.tx_Source: email +# tx_SourceCheckAudit(3) post db.tx_Source: cgi +# tx_SourceCheckAudit(4) post db.tx_Source: email +# tx_SourceCheckReact(4) pre db.tx_Source: email +# tx_SourceCheckReact(4) post db.tx_Source: email +# tx_SourceCheckReact(3) pre db.tx_Source: cgi +# tx_SourceCheckReact(3) post db.tx_Source: cgi +# +# Note that the calls are interleaved, but the proper +# tx_Source is associated with the same ticket. + +import time as time + +def tx_SourceCheckAudit(db, cl, nodeid, newvalues): + ''' An auditor to print the value of the source of the + transaction that trigger this change. The sleep call + is used to delay the transaction so that multiple changes will + overlap. The expected output from this detector are 2 lines + with the same value for tx_Source. Tx source is: + None - Reported when using a script or it is an error if + the change arrives by another method. + "cli" - reported when using roundup-admin + "web" - reported when using any web based technique + "email" - reported when using an unautheticated email based technique + "email-sig-openpgp" - reported when email with a valid pgp + signature is used + ''' + if __debug__: + print "\n tx_SourceCheckAudit(%s) db.tx_Source: %s"%(nodeid, db.tx_Source) + + newvalues['tx_Source'] = db.tx_Source + + # example use for real to prevent a change from happening if it's + # submited via email + # + # if db.tx_Source == "email": + # raise Reject, 'Change not allowed via email' + +def tx_SourceCheckReact(db, cl, nodeid, oldvalues): + ''' An reactor to print the value of the source of the + transaction that trigger this change. The sleep call + is used to delay the transaction so that multiple changes will + overlap. The expected output from this detector are 2 lines + with the same value for tx_Source. Tx source is: + None - Reported when using a script or it is an error if + the change arrives by another method. + "cli" - reported when using roundup-admin + "web" - reported when using any web based technique + "email" - reported when using an unautheticated email based technique + "email-sig-openpgp" - reported when email with a valid pgp + signature is used + ''' + + if __debug__: + print " tx_SourceCheckReact(%s) db.tx_Source: %s"%(nodeid, db.tx_Source) + + + +def init(db): + db.issue.audit('create', tx_SourceCheckAudit) + db.issue.audit('set', tx_SourceCheckAudit) + + db.issue.react('set', tx_SourceCheckReact) + db.issue.react('create', tx_SourceCheckReact) + + db.msg.audit('create', tx_SourceCheckAudit) diff -r 3adff0fb0207 tools/load_tracker.py --- a/tools/load_tracker.py Fri Mar 22 15:53:27 2013 +0100 +++ b/tools/load_tracker.py Sat Apr 20 21:56:57 2013 -0400 @@ -19,6 +19,7 @@ # open the tracker tracker = instance.open(tracker_home) db = tracker.open('admin') +db.tx_Source = "cli" priorities = db.priority.list() statuses = db.status.list() diff -r 3adff0fb0207 tools/migrate-queries.py --- a/tools/migrate-queries.py Fri Mar 22 15:53:27 2013 +0100 +++ b/tools/migrate-queries.py Sat Apr 20 21:56:57 2013 -0400 @@ -26,6 +26,7 @@ continue db = instance.open('admin') + db.tx_Source = "cli" print 'Migrating active queries in %s (%s):'%( instance.config.TRACKER_NAME, home)