diff --git a/roundup/mailgw.py b/roundup/mailgw.py index 8346b05..d384ee5 100644 --- a/roundup/mailgw.py +++ b/roundup/mailgw.py @@ -825,33 +825,14 @@ class MailGW: The implementation expects an opened database and a try/finally that closes the database. ''' - # detect loops - if message.getheader('x-roundup-loop', ''): - raise IgnoreLoop - - # handle the subject line - subject = message.getheader('subject', '') - if not subject: - raise MailUsageError, _(""" -Emails to Roundup trackers must include a Subject: line! -""") - - # detect Precedence: Bulk, or Microsoft Outlook autoreplies - if (message.getheader('precedence', '') == 'bulk' - or subject.lower().find("autoreply") > 0): - raise IgnoreBulk - - if subject.strip().lower() == 'help': - raise MailUsageHelp - - # config is used many times in this method. - # make local variable for easier access - config = self.instance.config - - # determine the sender's address - from_list = message.getaddrlist('resent-from') - if not from_list: - from_list = message.getaddrlist('from') + # Filter out messages to ignore + self._handle_message_ignore(message) + + # Check for usage/help requests + self._handle_message_help(message) + + # Check if the subeject line is valid + self._handle_message_check_subject(message) # XXX Don't enable. This doesn't work yet. # "[^A-z.]tracker\+(?P[^\d\s]+)(?P\d+)\@some.dom.ain[^A-z.]" @@ -869,25 +850,131 @@ Emails to Roundup trackers must include a Subject: line! # nodeid = issue.group('nodeid') # break - # Matches subjects like: - # Re: "[issue1234] title of issue [status=resolved]" + # Parse the subject line to get the importants parts + matches = self._handle_message_parse_subject(message) + + # check for registration OTK + if self._handle_message_rego_confirm(message, matches): + return + + # get the classname + classname = self._handle_message_get_classname(message, matches) + + # get the optional nodeid + nodeid = self._handle_message_get_nodeid(message, classname, matches) + + # Determine who the author is + author = self._handle_message_get_author_id(message) + + # make sure they're allowed to edit or create this class of information + self._handle_message_check_node_permissions(author, classname, nodeid) + + # the author may have been created - make sure the change is + # committed before we reopen the database + self.db.commit() + + # set the database user as the author + username = self.db.user.get(author, 'username') + self.db.setCurrentUser(username) + + # Get the recipients list + recipients = self._handle_message_get_recipients(message) + + # get the new/updated node props + props = self._handle_message_get_props(message, classname, matches, nodeid) + + # Handle PGP signed or encrypted messages + message = self._handle_message_get_pgp_message(message) + + # get the class properties + cl = self.db.getclass(classname) + properties = cl.getprops() + + # + # handle the attachments + # + files = [] + if properties.has_key('files'): + files = self._handle_message_create_files(message, author, classname, nodeid) + props['files'] = files + + # + # create the message if there's a message body (content) + # + if properties.has_key('messages'): + props['messages'] = self._handle_message_create_msg(message, author, classname, nodeid, files, recipients) + + # + # perform the node change / create + # + nodeid = self._handle_message_create_node(author, classname, props, nodeid) + + # commit the changes to the DB + self.db.commit() + + return nodeid + + def _handle_message_ignore(self, message): + ''' message - a Message instance + + Check to see if message can be safely ignored + ''' + # detect loops + if message.getheader('x-roundup-loop', ''): + raise IgnoreLoop + + # detect Precedence: Bulk, or Microsoft Outlook autoreplies + subject = message.getheader('subject', '') + if (message.getheader('precedence', '') == 'bulk' + or subject.lower().find("autoreply") > 0): + raise IgnoreBulk + + def _handle_message_check_subject(self, message): + ''' message - a Message instance + + Check to see if the message contains a valid subject line + ''' + subject = message.getheader('subject', '') + + if not subject: + raise MailUsageError, _(""" +Emails to Roundup trackers must include a Subject: line! +""") + + def _handle_message_help(self, message): + ''' message - a Message instance + + Check to see if the message contains a usage/help request + ''' + subject = message.getheader('subject', '') + + if subject.strip().lower() == 'help': + raise MailUsageHelp - # Alias since we need a reference to the original subject for - # later use in error messages - tmpsubject = subject + def _handle_message_parse_subject(self, message): + ''' message - a Message instance - sd_open, sd_close = config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] + Matches subjects like: + Re: "[issue1234] title of issue [status=resolved]" + + Each part of the subject is matched, stored, then removed from the + start of the subject string as needed. The stored values are then + returned + ''' + matches = dict.fromkeys(['refwd', 'quote', 'classname', + 'nodeid', 'title', 'args', + 'argswhole', 'has_prefix']) + + tmpsubject = message.getheader('subject', '') + + sd_open, sd_close = self.instance.config['MAILGW_SUBJECT_SUFFIX_DELIMITERS'] delim_open = re.escape(sd_open) if delim_open in '[(': delim_open = '\\' + delim_open delim_close = re.escape(sd_close) if delim_close in '[(': delim_close = '\\' + delim_close - matches = dict.fromkeys(['refwd', 'quote', 'classname', - 'nodeid', 'title', 'args', - 'argswhole']) - # Look for Re: et. al. Used later on for MAILGW_SUBJECT_CONTENT_MATCH - re_re = r"(?P%s)\s*" % config["MAILGW_REFWD_RE"].pattern + re_re = r"(?P%s)\s*" % self.instance.config["MAILGW_REFWD_RE"].pattern m = re.match(re_re, tmpsubject, re.IGNORECASE|re.VERBOSE|re.UNICODE) if m: m = m.groupdict() @@ -902,9 +989,11 @@ Emails to Roundup trackers must include a Subject: line! matches.update(m.groupdict()) tmpsubject = tmpsubject[len(matches['quote']):] # Consume quote - has_prefix = re.search(r'^%s(\w+)%s'%(delim_open, + # Check if the subject includes a prefix + matches['has_prefix'] = re.search(r'^%s(\w+)%s'%(delim_open, delim_close), tmpsubject.strip()) + # Match the classname if specified class_re = r'%s(?P(%s))(?P\d+)?%s'%(delim_open, "|".join(self.db.getclasses()), delim_close) # Note: re.search, not re.match as there might be garbage @@ -917,6 +1006,7 @@ Emails to Roundup trackers must include a Subject: line! tmpsubject = tmpsubject[m.end():] + # Match the title of the subject # if we've not found a valid classname prefix then force the # scanning to handle there being a leading delimiter title_re = r'(?P%s[^%s]*)'%( @@ -926,28 +1016,53 @@ Emails to Roundup trackers must include a Subject: line! matches.update(m.groupdict()) tmpsubject = tmpsubject[len(matches['title']):] # Consume title + if matches['title']: + matches['title'] = matches['title'].strip() + else: + matches['title'] = '' + + # strip off the quotes that dumb emailers put around the subject, like + # Re: "[issue1] bla blah" + if matches['quote'] and matches['title'].endswith('"'): + matches['title'] = matches['title'][:-1] + + # Match any arguments specified args_re = r'(?P<argswhole>%s(?P<args>.+?)%s)?'%(delim_open, delim_close) m = re.search(args_re, tmpsubject.strip(), re.IGNORECASE|re.VERBOSE) if m: matches.update(m.groupdict()) + + return matches - # figure subject line parsing modes - pfxmode = config['MAILGW_SUBJECT_PREFIX_PARSING'] - sfxmode = config['MAILGW_SUBJECT_SUFFIX_PARSING'] - - # check for registration OTK - # or fallback on the default class - if self.db.config['EMAIL_REGISTRATION_CONFIRMATION']: + def _handle_message_rego_confirm(self, message, matches): + ''' message - a Message instance + + Check for registration OTK and confirm the registration if found + ''' + from_list = message.getaddrlist('resent-from') or message.getaddrlist('from') + + if self.instance.config['EMAIL_REGISTRATION_CONFIRMATION']: otk_re = re.compile('-- key (?P<otk>[a-zA-Z0-9]{32})') otk = otk_re.search(matches['title'] or '') if otk: self.db.confirm_registration(otk.group('otk')) subject = 'Your registration to %s is complete' % \ - config['TRACKER_NAME'] + self.instance.config['TRACKER_NAME'] sendto = [from_list[0][1]] self.mailer.standard_message(sendto, subject, '') - return + return 1 + + return 0 + + def _handle_message_get_classname(self, message, matches): + ''' message - a Message instance + matches - the matched 'parts' of the message subject line + + Determine the classname of the node being created/edited + ''' + pfxmode = self.instance.config['MAILGW_SUBJECT_PREFIX_PARSING'] + subject = message.getheader('subject', '') # get the classname if pfxmode == 'none': @@ -955,7 +1070,7 @@ Emails to Roundup trackers must include a Subject: line! else: classname = matches['classname'] - if not classname and has_prefix and pfxmode == 'strict': + if not classname and matches['has_prefix'] and pfxmode == 'strict': raise MailUsageError, _(""" The message you sent to roundup did not contain a properly formed subject line. The subject must contain a class name or designator to indicate the @@ -979,7 +1094,7 @@ Subject was: '%(subject)s' if self.default_class: attempts.append(self.default_class) else: - attempts.append(config['MAILGW_DEFAULT_CLASS']) + attempts.append(self.instance.config['MAILGW_DEFAULT_CLASS']) # first valid class name wins cl = None @@ -1016,7 +1131,20 @@ designator to indicate the 'topic' of the message. For example: Subject was: '%(subject)s' """) % locals() - # get the optional nodeid + return classname + + def _handle_message_get_nodeid(self, message, classname, matches): + ''' message - a Message instance + classname - node type + matches - the matched 'parts' of the message subject line + + Determine the nodeid from the message and return it if found + ''' + pfxmode = self.instance.config['MAILGW_SUBJECT_PREFIX_PARSING'] + cl = self.db.getclass(classname) + title = matches['title'] + subject = message.getheader('subject', '') + if pfxmode == 'none': nodeid = None else: @@ -1029,17 +1157,6 @@ Subject was: '%(subject)s' if l: nodeid = cl.filter(None, {'messages':l})[0] - # title is optional too - title = matches['title'] - if title: - title = title.strip() - else: - title = '' - - # strip off the quotes that dumb emailers put around the subject, like - # Re: "[issue1] bla blah" - if matches['quote'] and title.endswith('"'): - title = title[:-1] # but we do need either a title or a nodeid... if nodeid is None and not title: @@ -1058,7 +1175,7 @@ Subject was: "%(subject)s" # recent...). The subject_content_match config may specify an # additional restriction based on the matched node's creation or # activity. - tmatch_mode = config['MAILGW_SUBJECT_CONTENT_MATCH'] + tmatch_mode = self.instance.config['MAILGW_SUBJECT_CONTENT_MATCH'] if tmatch_mode != 'never' and nodeid is None and matches['refwd']: l = cl.stringFind(title=title) limit = None @@ -1083,59 +1200,16 @@ The node specified by the designator in the subject of your message Subject was: "%(subject)s" """) % locals() else: - title = subject nodeid = None - # Handle the arguments specified by the email gateway command line. - # We do this by looping over the list of self.arguments looking for - # a -C to tell us what class then the -S setting string. - msg_props = {} - user_props = {} - file_props = {} - issue_props = {} - # so, if we have any arguments, use them - if self.arguments: - current_class = 'msg' - for option, propstring in self.arguments: - if option in ( '-C', '--class'): - current_class = propstring.strip() - # XXX this is not flexible enough. - # we should chect for subclasses of these classes, - # not for the class name... - if current_class not in ('msg', 'file', 'user', 'issue'): - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect class specified as: - %(current_class)s -""") % locals() - if option in ('-S', '--set'): - if current_class == 'issue' : - errors, issue_props = setPropArrayFromString(self, - cl, propstring.strip(), nodeid) - elif current_class == 'file' : - temp_cl = self.db.getclass('file') - errors, file_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'msg' : - temp_cl = self.db.getclass('msg') - errors, msg_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - elif current_class == 'user' : - temp_cl = self.db.getclass('user') - errors, user_props = setPropArrayFromString(self, - temp_cl, propstring.strip()) - if errors: - mailadmin = config['ADMIN_EMAIL'] - raise MailUsageError, _(""" -The mail gateway is not properly set up. Please contact -%(mailadmin)s and have them fix the incorrect properties: - %(errors)s -""") % locals() + return nodeid - # - # handle the users - # + def _handle_message_get_author_id(self, message): + ''' message - a Message instance + + Attempt to get the author id from the existing registered users, + otherwise attempt to register a new user and return their id + ''' # Don't create users if anonymous isn't allowed to register create = 1 anonid = self.db.user.lookup('anonymous') @@ -1145,6 +1219,7 @@ The mail gateway is not properly set up. Please contact # ok, now figure out who the author is - create a new user if the # "create" flag is true + from_list = message.getaddrlist('resent-from') or message.getaddrlist('from') author = uidFromAddress(self.db, from_list[0], create=create) # if we're not recognised, and we don't get added as a user, then we @@ -1177,7 +1252,19 @@ Unknown address: %(from_address)s raise Unauthorized, _( 'You are not permitted to access this tracker.') - # make sure they're allowed to edit or create this class of information + # the author may have been created - make sure the change is + # committed before we reopen the database + self.db.commit() + + return author + + def _handle_message_check_node_permissions(self, author, classname, nodeid): + ''' author - id of the user creating files + classname - node type + nodeid - id of an existing 'classname', or None for a new node + + Check if the author has permissio to edit or create this class of node + ''' if nodeid: if not self.db.security.hasPermission('Edit', author, classname, itemid=nodeid): @@ -1189,20 +1276,25 @@ Unknown address: %(from_address)s 'You are not permitted to create %(classname)s.' ) % locals() - # the author may have been created - make sure the change is - # committed before we reopen the database - self.db.commit() + def _handle_message_get_recipients(self, message): + ''' message - a Message instance - # set the database user as the author - username = self.db.user.get(author, 'username') - self.db.setCurrentUser(username) + Get the list of recipients who were included in message and + register them as users if possible + ''' + # Don't create users if anonymous isn't allowed to register + create = 1 + anonid = self.db.user.lookup('anonymous') + if not (self.db.security.hasPermission('Register', anonid, 'user') + and self.db.security.hasPermission('Email Access', anonid)): + create = 0 - # re-get the class with the new database connection - cl = self.db.getclass(classname) + # get the user class arguments from the commandline + user_props = self._get_class_arguments('user') # now update the recipients list recipients = [] - tracker_email = config['TRACKER_EMAIL'].lower() + tracker_email = self.instance.config['TRACKER_EMAIL'].lower() for recipient in message.getaddrlist('to') + message.getaddrlist('cc'): r = recipient[1].strip().lower() if r == tracker_email or not r: @@ -1216,6 +1308,28 @@ Unknown address: %(from_address)s if recipient: recipients.append(recipient) + return recipients + + def _handle_message_get_props(self, message, classname, matches, nodeid): + ''' message - a Message instance + classname - node type that the message will be associated with + matches - the matched 'parts' of the message subject line + nodeid - id of an existing 'classname', or None for a new node + + Generate all the props for the new/updated node and return them + ''' + # get the subject + subject = message.getheader('subject', '') + + # get the class + cl = self.db.getclass(classname) + + # get the commandline arguments for issues + issue_props = self._get_class_arguments('issue') + + # subject line parsing modes + sfxmode = self.instance.config['MAILGW_SUBJECT_SUFFIX_PARSING'] + # # handle the subject argument list # @@ -1224,6 +1338,12 @@ Unknown address: %(from_address)s props = {} args = matches['args'] argswhole = matches['argswhole'] + title = matches['title'] + + # Reform the title + if matches['nodeid'] and nodeid is None: + title = subject + if args: if sfxmode == 'none': title += ' ' + argswhole @@ -1249,30 +1369,24 @@ Subject was: "%(subject)s" issue_props.has_key('title')): issue_props['title'] = title if (nodeid and properties.has_key('title') and not - config['MAILGW_SUBJECT_UPDATES_TITLE']): + self.instance.config['MAILGW_SUBJECT_UPDATES_TITLE']): issue_props['title'] = cl.get(nodeid,'title') - # - # handle message-id and in-reply-to - # - messageid = message.getheader('message-id') - # generate a messageid if there isn't one - if not messageid: - messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), - classname, nodeid, config['MAIL_DOMAIN']) + # merge the command line props defined in issue_props into + # the props dictionary because function(**props, **issue_props) + # is a syntax error. + for prop in issue_props.keys() : + if not props.has_key(prop) : + props[prop] = issue_props[prop] - # if they've enabled PGP processing then verify the signature - # or decrypt the message + return props - # if PGP_ROLES is specified the user must have a Role in the list - # or we will skip PGP processing - def pgp_role(): - if self.instance.config.PGP_ROLES: - return self.db.user.has_role(author, - iter_roles(self.instance.config.PGP_ROLES)) - else: - return True + def _handle_message_get_pgp_message(self, message): + ''' message - a Message instance + If they've enabled PGP processing then verify the signature + or decrypt the message + ''' if self.instance.config.PGP_ENABLE and pgp_role(): assert pyme, 'pyme is not installed' # signed/encrypted mail must come from the primary address @@ -1287,30 +1401,41 @@ Subject was: "%(subject)s" # TODO: encrypted message handling is far from perfect # bounces probably include the decrypted message, for # instance :( - message = message.decrypt(author_address) + return message.decrypt(author_address) else: raise MailUsageError, _(""" This tracker has been configured to require all email be PGP signed or encrypted.""") - # now handle the body - find the message - ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES - content, attachments = message.extract_content(ignore_alternatives=ig, - unpack_rfc822=self.instance.config.MAILGW_UNPACK_RFC822) - if content is None: - raise MailUsageError, _(""" -Roundup requires the submission to be plain text. The message parser could -not find a text/plain part to use. -""") - # parse the body of the message, stripping out bits as appropriate - summary, content = parseContent(content, config=config) - content = content.strip() + return message - # - # handle the attachments - # + # if PGP_ROLES is specified the user must have a Role in the list + # or we will skip PGP processing + def pgp_role(): + if self.instance.config.PGP_ROLES: + return self.db.user.has_role(author, + iter_roles(self.instance.config.PGP_ROLES)) + else: + return True + + def _handle_message_create_files(self, message, author, classname, nodeid): + ''' message - a Message instance + author - id of the user creating files + classname - node type that the attachments will be associated with + nodeid - id of an existing 'classname', or None for a new node + + Create a file for each attachment in ther message + ''' files = [] - if attachments and properties.has_key('files'): + file_props = self._get_class_arguments('file') + cl = self.db.getclass(classname) + + # get the attachments + ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES + attachments = message.extract_content(ignore_alternatives=ig, + unpack_rfc822=self.instance.config.MAILGW_UNPACK_RFC822)[1] + + if attachments: for (name, mime_type, data) in attachments: if not self.db.security.hasPermission('Create', author, 'file'): raise Unauthorized, _( @@ -1335,15 +1460,46 @@ not find a text/plain part to use. # extend the existing files list fileprop = cl.get(nodeid, 'files') fileprop.extend(files) - props['files'] = fileprop - else: - # pre-load the files list - props['files'] = files + return fileprop - # - # create the message if there's a message body (content) - # - if (content and properties.has_key('messages')): + return files + + def _handle_message_create_msg(self, message, author, classname, nodeid, files, recipients): + ''' message - a Message instance + author - id of the user creating files + classname - node type that the message will be associated with + nodeid - id of an existing 'classname', or None for a new node + files - list of file ids associated with the msg + recipients - list of recipient ids associated with the msg + + Create a msg contain all the relevant information from the message + ''' + msg_props = self._get_class_arguments('msg') + cl = self.db.getclass(classname) + + # Get the message ids + inreplyto = message.getheader('in-reply-to') or '' + messageid = message.getheader('message-id') + # generate a messageid if there isn't one + if not messageid: + messageid = "<%s.%s.%s%s@%s>"%(time.time(), random.random(), + classname, nodeid, config['MAIL_DOMAIN']) + + # now handle the body - find the message + ig = self.instance.config.MAILGW_IGNORE_ALTERNATIVES + content = message.extract_content(ignore_alternatives=ig, + unpack_rfc822=self.instance.config.MAILGW_UNPACK_RFC822)[0] + if content is None: + raise MailUsageError, _(""" +Roundup requires the submission to be plain text. The message parser could +not find a text/plain part to use. +""") + + # parse the body of the message, stripping out bits as appropriate + summary, content = parseContent(content, config=self.instance.config) + content = content.strip() + + if content: if not self.db.security.hasPermission('Create', author, 'msg'): raise Unauthorized, _( 'You are not permitted to create messages.') @@ -1369,22 +1525,25 @@ Mail message was rejected by a detector. # add the message to the node's list messages = cl.get(nodeid, 'messages') messages.append(message_id) - props['messages'] = messages + return messages else: # pre-load the messages list - props['messages'] = [message_id] - - # - # perform the node change / create - # + return [message_id] + + # no message content, so return an empty list + return [] + + def _handle_message_create_node(self, author, classname, props, nodeid): + ''' author - id of the user creating files + classname - node type that the message will be associated with + props - properties for the new/updated node + nodeid - id of an existing 'classname', or None for a new node + + Create/update a node using the props + ''' + cl = self.db.getclass(classname) + try: - # merge the command line props defined in issue_props into - # the props dictionary because function(**props, **issue_props) - # is a syntax error. - for prop in issue_props.keys() : - if not props.has_key(prop) : - props[prop] = issue_props[prop] - if nodeid: # Check permissions for each property for prop in props.keys(): @@ -1407,12 +1566,60 @@ There was a problem with the message you sent: %(message)s """) % locals() - # commit the changes to the DB - self.db.commit() - return nodeid + def _get_class_arguments(self, class_type): + ''' class_type - a valid node class type + + Parse the commandline arguments and retrieve the properties that + are relevant to the class_type + ''' + props = {} + + # check if the class_type is valid + try: + self.db.getclass(class_type) + except KeyError: + raise MailUsageError, _(""" +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect class specified as: + %(current_class)s +""") % locals() + + if self.arguments: + # The default type on the commandline is msg + if class_type == 'msg': + current_class = class_type + else: + current_class = None + + # Handle the arguments specified by the email gateway command line. + # We do this by looping over the list of self.arguments looking for + # a -C to match the class we want, then use the -S setting string. + for option, propstring in self.arguments: + if option in ( '-C', '--class'): + current_class = propstring.strip() + + if current_class != class_type: + current_class = None + + elif current_class and option in ('-S', '--set'): + temp_cl = self.db.getclass(current_class) + errors, props = setPropArrayFromString(self, + temp_cl, propstring.strip()) + + if errors: + mailadmin = self.instance.config['ADMIN_EMAIL'] + raise MailUsageError, _(""" +The mail gateway is not properly set up. Please contact +%(mailadmin)s and have them fix the incorrect properties: + %(errors)s +""") % locals() + + return props + + def setPropArrayFromString(self, cl, propString, nodeid=None): ''' takes string of form prop=value,value;prop2=value and returns (error, prop[..])