Index: roundup/hyperdb.py =================================================================== --- roundup/hyperdb.py (revision 4205) +++ roundup/hyperdb.py (working copy) @@ -534,7 +534,6 @@ curdir = sa.sort_direction idx += 1 sortattr.append (val) - #print >> sys.stderr, "\nsortattr", sortattr sortattr = zip (*sortattr) for dir, i in reversed(zip(directions, dir_idx)): rev = dir == '-' @@ -568,6 +567,7 @@ """Error to be raised when there is some problem in the database code """ pass + class Database: """A database for storing records containing flexible data types. @@ -683,11 +683,18 @@ def getnode(self, classname, nodeid): """Get a node from the database. - - 'cache' exists for backwards compatibility, and is not used. """ raise NotImplementedError + def getnodes(self, classname, nodeids): + """Get a set of nodes from the database. + + The default implementation simply calls getnode() for each + id in nodeids. Individual Database backends may reimplement + this if they have more compact ways to fetch multiple nodes. + """ + return [self.getnode(classname, n) for n in nodeids] + def hasnode(self, classname, nodeid): """Determine if the database has a given node. """ @@ -841,8 +848,6 @@ 'nodeid' must be the id of an existing node of this class or an IndexError is raised. - - 'cache' exists for backwards compatibility, and is not used. """ return Node(self, nodeid) @@ -1038,8 +1043,7 @@ """ raise NotImplementedError - def _filter(self, search_matches, filterspec, sort=(None,None), - group=(None,None)): + def _filter(self, search_matches, filterspec, proptree): """For some backends this implements the non-transitive search, for more information see the filter method. """ @@ -1117,7 +1121,7 @@ sortattr.append(('+', 'id')) return sortattr - def filter(self, search_matches, filterspec, sort=[], group=[]): + def filter(self, search_matches, filterspec, sort=[], group=[], properties=[]): """Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec. @@ -1144,6 +1148,11 @@ the last week with a filterspec of {'messages.author' : '42', 'messages.creation' : '.-1w;'} + properties is a list with the properties to be returned. + + If properties is empty, only nodeids are returned. Otherwise, + the returned items are (nodeid, [prop-values]) tuples. + Implementation note: This implements a non-optimized version of Transitive search using _filter implemented in a backend class. A more efficient @@ -1154,8 +1163,17 @@ sortattr = self._sortattr(sort = sort, group = group) proptree = self._proptree(filterspec, sortattr) proptree.search(search_matches) - return proptree.sort() + ids = proptree.sort() + if properties: + def get_props(node): return [node.get(p) for p in properties] + nodes = self.db.getnodes(self.classname, ids) + result = zip(ids, [get_props(node) for node in nodes]) + else: + result = ids + + return result + def count(self): """Get the number of nodes in this class. @@ -1268,7 +1286,6 @@ values. """ properties = klass.getprops() - # ensure it's a valid property name propname = propname.strip() try: @@ -1340,6 +1357,26 @@ self.db.indexer.add_text((self.classname, nodeid, 'content'), self.get(nodeid, 'content'), mime_type) +class ExternalFileClass: + """ A class that requires the "filename" property to refer to external + storage. + """ + default_mime_type = 'text/plain' + + def __init__(self, db, classname, **properties): + """The newly-created class automatically includes the "content" + property. + """ + if not properties.has_key('filename'): + properties['filename'] = String(indexme='yes') + + def export_propnames(self): + """List the property names for export from this Class""" + propnames = self.getprops().keys() + propnames.sort() + return propnames + + class Node: """ A convenience wrapper for the given node """ Index: roundup/backends/rdbms_common.py =================================================================== --- roundup/backends/rdbms_common.py (revision 4205) +++ roundup/backends/rdbms_common.py (working copy) @@ -1063,6 +1063,87 @@ return node + def getnodes(self, classname, nodeids = None): + """ Get a set of nodes from the database. + """ + + if not nodeids: + nodeids = self.classes[classname].list() + nodes = [self.cache.get((classname, n)) for n in nodeids] + + # Find out which nodes to fetch from the DB + fetch_ids = [id for id in nodeids if (classname,id) not in self.cache] + + # Get them + new_nodes = {} + if fetch_ids: + start_t = time.time() + + # figure the columns we're fetching + cl = self.classes[classname] + cols, mls = self.determine_columns(cl.properties.items()) + scols = ','.join([col for col,dt in cols]) + + # perform the basic property fetch + sql = 'select %s from _%s where id in (%s)'%(scols, classname, ','.join(fetch_ids)) + self.sql(sql) + + props = cl.getprops(protected=1) + for id in fetch_ids: + values = self.sql_fetchone() + if values is None: + raise IndexError, 'no such %s node %s'%(classname, id) + new_nodes[id] = {} + for col in range(len(cols)): + name = cols[col][0][1:] + if name.endswith('_int__'): + # XXX eugh, this test suxxors + # ignore the special Interval-as-seconds column + continue + value = values[col] + if value is not None: + value = self.to_hyperdb_value(props[name].__class__)(value) + new_nodes[id][name] = value + + # now the multilinks + for col in mls: + # get the link ids + sql = 'select linkid from %s_%s where nodeid in (%s)'%(classname, col, ','.join(fetch_ids)) + self.sql(sql) + # extract the first column from the result + # XXX numeric ids + for id in fetch_ids: + items = [int(x[0]) for x in self.cursor.fetchone()] + items.sort() + new_nodes[id][col] = [str(x) for x in items] + + # Build list of nodes to return, and update the cache + for id in nodeids: + if id in new_nodes: + node = new_nodes[id] + nodes.append(node) + self.cache[key] = node + # update the LRU + self.cache_lru.insert(0, key) + if len(self.cache_lru) > self.cache_size: + del self.cache[self.cache_lru.pop()] + if __debug__: + self.stats['cache_misses'] += 1 + else: + key = (classname, id) + nodes.append(self.cache[key]) + self.cache_lru.remove(key) + self.cache_lru.insert(0, key) + if __debug__: + self.stats['cache_hits'] += 1 + + if __debug__: + self.stats['get_items'] += (time.time() - start_t) + + # Return total list + return nodes + + def destroynode(self, classname, nodeid): """Remove a node from the database. Called exclusively by the destroy() method on Class. @@ -2132,7 +2213,8 @@ # The format parameter is replaced with the attribute. order_by_null_values = None - def filter(self, search_matches, filterspec, sort=[], group=[]): + def filter(self, search_matches, filterspec, sort=[], group=[], + properties=[]): """Return a list of the ids of the active nodes in this class that match the 'filter' spec, sorted by the group spec and then the sort spec @@ -2151,6 +2233,11 @@ 1. String properties must match all elements in the list, and 2. Other properties must match any of the elements in the list. + + properties is a list with the properties to be returned. + + If properties is empty, only nodeids are returned. Otherwise, + the returned items are (nodeid, [prop-values]) tuples. """ # we can't match anything if search_matches is empty if not search_matches and search_matches is not None: @@ -2413,10 +2500,18 @@ l = [str(row[0]) for row in l] l = proptree.sort (l) + if properties: + def get_props(node): return [node.get(p) for p in properties] + nodes = self.db.getnodes(self.classname, l) + result = zip(l, [get_props(node) for node in nodes]) + else: + result = l + if __debug__: self.db.stats['filtering'] += (time.time() - start_t) - return l + return result + def filter_sql(self, sql): """Return a list of the ids of the items in this class that match the SQL provided. The SQL is a complete "select" statement.