From b4ae2803662c98c1397b1978686b0277a901cfd6 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 13 Feb 2009 11:38:10 +0100 Subject: [PATCH] - Enable hyperdb fetch-api(). --- roundup/backends/rdbms_common.py | 53 ++++++++++++++++++++++++++++++++++++++ roundup/hyperdb.py | 13 +++++++++ 2 files changed, 66 insertions(+), 0 deletions(-) diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 5168367..54ad93c 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1548,6 +1548,59 @@ class Class(hyperdb.Class): return d[propname] + def fetch(self, nodes, props = None): + clsprops = self.getprops(protected = 1) + if not props: props = clsprops.keys() + + # Split properties into simple columns and multilink properties + # ('id' is specially handled since it doesn't start with underscore in + # the RDBMS) + cols, mls = self.db.determine_columns([ + (prop, clsprops[prop]) for prop in props if prop != 'id']) + cols.insert(0,('id',None)) + scols = ','.join([col for col,dt in cols]) + + # We want the order of the db, so keep an ordered list, but we need to + # maintain complete the items with MLS items later, so keep a dict() + # nodetable as well + retVal = list() + nodetable = dict() + + # perform the basic property fetch + rowspec = '(%s)' % ','.join(nodes) + sql = 'select %s from _%s where id IN %s' % (scols, self.classname, rowspec) + self.db.sql(sql) + values = self.db.sql_fetchall() + + for row in values: + node = dict() + for col in xrange(len(cols)): + name = cols[col][0].lstrip('_') + if name.endswith('_int__'): + # XXX eugh, this test suxxors + # ignore the special Interval-as-seconds column + continue + value = row[col] + if value is not None: + value = self.db.sql_to_hyperdb_value[clsprops[name].__class__](value) + node[name] = value + retVal.append(node) + nodetable[node['id']] = node + + # now the multilinks + for col in mls: + # get the link ids + sql = '''select nodeid, linkid from %s_%s where nodeid IN %s + order by linkid''' % (self.classname, col, rowspec)#self.db.arg) + self.db.sql(sql) + # extract the first column from the result + # XXX numeric ids + for nodeid, linkid in self.db.sql_fetchall(): + node = nodetable[str(nodeid)] + node.setdefault(col, []).append(str(linkid)) + + return retVal + def set(self, nodeid, **propvalues): """Modify a property on an existing node of this class. diff --git a/roundup/hyperdb.py b/roundup/hyperdb.py index 637543c..df43c72 100644 --- a/roundup/hyperdb.py +++ b/roundup/hyperdb.py @@ -835,6 +835,19 @@ class Class: """ raise NotImplementedError + def fetch(self, nodes, props = None): + """Fast fetching of a batch of nodes. + "props" may be a list of properties to fetch. The implementing backend is free to ignore this parameter, and return all properties, so take this into account. + """ + if not columns: columns = self.getprops().keys() + nodes = [] + for id in nodes: + node = {} + for col in columns: + node[col] = self.get(id, col) + nodes.append(node) + return nodes + # not in spec def getnode(self, nodeid): ''' Return a convenience wrapper for the node. -- 1.5.5.6 From 68960a1b14c0cb35ab5d796231b66f1f4f1349e6 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 13 Feb 2009 11:39:01 +0100 Subject: [PATCH] - Change admin-list to use the new HyperDB.fetch()-call. Improves performance, and serves as a useful benchmark. --- roundup/admin.py | 26 ++++++++++++-------------- 1 files changed, 12 insertions(+), 14 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index 51695e2..6a4d916 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -854,26 +854,24 @@ Erase it? Y/N: """)) if self.separator: if len(args) == 2: # create a list of propnames since user specified propname - proplist=[] - for nodeid in cl.list(): - try: - proplist.append(cl.get(nodeid, propname)) - except KeyError: - raise UsageError, _('%(classname)s has no property ' - '"%(propname)s"')%locals() + proplist = [] + try: + proplist = [x[propname] for x in cl.fetch(nodes, props=(propname,))] + except KeyError: + raise UsageError, _('%(classname)s has no property ' + '"%(propname)s"')%locals() print self.separator.join(proplist) else: # create a list of index id's since user didn't specify # otherwise print self.separator.join(cl.list()) else: - for nodeid in cl.list(): - try: - value = cl.get(nodeid, propname) - except KeyError: - raise UsageError, _('%(classname)s has no property ' - '"%(propname)s"')%locals() - print _('%(nodeid)4s: %(value)s')%locals() + try: + for node in cl.fetch(cl.list(), props=('id',propname)): + print _('%4s: %s')%(node['id'], node[propname]) + except KeyError: + raise UsageError, _('%(classname)s has no property ' + '"%(propname)s"')%locals() return 0 def do_table(self, args): -- 1.5.5.6 From d185c3004738801b658e9c05c7caae3f251906d6 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 13 Feb 2009 22:00:02 +0100 Subject: [PATCH] - Change menu-generation to use fetch-API. --- roundup/cgi/templating.py | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) diff --git a/roundup/cgi/templating.py b/roundup/cgi/templating.py index 6c7a1f8..9037b8a 100644 --- a/roundup/cgi/templating.py +++ b/roundup/cgi/templating.py @@ -1902,30 +1902,28 @@ class LinkHTMLProperty(HTMLProperty): if value and value not in options: options.insert(0, value) - for optionid in options: - # get the option value, and if it's None use an empty string - option = linkcl.get(optionid, k) or '' + for option in linkcl.fetch(options, [k] + (additional or [])): + optionid = option['id'] + optionlabel = option[k] # figure if this option is selected s = '' - if value in [optionid, option]: + if value in (optionid, optionlabel): s = 'selected="selected" ' # figure the label if showid: - lab = '%s%s: %s'%(self._prop.classname, optionid, option) + lab = '%s%s: %s'%(self._prop.classname, optionid, optionlabel) elif not option: lab = '%s%s'%(self._prop.classname, optionid) else: - lab = option + lab = optionlabel # truncate if it's too long if size is not None and len(lab) > size: lab = lab[:size-3] + '...' if additional: - m = [] - for propname in additional: - m.append(linkcl.get(optionid, propname)) + m = [option[propname] for propname in additional] lab = lab + ' (%s)'%', '.join(map(str, m)) # and generate -- 1.5.5.6 From cfc7f4c86d9e61136aea9aad4a732c81502a7663 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Sun, 15 Feb 2009 14:55:19 +0100 Subject: [PATCH] - Add fetch():ed rows to cache. --- roundup/backends/rdbms_common.py | 14 +++++++++++++- 1 files changed, 13 insertions(+), 1 deletions(-) diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 54ad93c..5eea16c 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -1591,7 +1591,7 @@ class Class(hyperdb.Class): for col in mls: # get the link ids sql = '''select nodeid, linkid from %s_%s where nodeid IN %s - order by linkid''' % (self.classname, col, rowspec)#self.db.arg) + order by linkid''' % (self.classname, col, rowspec) self.db.sql(sql) # extract the first column from the result # XXX numeric ids @@ -1599,6 +1599,18 @@ class Class(hyperdb.Class): node = nodetable[str(nodeid)] node.setdefault(col, []).append(str(linkid)) + # save off in the cache + # (only if all props were fetched, the cache doesn't handle partial rows) + if props == clsprops.keys(): + db = self.db + for node in retVal: + key = (self.classname, node['id']) + db.cache[key] = node + # update the LRU + db.cache_lru.insert(0, key) + if len(db.cache_lru) > ROW_CACHE_SIZE: + del db.cache[db.cache_lru.pop()] + return retVal def set(self, nodeid, **propvalues): -- 1.5.5.6 From 50565b04c694c21cc51cb065d7b431c8422a4324 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 20 Feb 2009 16:39:46 +0100 Subject: [PATCH] - Improve db-performance further by converting attr-lookup to locals. --- roundup/backends/rdbms_common.py | 32 +++++++++++++++++++++----------- 1 files changed, 21 insertions(+), 11 deletions(-) diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 5eea16c..36084d8 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -966,14 +966,16 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): """ # see if we have this node cached key = (classname, nodeid) - if self.cache.has_key(key): + cache = self.cache + cache_lru = self.cache_lru + if cache.has_key(key): # push us back to the top of the LRU - self.cache_lru.remove(key) - self.cache_lru.insert(0, key) + cache_lru.remove(key) + cache_lru.insert(0, key) if __debug__: self.stats['cache_hits'] += 1 # return the cached information - return self.cache[key] + return cache[key] if __debug__: self.stats['cache_misses'] += 1 @@ -1549,13 +1551,17 @@ class Class(hyperdb.Class): return d[propname] def fetch(self, nodes, props = None): + if __debug__: + start_t = time.time() + db = self.db + clsprops = self.getprops(protected = 1) if not props: props = clsprops.keys() # Split properties into simple columns and multilink properties # ('id' is specially handled since it doesn't start with underscore in # the RDBMS) - cols, mls = self.db.determine_columns([ + cols, mls = db.determine_columns([ (prop, clsprops[prop]) for prop in props if prop != 'id']) cols.insert(0,('id',None)) scols = ','.join([col for col,dt in cols]) @@ -1569,8 +1575,8 @@ class Class(hyperdb.Class): # perform the basic property fetch rowspec = '(%s)' % ','.join(nodes) sql = 'select %s from _%s where id IN %s' % (scols, self.classname, rowspec) - self.db.sql(sql) - values = self.db.sql_fetchall() + db.sql(sql) + values = db.sql_fetchall() for row in values: node = dict() @@ -1582,7 +1588,7 @@ class Class(hyperdb.Class): continue value = row[col] if value is not None: - value = self.db.sql_to_hyperdb_value[clsprops[name].__class__](value) + value = db.sql_to_hyperdb_value[clsprops[name].__class__](value) node[name] = value retVal.append(node) nodetable[node['id']] = node @@ -1592,25 +1598,29 @@ class Class(hyperdb.Class): # get the link ids sql = '''select nodeid, linkid from %s_%s where nodeid IN %s order by linkid''' % (self.classname, col, rowspec) - self.db.sql(sql) + db.sql(sql) # extract the first column from the result # XXX numeric ids - for nodeid, linkid in self.db.sql_fetchall(): + for nodeid, linkid in db.sql_fetchall(): node = nodetable[str(nodeid)] node.setdefault(col, []).append(str(linkid)) # save off in the cache # (only if all props were fetched, the cache doesn't handle partial rows) if props == clsprops.keys(): - db = self.db for node in retVal: key = (self.classname, node['id']) db.cache[key] = node # update the LRU + if key in db.cache_lru: + db.cache_lru.remove(key) db.cache_lru.insert(0, key) if len(db.cache_lru) > ROW_CACHE_SIZE: del db.cache[db.cache_lru.pop()] + if __debug__: + db.stats['get_items'] += (time.time() - start_t) + return retVal def set(self, nodeid, **propvalues): -- 1.5.5.6 From b4adb2db1d0cdc653ba686c5ba14fe28051d3f74 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 20 Feb 2009 16:44:28 +0100 Subject: [PATCH] - Preseed the row-cache when batch()ing. --- roundup/cgi/templating.py | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diff --git a/roundup/cgi/templating.py b/roundup/cgi/templating.py index 9037b8a..6017704 100644 --- a/roundup/cgi/templating.py +++ b/roundup/cgi/templating.py @@ -2541,8 +2541,13 @@ function help_window(helpurl, width, height) { # filter for visibility check = self._client.db.security.hasPermission userid = self._client.userid - l = [id for id in klass.filter(matches, filterspec, sort, group) - if check('View', userid, self.classname, itemid=id)] + l = [id for id in klass.filter(matches, filterspec, sort, group)] + + # Prefetch for performance, since security access-control is likely to + # look at the rows to determine access + klass.fetch(l) + + l = [id for id in l if check('View', userid, self.classname, itemid=id)] # return the batch object, using IDs only return Batch(self.client, l, self.pagesize, self.startwith, -- 1.5.5.6 From ac349314b01bda3b51caf8e55f49f2207ffa69f8 Mon Sep 17 00:00:00 2001 From: Ulrik Mikaelsson Date: Fri, 20 Feb 2009 16:49:43 +0100 Subject: [PATCH] - Increase the size of the row-cache. Memory is cheap, SQL-queries are not. --- roundup/backends/rdbms_common.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/roundup/backends/rdbms_common.py b/roundup/backends/rdbms_common.py index 36084d8..7eecb78 100644 --- a/roundup/backends/rdbms_common.py +++ b/roundup/backends/rdbms_common.py @@ -73,7 +73,7 @@ from sessions_rdbms import Sessions, OneTimeKeys from roundup.date import Range # number of rows to keep in memory -ROW_CACHE_SIZE = 100 +ROW_CACHE_SIZE = 2048 # dummy value meaning "argument not passed" _marker = [] -- 1.5.5.6