From 427c20d4e3ead3a6f988a07a41788fbb03ecfedb Mon Sep 17 00:00:00 2001 From: John Kristensen Date: Fri, 21 Aug 2015 18:35:08 +1000 Subject: [PATCH] Remove the tsearch2 backend The documentation within the tsearch2 backend labels it as being experimental and that it should not be used. The have_backend() function in roundup.backend returns False indicating that it does not exist and is labeled as "currently not working". The PostgreSQL website also seems to indicate that it has been deprecated since v8.3 when text searching was integrated into the core[1]. Considering all this, it seems like the best option is to just remove the tsearch2 backend. [1] http://www.postgresql.org/docs/9.4/static/tsearch2.html --- 2to3-done.txt | 3 - roundup/backends/__init__.py | 4 - roundup/backends/back_tsearch2.py | 176 --------- roundup/backends/tsearch2_setup.py | 735 ------------------------------------- test/README.txt | 1 - test/test_tsearch2.py | 122 ------ 6 files changed, 1041 deletions(-) delete mode 100644 roundup/backends/back_tsearch2.py delete mode 100644 roundup/backends/tsearch2_setup.py delete mode 100644 test/test_tsearch2.py diff --git a/2to3-done.txt b/2to3-done.txt index e82e1f0705..f31d614745 100644 --- a/2to3-done.txt +++ b/2to3-done.txt @@ -1,7 +1,6 @@ CAN'T VERIFY ./roundup/backends/back_mysql.py -./roundup/backends/back_tsearch2.py TODO @@ -109,7 +108,6 @@ TODO ./test/test_templating.py ./test/test_textfmt.py ./test/test_token.py -./test/test_tsearch2.py ./test/test_userauditor.py ./test/test_xmlrpc.py ./test.py @@ -149,7 +147,6 @@ NOTHING DONE ./roundup/anypy/__init__.py ./roundup/backends/blobfiles.py ./roundup/backends/indexer_xapian.py -./roundup/backends/tsearch2_setup.py ./roundup/cgi/__init__.py ./roundup/cgi/apache.py ./roundup/cgi/client.py diff --git a/roundup/backends/__init__.py b/roundup/backends/__init__.py index 93ecb31d4f..d439751d54 100644 --- a/roundup/backends/__init__.py +++ b/roundup/backends/__init__.py @@ -28,7 +28,6 @@ import sys _modules = { 'mysql': ('MySQLdb',), 'postgresql': ('psycopg',), - 'tsearch2': ('psycopg',), 'sqlite': ('pysqlite', 'pysqlite2', 'sqlite3', '_sqlite3', 'sqlite'), } @@ -46,9 +45,6 @@ def get_backend(name): def have_backend(name): '''Is backend "name" available?''' - if name == 'tsearch2': - # currently not working - return 0 try: get_backend(name) return 1 diff --git a/roundup/backends/back_tsearch2.py b/roundup/backends/back_tsearch2.py deleted file mode 100644 index 2b67ad421c..0000000000 --- a/roundup/backends/back_tsearch2.py +++ /dev/null @@ -1,176 +0,0 @@ -# Note: this backend is EXPERIMENTAL. Do not use if you value your data. -import re - -import psycopg - -from roundup import hyperdb -from roundup.support import ensureParentsExist -from roundup.backends import back_postgresql, tsearch2_setup, indexer_rdbms -from roundup.backends.back_postgresql import db_create, db_nuke, db_command -from roundup.backends.back_postgresql import pg_command, db_exists, Class, IssueClass, FileClass -from roundup.backends.indexer_common import _isLink, Indexer - -# XXX: Should probably be on the Class class. -def _indexedProps(spec): - """Get a list of properties to be indexed on 'spec'.""" - return [prop for prop, propclass in spec.getprops().items() - if isinstance(propclass, hyperdb.String) and propclass.indexme] - -def _getQueryDict(spec): - """Get a convenience dictionary for creating tsearch2 indexes.""" - query_dict = {'classname': spec.classname, - 'indexedColumns': ['_' + prop for prop in _indexedProps(spec)]} - query_dict['tablename'] = "_%(classname)s" % query_dict - query_dict['triggername'] = "%(tablename)s_tsvectorupdate" % query_dict - return query_dict - -class Database(back_postgresql.Database): - def __init__(self, config, journaltag=None): - back_postgresql.Database.__init__(self, config, journaltag) - self.indexer = Indexer(self) - - def create_version_2_tables(self): - back_postgresql.Database.create_version_2_tables(self) - tsearch2_setup.setup(self.cursor) - - def create_class_table_indexes(self, spec): - back_postgresql.Database.create_class_table_indexes(self, spec) - self.cursor.execute("""CREATE INDEX _%(classname)s_idxFTI_idx - ON %(tablename)s USING gist(idxFTI);""" % - _getQueryDict(spec)) - - self.create_tsearch2_trigger(spec) - - def create_tsearch2_trigger(self, spec): - d = _getQueryDict(spec) - if d['indexedColumns']: - - d['joined'] = " || ' ' ||".join(d['indexedColumns']) - query = """UPDATE %(tablename)s - SET idxFTI = to_tsvector('default', %(joined)s)""" % d - self.cursor.execute(query) - - d['joined'] = ", ".join(d['indexedColumns']) - query = """CREATE TRIGGER %(triggername)s - BEFORE UPDATE OR INSERT ON %(tablename)s - FOR EACH ROW EXECUTE PROCEDURE - tsearch2(idxFTI, %(joined)s);""" % d - self.cursor.execute(query) - - def drop_tsearch2_trigger(self, spec): - # Check whether the trigger exists before trying to drop it. - query_dict = _getQueryDict(spec) - self.sql("""SELECT tgname FROM pg_catalog.pg_trigger - WHERE tgname = '%(triggername)s'""" % query_dict) - if self.cursor.fetchall(): - self.sql("""DROP TRIGGER %(triggername)s ON %(tablename)s""" % - query_dict) - - def update_class(self, spec, old_spec, force=0): - result = back_postgresql.Database.update_class(self, spec, old_spec, force) - - # Drop trigger... - self.drop_tsearch2_trigger(spec) - - # and recreate if necessary. - self.create_tsearch2_trigger(spec) - - return result - - def determine_all_columns(self, spec): - cols, mls = back_postgresql.Database.determine_all_columns(self, spec) - cols.append(('idxFTI', 'tsvector')) - return cols, mls - -class Indexer(Indexer): - def __init__(self, db): - self.db = db - - # This indexer never needs to reindex. - def should_reindex(self): - return 0 - - def getHits(self, search_terms, klass): - return self.find(search_terms, klass) - - def find(self, search_terms, klass): - if not search_terms: - return None - - hits = self.tsearchQuery(klass.classname, search_terms) - designator_propname = {} - - for nm, propclass in klass.getprops().items(): - if _isLink(propclass): - hits.extend(self.tsearchQuery(propclass.classname, search_terms)) - - return hits - - def tsearchQuery(self, classname, search_terms): - query = """SELECT id FROM _%(classname)s - WHERE idxFTI @@ to_tsquery('default', '%(terms)s')""" - - query = query % {'classname': classname, - 'terms': ' & '.join(search_terms)} - self.db.cursor.execute(query) - klass = self.db.getclass(classname) - nodeids = [str(row[0]) for row in self.db.cursor.fetchall()] - - # filter out files without text/plain mime type - # XXX: files without text/plain shouldn't be indexed at all, we - # should take care of this in the trigger - if klass.getprops().has_key('type'): - nodeids = [nodeid for nodeid in nodeids - if klass.get(nodeid, 'type') == 'text/plain'] - - # XXX: We haven't implemented property-level search, so I'm just faking - # it here with a property named 'XXX'. We still need to fix the other - # backends and indexer_common.Indexer.search to only want to unpack two - # values. - return [(classname, nodeid, 'XXX') for nodeid in nodeids] - - # These only exist to satisfy the interface that's expected from indexers. - def force_reindex(self): - pass - - def add_text(self, identifier, text, mime_type=None): - pass - - def close(self): - pass - -class FileClass(hyperdb.FileClass, Class): - '''This class defines a large chunk of data. To support this, it has a - mandatory String property "content" which is typically saved off - externally to the hyperdb. - - However, this implementation just stores it in the hyperdb. - ''' - def __init__(self, db, classname, **properties): - '''The newly-created class automatically includes the "content" property., - ''' - properties['content'] = hyperdb.String(indexme='yes') - Class.__init__(self, db, classname, **properties) - - default_mime_type = 'text/plain' - def create(self, **propvalues): - # figure the mime type - if self.getprops().has_key('type') and not propvalues.get('type'): - propvalues['type'] = self.default_mime_type - return Class.create(self, **propvalues) - - def export_files(self, dirname, nodeid): - dest = self.exportFilename(dirname, nodeid) - ensureParentsExist(dest) - fp = open(dest, "w") - fp.write(self.get(nodeid, "content", default='')) - fp.close() - - def import_files(self, dirname, nodeid): - source = self.exportFilename(dirname, nodeid) - - fp = open(source, "r") - # Use Database.setnode instead of self.set or self.set_inner here, as - # Database.setnode doesn't update the "activity" or "actor" properties. - self.db.setnode(self.classname, nodeid, values={'content': fp.read()}) - fp.close() diff --git a/roundup/backends/tsearch2_setup.py b/roundup/backends/tsearch2_setup.py deleted file mode 100644 index e28fd81cb5..0000000000 --- a/roundup/backends/tsearch2_setup.py +++ /dev/null @@ -1,735 +0,0 @@ -# All the SQL in this module is taken from the tsearch2 module in the contrib -# tree of PostgreSQL 7.4.6. PostgreSQL, and this code, has the following -# license: -# -# PostgreSQL Data Base Management System -# (formerly known as Postgres, then as Postgres95). -# -# Portions Copyright (c) 1996-2003, The PostgreSQL Global Development Group -# -# Portions Copyright (c) 1994, The Regents of the University of California -# -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose, without fee, and without a written agreement -# is hereby granted, provided that the above copyright notice and this -# paragraph and the following two paragraphs appear in all copies. -# -# IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR -# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING -# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS -# DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, -# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS -# ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO -# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. - -tsearch_sql = """ -- Adjust this setting to control where the objects get CREATEd. -SET search_path = public; - ---dict conf -CREATE TABLE pg_ts_dict ( - dict_name text not null primary key, - dict_init oid, - dict_initoption text, - dict_lexize oid not null, - dict_comment text -) with oids; - ---dict interface -CREATE FUNCTION lexize(oid, text) - returns _text - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION lexize(text, text) - returns _text - as '$libdir/tsearch2', 'lexize_byname' - language 'C' - with (isstrict); - -CREATE FUNCTION lexize(text) - returns _text - as '$libdir/tsearch2', 'lexize_bycurrent' - language 'C' - with (isstrict); - -CREATE FUNCTION set_curdict(int) - returns void - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION set_curdict(text) - returns void - as '$libdir/tsearch2', 'set_curdict_byname' - language 'C' - with (isstrict); - ---built-in dictionaries -CREATE FUNCTION dex_init(text) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION dex_lexize(internal,internal,int4) - returns internal - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -insert into pg_ts_dict select - 'simple', - (select oid from pg_proc where proname='dex_init'), - null, - (select oid from pg_proc where proname='dex_lexize'), - 'Simple example of dictionary.' -; - -CREATE FUNCTION snb_en_init(text) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION snb_lexize(internal,internal,int4) - returns internal - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -insert into pg_ts_dict select - 'en_stem', - (select oid from pg_proc where proname='snb_en_init'), - '/usr/share/postgresql/contrib/english.stop', - (select oid from pg_proc where proname='snb_lexize'), - 'English Stemmer. Snowball.' -; - -CREATE FUNCTION snb_ru_init(text) - returns internal - as '$libdir/tsearch2' - language 'C'; - -insert into pg_ts_dict select - 'ru_stem', - (select oid from pg_proc where proname='snb_ru_init'), - '/usr/share/postgresql/contrib/russian.stop', - (select oid from pg_proc where proname='snb_lexize'), - 'Russian Stemmer. Snowball.' -; - -CREATE FUNCTION spell_init(text) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION spell_lexize(internal,internal,int4) - returns internal - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -insert into pg_ts_dict select - 'ispell_template', - (select oid from pg_proc where proname='spell_init'), - null, - (select oid from pg_proc where proname='spell_lexize'), - 'ISpell interface. Must have .dict and .aff files' -; - -CREATE FUNCTION syn_init(text) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION syn_lexize(internal,internal,int4) - returns internal - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -insert into pg_ts_dict select - 'synonym', - (select oid from pg_proc where proname='syn_init'), - null, - (select oid from pg_proc where proname='syn_lexize'), - 'Example of synonym dictionary' -; - ---dict conf -CREATE TABLE pg_ts_parser ( - prs_name text not null primary key, - prs_start oid not null, - prs_nexttoken oid not null, - prs_end oid not null, - prs_headline oid not null, - prs_lextype oid not null, - prs_comment text -) with oids; - ---sql-level interface -CREATE TYPE tokentype - as (tokid int4, alias text, descr text); - -CREATE FUNCTION token_type(int4) - returns setof tokentype - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION token_type(text) - returns setof tokentype - as '$libdir/tsearch2', 'token_type_byname' - language 'C' - with (isstrict); - -CREATE FUNCTION token_type() - returns setof tokentype - as '$libdir/tsearch2', 'token_type_current' - language 'C' - with (isstrict); - -CREATE FUNCTION set_curprs(int) - returns void - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION set_curprs(text) - returns void - as '$libdir/tsearch2', 'set_curprs_byname' - language 'C' - with (isstrict); - -CREATE TYPE tokenout - as (tokid int4, token text); - -CREATE FUNCTION parse(oid,text) - returns setof tokenout - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION parse(text,text) - returns setof tokenout - as '$libdir/tsearch2', 'parse_byname' - language 'C' - with (isstrict); - -CREATE FUNCTION parse(text) - returns setof tokenout - as '$libdir/tsearch2', 'parse_current' - language 'C' - with (isstrict); - ---default parser -CREATE FUNCTION prsd_start(internal,int4) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION prsd_getlexeme(internal,internal,internal) - returns int4 - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION prsd_end(internal) - returns void - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION prsd_lextype(internal) - returns internal - as '$libdir/tsearch2' - language 'C'; - -CREATE FUNCTION prsd_headline(internal,internal,internal) - returns internal - as '$libdir/tsearch2' - language 'C'; - -insert into pg_ts_parser select - 'default', - (select oid from pg_proc where proname='prsd_start'), - (select oid from pg_proc where proname='prsd_getlexeme'), - (select oid from pg_proc where proname='prsd_end'), - (select oid from pg_proc where proname='prsd_headline'), - (select oid from pg_proc where proname='prsd_lextype'), - 'Parser from OpenFTS v0.34' -; - ---tsearch config - -CREATE TABLE pg_ts_cfg ( - ts_name text not null primary key, - prs_name text not null, - locale text -) with oids; - -CREATE TABLE pg_ts_cfgmap ( - ts_name text not null, - tok_alias text not null, - dict_name text[], - primary key (ts_name,tok_alias) -) with oids; - -CREATE FUNCTION set_curcfg(int) - returns void - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -CREATE FUNCTION set_curcfg(text) - returns void - as '$libdir/tsearch2', 'set_curcfg_byname' - language 'C' - with (isstrict); - -CREATE FUNCTION show_curcfg() - returns oid - as '$libdir/tsearch2' - language 'C' - with (isstrict); - -insert into pg_ts_cfg values ('default', 'default','C'); -insert into pg_ts_cfg values ('default_russian', 'default','ru_RU.KOI8-R'); -insert into pg_ts_cfg values ('simple', 'default'); - -insert into pg_ts_cfgmap values ('default', 'lword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default', 'nlword', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'word', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'email', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'url', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'host', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'sfloat', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'version', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'part_hword', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'nlpart_hword', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'lpart_hword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default', 'hword', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'lhword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default', 'nlhword', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'uri', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'file', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'float', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'int', '{simple}'); -insert into pg_ts_cfgmap values ('default', 'uint', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'lword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'nlword', '{ru_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'word', '{ru_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'email', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'url', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'host', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'sfloat', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'version', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'part_hword', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'nlpart_hword', '{ru_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'lpart_hword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'hword', '{ru_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'lhword', '{en_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'nlhword', '{ru_stem}'); -insert into pg_ts_cfgmap values ('default_russian', 'uri', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'file', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'float', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'int', '{simple}'); -insert into pg_ts_cfgmap values ('default_russian', 'uint', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'lword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'nlword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'word', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'email', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'url', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'host', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'sfloat', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'version', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'part_hword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'nlpart_hword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'lpart_hword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'hword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'lhword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'nlhword', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'uri', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'file', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'float', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'int', '{simple}'); -insert into pg_ts_cfgmap values ('simple', 'uint', '{simple}'); - ---tsvector type -CREATE FUNCTION tsvector_in(cstring) -RETURNS tsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE FUNCTION tsvector_out(tsvector) -RETURNS cstring -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE TYPE tsvector ( - INTERNALLENGTH = -1, - INPUT = tsvector_in, - OUTPUT = tsvector_out, - STORAGE = extended -); - -CREATE FUNCTION length(tsvector) -RETURNS int4 -AS '$libdir/tsearch2', 'tsvector_length' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION to_tsvector(oid, text) -RETURNS tsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION to_tsvector(text, text) -RETURNS tsvector -AS '$libdir/tsearch2', 'to_tsvector_name' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION to_tsvector(text) -RETURNS tsvector -AS '$libdir/tsearch2', 'to_tsvector_current' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION strip(tsvector) -RETURNS tsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION setweight(tsvector,"char") -RETURNS tsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE FUNCTION concat(tsvector,tsvector) -RETURNS tsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict,iscachable); - -CREATE OPERATOR || ( - LEFTARG = tsvector, - RIGHTARG = tsvector, - PROCEDURE = concat -); - ---query type -CREATE FUNCTION tsquery_in(cstring) -RETURNS tsquery -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE FUNCTION tsquery_out(tsquery) -RETURNS cstring -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE TYPE tsquery ( - INTERNALLENGTH = -1, - INPUT = tsquery_in, - OUTPUT = tsquery_out -); - -CREATE FUNCTION querytree(tsquery) -RETURNS text -AS '$libdir/tsearch2', 'tsquerytree' -LANGUAGE 'C' with (isstrict); - -CREATE FUNCTION to_tsquery(oid, text) -RETURNS tsquery -AS '$libdir/tsearch2' -LANGUAGE 'c' with (isstrict,iscachable); - -CREATE FUNCTION to_tsquery(text, text) -RETURNS tsquery -AS '$libdir/tsearch2','to_tsquery_name' -LANGUAGE 'c' with (isstrict,iscachable); - -CREATE FUNCTION to_tsquery(text) -RETURNS tsquery -AS '$libdir/tsearch2','to_tsquery_current' -LANGUAGE 'c' with (isstrict,iscachable); - ---operations -CREATE FUNCTION exectsq(tsvector, tsquery) -RETURNS bool -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict, iscachable); - -COMMENT ON FUNCTION exectsq(tsvector, tsquery) IS 'boolean operation with text index'; - -CREATE FUNCTION rexectsq(tsquery, tsvector) -RETURNS bool -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict, iscachable); - -COMMENT ON FUNCTION rexectsq(tsquery, tsvector) IS 'boolean operation with text index'; - -CREATE OPERATOR @@ ( - LEFTARG = tsvector, - RIGHTARG = tsquery, - PROCEDURE = exectsq, - COMMUTATOR = '@@', - RESTRICT = contsel, - JOIN = contjoinsel -); -CREATE OPERATOR @@ ( - LEFTARG = tsquery, - RIGHTARG = tsvector, - PROCEDURE = rexectsq, - COMMUTATOR = '@@', - RESTRICT = contsel, - JOIN = contjoinsel -); - ---Trigger -CREATE FUNCTION tsearch2() -RETURNS trigger -AS '$libdir/tsearch2' -LANGUAGE 'C'; - ---Relevation -CREATE FUNCTION rank(float4[], tsvector, tsquery) -RETURNS float4 -AS '$libdir/tsearch2' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank(float4[], tsvector, tsquery, int4) -RETURNS float4 -AS '$libdir/tsearch2' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank(tsvector, tsquery) -RETURNS float4 -AS '$libdir/tsearch2', 'rank_def' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank(tsvector, tsquery, int4) -RETURNS float4 -AS '$libdir/tsearch2', 'rank_def' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank_cd(int4, tsvector, tsquery) -RETURNS float4 -AS '$libdir/tsearch2' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank_cd(int4, tsvector, tsquery, int4) -RETURNS float4 -AS '$libdir/tsearch2' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank_cd(tsvector, tsquery) -RETURNS float4 -AS '$libdir/tsearch2', 'rank_cd_def' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION rank_cd(tsvector, tsquery, int4) -RETURNS float4 -AS '$libdir/tsearch2', 'rank_cd_def' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(oid, text, tsquery, text) -RETURNS text -AS '$libdir/tsearch2', 'headline' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(oid, text, tsquery) -RETURNS text -AS '$libdir/tsearch2', 'headline' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(text, text, tsquery, text) -RETURNS text -AS '$libdir/tsearch2', 'headline_byname' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(text, text, tsquery) -RETURNS text -AS '$libdir/tsearch2', 'headline_byname' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(text, tsquery, text) -RETURNS text -AS '$libdir/tsearch2', 'headline_current' -LANGUAGE 'C' WITH (isstrict, iscachable); - -CREATE FUNCTION headline(text, tsquery) -RETURNS text -AS '$libdir/tsearch2', 'headline_current' -LANGUAGE 'C' WITH (isstrict, iscachable); - ---GiST ---GiST key type -CREATE FUNCTION gtsvector_in(cstring) -RETURNS gtsvector -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE FUNCTION gtsvector_out(gtsvector) -RETURNS cstring -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE TYPE gtsvector ( - INTERNALLENGTH = -1, - INPUT = gtsvector_in, - OUTPUT = gtsvector_out -); - --- support FUNCTIONs -CREATE FUNCTION gtsvector_consistent(gtsvector,internal,int4) -RETURNS bool -AS '$libdir/tsearch2' -LANGUAGE 'C'; - -CREATE FUNCTION gtsvector_compress(internal) -RETURNS internal -AS '$libdir/tsearch2' -LANGUAGE 'C'; - -CREATE FUNCTION gtsvector_decompress(internal) -RETURNS internal -AS '$libdir/tsearch2' -LANGUAGE 'C'; - -CREATE FUNCTION gtsvector_penalty(internal,internal,internal) -RETURNS internal -AS '$libdir/tsearch2' -LANGUAGE 'C' with (isstrict); - -CREATE FUNCTION gtsvector_picksplit(internal, internal) -RETURNS internal -AS '$libdir/tsearch2' -LANGUAGE 'C'; - -CREATE FUNCTION gtsvector_union(bytea, internal) -RETURNS _int4 -AS '$libdir/tsearch2' -LANGUAGE 'C'; - -CREATE FUNCTION gtsvector_same(gtsvector, gtsvector, internal) -RETURNS internal -AS '$libdir/tsearch2' -LANGUAGE 'C'; - --- CREATE the OPERATOR class -CREATE OPERATOR CLASS gist_tsvector_ops -DEFAULT FOR TYPE tsvector USING gist -AS - OPERATOR 1 @@ (tsvector, tsquery) RECHECK , - FUNCTION 1 gtsvector_consistent (gtsvector, internal, int4), - FUNCTION 2 gtsvector_union (bytea, internal), - FUNCTION 3 gtsvector_compress (internal), - FUNCTION 4 gtsvector_decompress (internal), - FUNCTION 5 gtsvector_penalty (internal, internal, internal), - FUNCTION 6 gtsvector_picksplit (internal, internal), - FUNCTION 7 gtsvector_same (gtsvector, gtsvector, internal), - STORAGE gtsvector; - - ---stat info -CREATE TYPE statinfo - as (word text, ndoc int4, nentry int4); - ---CREATE FUNCTION tsstat_in(cstring) ---RETURNS tsstat ---AS '$libdir/tsearch2' ---LANGUAGE 'C' with (isstrict); --- ---CREATE FUNCTION tsstat_out(tsstat) ---RETURNS cstring ---AS '$libdir/tsearch2' ---LANGUAGE 'C' with (isstrict); --- ---CREATE TYPE tsstat ( --- INTERNALLENGTH = -1, --- INPUT = tsstat_in, --- OUTPUT = tsstat_out, --- STORAGE = plain ---); --- ---CREATE FUNCTION ts_accum(tsstat,tsvector) ---RETURNS tsstat ---AS '$libdir/tsearch2' ---LANGUAGE 'C' with (isstrict); --- ---CREATE FUNCTION ts_accum_finish(tsstat) --- returns setof statinfo --- as '$libdir/tsearch2' --- language 'C' --- with (isstrict); --- ---CREATE AGGREGATE stat ( --- BASETYPE=tsvector, --- SFUNC=ts_accum, --- STYPE=tsstat, --- FINALFUNC = ts_accum_finish, --- initcond = '' ---); - -CREATE FUNCTION stat(text) - returns setof statinfo - as '$libdir/tsearch2', 'ts_stat' - language 'C' - with (isstrict); - ---reset - just for debuging -CREATE FUNCTION reset_tsearch() - returns void - as '$libdir/tsearch2' - language 'C' - with (isstrict); - ---get cover (debug for rank_cd) -CREATE FUNCTION get_covers(tsvector,tsquery) - returns text - as '$libdir/tsearch2' - language 'C' - with (isstrict); - ---debug function -create type tsdebug as ( - ts_name text, - tok_type text, - description text, - token text, - dict_name text[], - "tsvector" tsvector -); - -create function _get_parser_from_curcfg() -returns text as -' select prs_name from pg_ts_cfg where oid = show_curcfg() ' -language 'SQL' with(isstrict,iscachable); - -create function ts_debug(text) -returns setof tsdebug as ' -select - m.ts_name, - t.alias as tok_type, - t.descr as description, - p.token, - m.dict_name, - strip(to_tsvector(p.token)) as tsvector -from - parse( _get_parser_from_curcfg(), $1 ) as p, - token_type() as t, - pg_ts_cfgmap as m, - pg_ts_cfg as c -where - t.tokid=p.tokid and - t.alias = m.tok_alias and - m.ts_name=c.ts_name and - c.oid=show_curcfg() -' language 'SQL' with(isstrict); -""" - -def setup(cursor): - sql = '\n'.join([line for line in tsearch_sql.split('\n') - if not line.startswith('--')]) - for query in sql.split(';'): - if query.strip(): - cursor.execute(query) diff --git a/test/README.txt b/test/README.txt index f77a0ac68f..2104ae2cf0 100644 --- a/test/README.txt +++ b/test/README.txt @@ -11,7 +11,6 @@ test_mysql.py test_postgresql.py test_security.py test_sqlite.py -test_tsearch2.py test_userauditor.py grep "import db_test_base" -l *.py diff --git a/test/test_tsearch2.py b/test/test_tsearch2.py deleted file mode 100644 index 5b16f6380a..0000000000 --- a/test/test_tsearch2.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/) -# This module is free software, and you may redistribute it and/or modify -# under the same terms as Python, so long as this copyright message and -# disclaimer are retained in their original form. -# -# IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR -# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING -# OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" -# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, -# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. - -import unittest - -from roundup.hyperdb import DatabaseError - -from db_test_base import DBTest, ROTest, config, SchemaTest, ClassicInitTest - -from roundup.backends import get_backend, have_backend - -class tsearch2Opener: - if have_backend('tsearch2'): - module = get_backend('tsearch2') - - def setUp(self): - pass - - def tearDown(self): - self.nuke_database() - - def nuke_database(self): - # clear out the database - easiest way is to nuke and re-create it - self.module.db_nuke(config) - -class tsearch2DBTest(tsearch2Opener, DBTest): - def setUp(self): - tsearch2Opener.setUp(self) - DBTest.setUp(self) - - def tearDown(self): - DBTest.tearDown(self) - tsearch2Opener.tearDown(self) - - def testFilteringIntervalSort(self): - # Tsearch2 sorts NULLs differently to other databases (others - # treat it as lower than real values, PG treats it as higher) - ae, filt = self.filteringSetup() - # ascending should sort None, 1:10, 1d - ae(filt(None, {}, ('+','foo'), (None,None)), ['4', '1', '2', '3']) - # descending should sort 1d, 1:10, None - ae(filt(None, {}, ('-','foo'), (None,None)), ['3', '2', '1', '4']) - - def testTransactions(self): - # XXX: in its current form, this test doesn't make sense for tsearch2. - # It tests the transactions mechanism by counting the number of files - # in the FileStorage. As tsearch2 doesn't use the FileStorage, this - # fails. The test should probably be rewritten with some other way of - # checking rollbacks/commits. - pass - -class tsearch2ROTest(tsearch2Opener, ROTest): - def setUp(self): - tsearch2Opener.setUp(self) - ROTest.setUp(self) - - def tearDown(self): - ROTest.tearDown(self) - tsearch2Opener.tearDown(self) - -class tsearch2SchemaTest(tsearch2Opener, SchemaTest): - def setUp(self): - tsearch2Opener.setUp(self) - SchemaTest.setUp(self) - - def tearDown(self): - SchemaTest.tearDown(self) - tsearch2Opener.tearDown(self) - -class tsearch2ClassicInitTest(tsearch2Opener, ClassicInitTest): - backend = 'tsearch2' - def setUp(self): - tsearch2Opener.setUp(self) - ClassicInitTest.setUp(self) - - def tearDown(self): - ClassicInitTest.tearDown(self) - tsearch2Opener.tearDown(self) - -from session_common import RDBMSTest -class tsearch2SessionTest(tsearch2Opener, RDBMSTest): - def setUp(self): - tsearch2Opener.setUp(self) - RDBMSTest.setUp(self) - def tearDown(self): - RDBMSTest.tearDown(self) - tsearch2Opener.tearDown(self) - -def test_suite(): - suite = unittest.TestSuite() - if not have_backend('tsearch2'): - print "Skipping tsearch2 tests" - return suite - - # make sure we start with a clean slate - if tsearch2Opener.module.db_exists(config): - tsearch2Opener.module.db_nuke(config, 1) - - # TODO: Check if we can run postgresql tests - print 'Including tsearch2 tests' - suite.addTest(unittest.makeSuite(tsearch2DBTest)) - suite.addTest(unittest.makeSuite(tsearch2ROTest)) - suite.addTest(unittest.makeSuite(tsearch2SchemaTest)) - suite.addTest(unittest.makeSuite(tsearch2ClassicInitTest)) - suite.addTest(unittest.makeSuite(tsearch2SessionTest)) - return suite - -# vim: set et sts=4 sw=4 : -- 2.5.0