"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "roundup/backends/indexer_rdbms.py" between
roundup-1.6.1.tar.gz and roundup-2.0.0.tar.gz

About: Roundup is an highly customisable issue-tracking system with command-line, web and e-mail interfaces (written in Python).

indexer_rdbms.py  (roundup-1.6.1):indexer_rdbms.py  (roundup-2.0.0)
""" This implements the full-text indexer over two RDBMS tables. The first """ This implements the full-text indexer over two RDBMS tables. The first
is a mapping of words to occurance IDs. The second maps the IDs to (Class, is a mapping of words to occurance IDs. The second maps the IDs to (Class,
propname, itemid) instances. propname, itemid) instances.
""" """
import re import re
from roundup.backends.indexer_common import Indexer as IndexerBase from roundup.backends.indexer_common import Indexer as IndexerBase
from roundup.anypy.strings import us2u, u2s
class Indexer(IndexerBase): class Indexer(IndexerBase):
def __init__(self, db): def __init__(self, db):
IndexerBase.__init__(self, db) IndexerBase.__init__(self, db)
self.db = db self.db = db
self.reindex = 0 self.reindex = 0
def close(self): def close(self):
"""close the indexing database""" """close the indexing database"""
# just nuke the circular reference # just nuke the circular reference
skipping to change at line 64 skipping to change at line 65
sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\ sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\
' values (%s, %s, %s, %s)'%(a, a, a, a) ' values (%s, %s, %s, %s)'%(a, a, a, a)
self.db.cursor.execute(sql, (id, ) + identifier) self.db.cursor.execute(sql, (id, ) + identifier)
else: else:
id = int(r[0]) id = int(r[0])
# clear out any existing indexed values # clear out any existing indexed values
sql = 'delete from __words where _textid=%s'%a sql = 'delete from __words where _textid=%s'%a
self.db.cursor.execute(sql, (id, )) self.db.cursor.execute(sql, (id, ))
# ok, find all the unique words in the text # ok, find all the unique words in the text
if not isinstance(text, unicode): text = us2u(text, "replace")
text = unicode(text, "utf-8", "replace")
text = text.upper() text = text.upper()
wordlist = [w.encode("utf-8") wordlist = [u2s(w)
for w in re.findall(r'(?u)\b\w{%d,%d}\b' for w in re.findall(r'(?u)\b\w{%d,%d}\b'
% (self.minlength, self.maxlength), text )] % (self.minlength, self.maxlength), text )]
words = set() words = set()
for word in wordlist: for word in wordlist:
if self.is_stopword(word): continue if self.is_stopword(word): continue
words.add(word) words.add(word)
# for each word, add an entry in the db # for each word, add an entry in the db
sql = 'insert into __words (_word, _textid) values (%s, %s)'%(a, a) sql = 'insert into __words (_word, _textid) values (%s, %s)'%(a, a)
words = [(word, id) for word in words] words = [(word, id) for word in words]
skipping to change at line 122 skipping to change at line 122
# multiple times. # multiple times.
sql = """select distinct(__words1._textid) sql = """select distinct(__words1._textid)
from __words as __words1 %s from __words as __words1 %s
where __words1._word=%s %s""" where __words1._word=%s %s"""
join_tmpl = ' left join __words as __words%d using (_textid) \n' join_tmpl = ' left join __words as __words%d using (_textid) \n'
match_tmpl = ' and __words%d._word=%s \n' match_tmpl = ' and __words%d._word=%s \n'
join_list = [] join_list = []
match_list = [] match_list = []
for n in xrange(len(l) - 1): for n in range(len(l) - 1):
join_list.append(join_tmpl % (n + 2)) join_list.append(join_tmpl % (n + 2))
match_list.append(match_tmpl % (n + 2, self.db.arg)) match_list.append(match_tmpl % (n + 2, self.db.arg))
sql = sql%(' '.join(join_list), self.db.arg, ' '.join(match_list)) sql = sql%(' '.join(join_list), self.db.arg, ' '.join(match_list))
self.db.cursor.execute(sql, l) self.db.cursor.execute(sql, l)
r = [x[0] for x in self.db.cursor.fetchall()] r = [x[0] for x in self.db.cursor.fetchall()]
if not r: if not r:
return [] return []
 End of changes. 4 change blocks. 
4 lines changed or deleted 4 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)