[enh] introducing woosh

apps/meps/models.py

@@ -1,7 +1,9 @@
 from datetime import date
 from django.db import models
 from django.contrib.comments.moderation import CommentModerator, moderator
+from django.core.urlresolvers import reverse
 from memopol2.utils import reify
+from memopol2 import search
 
 from reps.models import Representative, Party
 
@@ -27,6 +29,7 @@ def meps(self):
         return self.mep_set.filter(active=True, countrymep__end=date(9999, 12, 31)).distinct()
 
 
+@search.searchable
 class Group(models.Model):
     abbreviation = models.CharField(max_length=10, unique=True)
     name = models.CharField(max_length=100, unique=True)
@@ -35,25 +38,34 @@ class Group(models.Model):
 
     def __unicode__(self):
         return u"%s - %s" % (self.abbreviation, self.name)
+    content = __unicode__
+
+    def get_absolute_url(self):
+        return reverse('meps:index_by_group', args=(self.abbreviation,))
 
     @property
     def meps(self):
         return self.mep_set.filter(active=True, groupmep__end=date(9999, 12, 31)).distinct()
 
-
+@search.searchable
 class Delegation(models.Model):
     name = models.CharField(max_length=255, unique=True)
 
     objects = MepsContainerManager()
 
     def __unicode__(self):
         return self.name
+    content = __unicode__
+
+    def get_absolute_url(self):
+        return reverse('meps:index_by_delegation', args=(self.id,))
 
     @property
     def meps(self):
         return self.mep_set.filter(active=True, delegationrole__end=date(9999, 12, 31)).distinct()
 
 
+@search.searchable
 class Committee(models.Model):
     name = models.CharField(max_length=255, unique=True)
     abbreviation = models.CharField(max_length=30, unique=True)
@@ -62,6 +74,10 @@ class Committee(models.Model):
 
     def __unicode__(self):
         return u"%s: %s" % (self.abbreviation, self.name)
+    content = __unicode__
+
+    def get_absolute_url(self):
+        return reverse('meps:index_by_committee', args=(self.abbreviation,))
 
     @property
     def meps(self):
@@ -76,18 +92,24 @@ class Building(models.Model):
     postcode = models.CharField(max_length=255)
 
 
+@search.searchable
 class Organization(models.Model):
     name = models.CharField(max_length=255, unique=True)
 
     def __unicode__(self):
         return self.name
+    content = __unicode__
+
+    def get_absolute_url(self):
+        return reverse('meps:index_by_organization', args=(self.id,))
 
     @property
     def meps(self):
         return self.mep_set.filter(active=True, organizationmep__end=date(9999, 12, 31)).distinct()
 
 
 
+@search.searchable
 class MEP(Representative):
     active = models.BooleanField()
     ep_id = models.IntegerField(unique=True)
@@ -118,6 +140,10 @@ def __unicode__(self):
         if self.full_name:
             return self.full_name
         return u'%s %s' (self.first_name, self.last_name)
+    content = __unicode__
+
+    def get_absolute_url(self):
+        return reverse('meps:mep', args=(self.id,))
 
     @reify
     def group(self):

memopol2/search.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+import os
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+import logging
+from django.db.models import signals
+from django.conf import settings
+from whoosh import fields, index
+from whoosh.filedb.filestore import FileStorage
+
+log = logging.getLogger(__name__)
+
+WHOOSH_SCHEMA = fields.Schema(title=fields.TEXT(stored=True),
+                              content=fields.TEXT,
+                              url=fields.ID(stored=True, unique=True))
+
+def create_index(sender=None, **kwargs):
+    if not os.path.exists(settings.WHOOSH_INDEX):
+        os.mkdir(settings.WHOOSH_INDEX)
+        storage = FileStorage(settings.WHOOSH_INDEX)
+        storage.create_index(WHOOSH_SCHEMA, indexname='memopol')
+
+signals.post_syncdb.connect(create_index)
+
+def update_index(sender, instance, created, **kwargs):
+    storage = FileStorage(settings.WHOOSH_INDEX)
+    ix = storage.open_index(indexname='memopol')
+    writer = ix.writer()
+    content = getattr(instance, 'content', None)
+    if content is None:
+        content = unicode(instance)
+    elif callable(content):
+        content = content()
+    if created:
+        writer.add_document(title=unicode(instance), content=content,
+                                    url=unicode(instance.get_absolute_url()))
+        writer.commit()
+    else:
+        writer.update_document(title=unicode(instance), content=content,
+                                    url=unicode(instance.get_absolute_url()))
+        writer.commit()
+
+_searchables = []
+def searchable(klass):
+    if hasattr(klass, 'get_absolute_url'):
+        signals.post_save.connect(update_index, sender=klass)
+        _searchables.append(klass)
+        if not hasattr(klass, 'content'):
+            log.warn('%s is declared as searchable but has no content attribute' % klass)
+    else:
+        log.warn('%s is declared as searchable but has no get_absolute_url' % klass)
+    return klass
+
+def update():
+    from meps import models
+    from mps import models
+    from reps import models
+    create_index()
+    for klass in _searchables:
+        for i in klass.objects.all():
+            update_index(None, i, created=False)
+

memopol2/settings.py

@@ -22,9 +22,12 @@
     },
 }
 
+WHOOSH_INDEX = '/tmp/%s-memopol2.index' % os.getenv('USER')
+
 APPS_DEBUG = False
 if os.getenv('VIRTUAL_ENV'):
     DATABASES['default']['NAME'] = '%s/memopol2.sqlite' % os.getenv('VIRTUAL_ENV')
+    WHOOSH_INDEX = '%s/memopol2.index' % os.getenv('VIRTUAL_ENV')
     APPS_DEBUG = True
 elif not os.path.isfile('bin/django-manage'):
     APPS_DEBUG = True

requirements.txt

@@ -5,6 +5,7 @@ numpy==1.5.1
 git+http://github.com/matplotlib/matplotlib.git@3bf76b9fe6152280950168f9a3791f56c508eeed#egg=matplotlib
 hg+https://bitbucket.org/gregmuellegger/django-contact-form-i18n#egg=django-contact-form
 django-flatblocks==0.5
+whoosh
 lxml
 pyquery
 WebTest

setup.py

@@ -4,16 +4,6 @@
 import os
 from setuptools import setup, find_packages
 
-def find_files(dirname):
-    files = []
-    for entryname in os.listdir(dirname):
-        pathname = "%s/%s" % (dirname, entryname)
-        if os.path.isfile(pathname):
-            files.append(pathname)
-        elif os.path.isdir(pathname):
-            files += find_files(pathname)        
-    return files
-
 setup(
     name='Memopol',
     version = '1.99.1',
@@ -25,6 +15,9 @@ def find_files(dirname):
     url = 'http://projets.lqdn.fr/projects/mempol',
     packages = find_packages(),
     include_package_data = True,
-    scripts = find_files('bin'),
     install_requires = [],
+    entry_points = """
+    [console_scripts]
+    update_search_index = memopol2.search:update
+    """
 )