search: Stub out a preliminary search index

This implements the

  GET /v1/search

endpoint using a simple JSON object for storage and a global Python
dict for live access.  That's not going to scale well for large
registries, but it's a start.  Benefits of this approach:

* We automatically support all the existing storage backends.
* The in-memory Python dict gives fast read-only access.

Drawbacks to this approach:

* Searches iterate over all key/value pairs (there's no precompiled
  index), so they'll be slow if you have tons of repositories.
* We write the whole index to the storage backend whenever a
  repository is created, updated, or deleted.
* If you have multiple registries sharing the same storage backend
  (e.g. gunicorn workers), your in-memory Python indexes will go
  out-of-sync, and may end up clobbering changes that other registries
  made to the stored index.

So this is basically a proof-of-concept for small registries.  A more
serious implementation would use an external SQL database to overcome
all of these limitations.  That should be easy to add later; for now
I'm focusing on getting the internal logic right.  Slotting in a
better backend later should be easy.

Because the search index code is only bound to registry.index by
signals, I've moved it into it's own registry.search module.  I added
a commented-out:

  #index = Index()

in the global-declaration section of the new module, to remind folks
that we would be creating a global 'index' object.  We can't actually
declare the global at that point though, because the Index class
hasn't been defined yet.

There's a version stamp in the saved index, and a missing index file
counts as v0, so folks who have been running earlier versions of
docker-registry will have their index built automatically the next
time they start the app.

Author: wking

lib/storage/__init__.py

@@ -12,13 +12,15 @@ class Storage(object):
     """Storage is organized as follow:
     $ROOT/images/<image_id>/json
     $ROOT/images/<image_id>/layer
+    $ROOT/index/json
     $ROOT/repositories/<namespace>/<repository_name>/<tag_name>
     """
 
     # Useful if we want to change those locations later without rewriting
     # the code which uses Storage
     repositories = 'repositories'
     images = 'images'
+    index = 'index'
     # Set the IO buffer to 128kB
     buffer_size = 128 * 1024
     # By default no storage plugin supports it
@@ -85,6 +87,9 @@ def private_flag_path(self, namespace, repository):
     def is_private(self, namespace, repository):
         return self.exists(self.private_flag_path(namespace, repository))
 
+    def index_path(self):
+        return '{0}/json'.format(self.index)
+
     def get_content(self, path):
         raise NotImplementedError
 

registry/__init__.py

@@ -4,6 +4,7 @@
 from .app import app
 from .tags import *
 from .images import *
+from . import search as search
 from .status import *
 
 import config

registry/index.py

@@ -133,8 +133,3 @@ def delete_repository_images(namespace, repository):
 @toolkit.parse_repository_name
 def put_repository_auth(namespace, repository):
     return toolkit.response('OK')
-
-
-@app.route('/v1/search', methods=['GET'])
-def get_search():
-    return toolkit.response({})

registry/search.py

@@ -0,0 +1,116 @@
+__all__ = ['get_search']
+
+import flask
+import simplejson as json
+
+import signals
+import storage
+import toolkit
+
+from .app import app
+
+
+store = storage.load()
+#index = Index()
+
+
+class Index (dict):
+    """Maintain an index of repository data
+
+    The index is a dictionary.  The keys are
+    '{namespace}/{repository}' strings, and the values are description
+    strings.  For example:
+
+      index['library/ubuntu'] = 'An ubuntu image...'
+    """
+    def __init__(self):
+        super(Index, self).__init__()
+        self.version = 1
+        self.load()
+        signals.repository_created.connect(self._handler_repository_created)
+        signals.repository_updated.connect(self._handler_repository_created)
+        signals.repository_deleted.connect(self._handler_repository_deleted)
+
+    def load(self):
+        regenerated = False
+        try:
+            index_content = store.get_content(store.index_path())
+        except (OSError, IOError):
+            index_data = self._regenerate_index()
+            regenerated = True
+        else:
+            data = json.loads(index_content)
+            if data['version'] != self.version:
+                raise NotImplementedError(
+                    'unrecognized search index version {0}'.format(
+                        data['version']))
+            index_data = data['index']
+        self.clear()
+        self.update(index_data)
+        if regenerated:
+            self.save()
+
+    def save(self):
+        index_data = {
+            'version': self.version,
+            'index': dict(self),
+        }
+        store.put_content(store.index_path(), json.dumps(index_data))
+
+    def _regenerate_index(self):
+        index_data = {}
+        description = ''  # TODO(wking): store descriptions
+        try:
+            namespace_paths = list(
+                store.list_directory(path=store.repositories))
+        except OSError:
+            namespace_paths = []
+        for namespace_path in namespace_paths:
+            namespace = namespace_path.rsplit('/', 1)[-1]
+            try:
+                repository_paths = list(
+                    store.list_directory(path=namespace_path))
+            except OSError:
+                repository_paths = []
+            for path in repository_paths:
+                repository = path.rsplit('/', 1)[-1]
+                key = '{0}/{1}'.format(namespace, repository)
+                index_data[key] = description
+        return index_data
+
+    def _handler_repository_created(
+            self, sender, namespace, repository, value):
+        key = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        self[key] = description
+        self.save()
+
+    def _handler_repository_deleted(self, sender, namespace, repository):
+        key = '{0}/{1}'.format(namespace, repository)
+        try:
+            self.pop(key)
+        except KeyError:
+            pass
+        else:
+            self.save()
+
+
+index = Index()
+
+
+@app.route('/v1/search', methods=['GET'])
+def get_search():
+    search_term = flask.request.args.get('q', '')
+    results = [
+        {
+            'name': name,
+            'description': description,
+        }
+        for name, description in index.items()
+        if search_term in name
+        or search_term in description]
+    return toolkit.response({
+        'query': search_term,
+        'num_results': len(results),
+        'results': results,
+    })