feat: add cache options

.changeset/cyan-apricots-tie.md

@@ -2,4 +2,6 @@
 '@koopjs/koop-core': minor
 ---
 
-- make 'cache', 'before', and 'after' private props
+- make 'cache', 'before', and 'after' private model props
+- add `cacheSize` option
+- add `cacheTtl` option for provider registration

demo/provider-data/point-cached.geojson

@@ -0,0 +1,19 @@
+{
+  "type": "FeatureCollection",
+  "metadata": {
+    "ttl": 30
+  },
+  "features": [
+    {
+      "type": "Feature",
+      "properties": {},
+      "geometry": {
+        "type": "Point",
+        "coordinates": [
+          -104.9476,
+          39.9448
+        ]
+      }
+    }
+  ]
+}

packages/koop-core/README.md

@@ -94,6 +94,44 @@ const options = {
 }
 ```
 
+### Registering Providers
+When registering a provider you can pass an options object that provides some useful functionality:
+
+```js
+const Koop = require('@koopjs/koop-core')
+const koop = new Koop()
+
+const providerOptions = {
+  name: 'special-name',
+  cacheTtl: 600,
+  before: (req, callback) => {
+    req.query.myHardCodedParam = 'foo';
+    callback();
+  }
+  after: (req, geojson, callback) => {
+    geojson.crs = 'my coordinate system';
+    callback(null, geojson);
+  }
+}
+
+/* Register Koop data providers with options */
+const provider = require('@koopjs/provider-github')
+koop.register(provider, providerOptions)
+```
+
+#### Provider registration options
+##### name
+Use this param to override the name of the provider.  If you supply a value for `name` it will be used in the path for routes registered to this provider.
+
+##### cacheTtl
+Use this param to set the default caching time (in seconds) for any data acquired by this provider. It can be overridden on a request-by-request basis by adding a `ttl` property to the root level of the geojson produced by the provider.
+
+##### before
+Supply a function that is executed _before_ a provider's `getData` method and has access to the Express request object.  This is useful is you want to modify incoming request params before they arrive in a providers `getData` method.
+
+##### after
+Supply a function that is executed _after_ a provider's `getData` method and has access to the Express request object.  This is useful is you want to modify the GeoJSON produce by the provider before it is passed on to the cache or any output plugin.  It's a way of modifying a provider's getData functionality without having to modify the code of the provider itself.
+
 ## Issues
 
 Find a bug or want to request a new feature? Please let us know by submitting an [issue](https://github.com/koopjs/koop/issues).

packages/koop-core/package.json

@@ -22,6 +22,7 @@
     "@koopjs/cache-memory": "5.0.0",
     "@koopjs/logger": "5.0.0",
     "@koopjs/output-geoservices": "7.0.0",
+    "@sindresorhus/fnv1a": "^2.0.1",
     "body-parser": "^1.19.0",
     "compression": "^1.7.4",
     "config": "^3.3.9",

packages/koop-core/src/index.js

@@ -104,7 +104,6 @@ Koop.prototype._registerAuth = function (auth) {
 Koop.prototype._registerProvider = function (provider, options = {}) {
   const providerRegistration = ProviderRegistration.create({ koop: this, provider, options });
   this.providers.push(providerRegistration);
-  this.log.info('registered provider:', providerRegistration.namespace, providerRegistration.version);
 };
 
 /**
@@ -115,7 +114,7 @@ Koop.prototype._registerProvider = function (provider, options = {}) {
  */
 Koop.prototype._registerOutput = function (outputClass, options) {
   this.outputs.push({ outputClass, options });
-  this.log.info('registered output:', outputClass.name, outputClass.version);
+  this.log.info(`registered output: ${outputClass.name} ${outputClass.version}`);
 };
 
 /**

packages/koop-core/src/provider-registration/create-model.js

@@ -1,20 +1,26 @@
 const { promisify } = require('util');
+const hasher = require('@sindresorhus/fnv1a');
+
 const before = (req, callback) => { callback(); };
 const after = (req, data, callback) => { callback(null, data); };
 
 module.exports = function createModel ({ ProviderModel, koop, namespace }, options = {}) {
   class Model extends ProviderModel {
     #cache;
+    #cacheTtl;
     #before;
     #after;
     #cacheRetrieve;
     #cacheInsert;
     #getProviderData;
+    #getLayer;
+    #getCatalog;
 
     constructor (koop, options) {
 
       super(koop, options);
       // Provider constructor's may assign values to this.cache
+      this.#cacheTtl = options.cacheTtl;
       this.#cache = this.cache || options.cache || koop.cache;
       this.namespace = namespace;
       this.logger = koop.log;
@@ -23,13 +29,15 @@ module.exports = function createModel ({ ProviderModel, koop, namespace }, optio
       this.#cacheRetrieve = promisify(this.#cache.retrieve).bind(this.#cache);
       this.#cacheInsert = promisify(this.#cache.insert).bind(this.#cache);
       this.#getProviderData = promisify(this.getData).bind(this);
+      this.#getLayer = this.getLayer ? promisify(this.getLayer).bind(this) : undefined;
+      this.#getCatalog = this.getCatalog ? promisify(this.getCatalog).bind(this) : undefined;
     }
 
     async pull (req, callback) {
-      const key = (this.createKey) ? this.createKey(req) : createKey(req);
+      const key = this.#createCacheKey(req);
 
       try {
-        const cached = await this.#cacheRetrieve(key, req.query);
+        const cached = await this.#cacheRetrieve(key, {});
         if (shouldUseCache(cached)) {
           return callback(null, cached);
         }
@@ -41,7 +49,7 @@ module.exports = function createModel ({ ProviderModel, koop, namespace }, optio
         await this.#before(req);
         const providerGeojson = await this.#getProviderData(req);
         const afterFuncGeojson = await this.#after(req, providerGeojson);
-        const { ttl } = afterFuncGeojson;
+        const { ttl = this.#cacheTtl } = afterFuncGeojson;
         if (ttl) {
           this.#cacheInsert(key, afterFuncGeojson, { ttl });
         }
@@ -53,38 +61,60 @@ module.exports = function createModel ({ ProviderModel, koop, namespace }, optio
 
     // TODO: the pullLayer() and the pullCatalog() are very similar to the pull()
     // function. We may consider to merging them in the future.
-    pullLayer (req, callback) {
-      const key = (this.createKey) ? this.createKey(req) : `${createKey(req)}::layer`;
-      this.#cache.retrieve(key, req.query, (err, cached) => {
-        if (!err && shouldUseCache(cached)) {
-          callback(null, cached);
-        } else if (this.getLayer) {
-          this.getLayer(req, (err, data) => {
-            if (err) return callback(err);
-            callback(null, data);
-            if (data.ttl) this.#cache.insert(key, data, { ttl: data.ttl });
-          });
-        } else {
-          callback(new Error('getLayer() function is not implemented in the provider.'));
+    async pullLayer (req, callback) {
+      if (!this.#getLayer) {
+        callback(new Error(`getLayer() method is not implemented in the ${this.namespace} provider.`));
+      }
+
+      const key = `${this.#createCacheKey(req)}::layer`;
+
+      try {
+        const cached = await this.#cacheRetrieve(key, req.query);
+        if (shouldUseCache(cached)) {
+          return callback(null, cached);
         }
-      });
+      } catch (err) {
+        this.logger.debug(err);
+      }
+
+      try {
+        const data = await this.#getLayer(req);
+        const ttl = data.ttl || this.#cacheTtl;
+        if (ttl) {
+          this.#cacheInsert(key, data, { ttl });
+        }
+        callback(null, data);
+      } catch (err) {
+        callback(err);
+      }
     }
 
-    pullCatalog (req, callback) {
-      const key = (this.createKey) ? this.createKey(req) : `${createKey(req)}::catalog`;
-      this.#cache.retrieve(key, req.query, (err, cached) => {
-        if (!err && shouldUseCache(cached)) {
-          callback(null, cached);
-        } else if (this.getCatalog) {
-          this.getCatalog(req, (err, data) => {
-            if (err) return callback(err);
-            callback(null, data);
-            if (data.ttl) this.#cache.insert(key, data, { ttl: data.ttl });
-          });
-        } else {
-          callback(new Error('getCatalog() function is not implemented in the provider.'));
+    async pullCatalog (req, callback) {
+      if (!this.#getCatalog) {
+        callback(new Error(`getCatalog() method is not implemented in the ${this.namespace} provider.`));
+      }
+
+      const key = `${this.#createCacheKey(req)}::catalog`;
+
+      try {
+        const cached = await this.#cacheRetrieve(key, req.query);
+        if (shouldUseCache(cached)) {
+          return callback(null, cached);
+        }
+      } catch (err) {
+        this.logger.debug(err);
+      }
+
+      try {
+        const data = await this.#getCatalog(req);
+        const ttl = data.ttl || this.#cacheTtl;
+        if (ttl) {
+          this.#cacheInsert(key, data, { ttl });
         }
-      });
+        callback(null, data);
+      } catch (err) {
+        callback(err);
+      }
     }
 
     async pullStream (req) {
@@ -96,6 +126,14 @@ module.exports = function createModel ({ ProviderModel, koop, namespace }, optio
         throw new Error('getStream() function is not implemented in the provider.');
       }
     }
+
+    #createCacheKey (req) {
+      const providerKeyGenerator = this.createCacheKey || this.createKey;
+      if (providerKeyGenerator) {
+        return providerKeyGenerator(req);
+      }
+      return hasher(req.url).toString();
+    }
   }
 
   // Add auth methods if auth plugin registered with Koop
@@ -113,13 +151,7 @@ module.exports = function createModel ({ ProviderModel, koop, namespace }, optio
   return new Model(koop, options);
 };
 
-function createKey (req) {
-  let key = req.url.split('/')[1];
-  if (req.params.host) key = [key, req.params.host].join('::');
-  if (req.params.id) key = [key, req.params.id].join('::');
-  if (req.params.layer) key = [key, req.params.layer].join('::');
-  return key;
-}
+
 
 function shouldUseCache (cacheEntry) {
   // older cache plugins stored expiry time explicitly; all caches should move to returning empty if expired