Cache latest versions in Redis

- issue
  typings#35
- add files
  - `src/api/routes/support/redis.ts`
  - `src/worker/jobs/support/redis.ts`
- add libraries
  - `lodash`
  - 'redis'
- define `redisKey`
  - {source}:{name}

acquire package info with precedence from Redis when `version/latest`

package.json

@@ -71,13 +71,15 @@
     "invariant": "^2.2.0",
     "knex": "^0.10.0",
     "kue": "^0.10.4",
+    "lodash": "^4.6.1",
     "minimatch": "^3.0.0",
     "ms": "^0.7.1",
     "newrelic": "^1.25.1",
     "node-uuid": "^1.4.7",
     "pad-left": "^2.0.1",
     "pg": "^4.4.3",
     "promise-finally": "^2.0.1",
+    "redis": "^2.5.3",
     "semver": "^5.1.0",
     "split": "^1.0.0",
     "thenify": "^3.1.1",

src/api/routes/support/db.ts

@@ -4,6 +4,7 @@ import db from '../../../support/knex'
 import arrify = require('arrify')
 import createError = require('http-errors')
 import { AMBIENT_SOURCES, MAIN_SOURCES, ALL_SOURCES } from '../../../support/constants'
+import { getVersion as getVersionInRedis } from './redis'
 
 export function getEntry (source: string, name: string) {
   return db('entries')
@@ -121,6 +122,16 @@ export function getMatchingVersions (source: string, name: string, version: stri
  * Retrieve the latest available version.
  */
 export function getLatest (source: string, name: string, version?: string) {
+  return getVersionInRedis(source, name).then((data) => {
+    if (data) {
+      return JSON.parse(data)
+    } else {
+      return getVersionInDB()
+    }
+  }).catch(() => {
+    return getVersionInDB()
+  })
+
   // Pick the first result.
   function result (versions: Version[]) {
     if (versions.length === 0) {
@@ -130,11 +141,13 @@ export function getLatest (source: string, name: string, version?: string) {
     return Promise.resolve(versions[0])
   }
 
-  if (version) {
-    return getMatchingVersions(source, name, version).then(result)
-  }
+  function getVersionInDB() {
+    if (version) {
+      return getMatchingVersions(source, name, version).then(result)
+    }
 
-  return getVersions(source, name).then(result)
+    return getVersions(source, name).then(result)
+  }
 }
 
 export interface SearchOptions {

src/api/routes/support/redis.ts

@@ -0,0 +1,18 @@
+import redis = require('redis')
+import Promise = require('any-promise')
+
+const client = redis.createClient()
+
+export function getVersion (source: string, name: string): Promise<any> {
+  const key = `${source}:${name}`
+
+  return new Promise((resolve: Function, reject: Function) => {
+    client.get(key, (err, data) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(data)
+      }
+    })
+  })
+}

src/worker/jobs/support/db.ts

@@ -3,6 +3,10 @@ import Promise = require('any-promise')
 import semver = require('semver')
 import pad = require('pad-left')
 import db from '../../../support/knex'
+import {
+  setVersion as setVersionToRedis,
+  deleteVersion as deleteVersionsInRedis
+} from './redis'
 
 export interface UpsertOptions {
   table: string
@@ -12,9 +16,17 @@ export interface UpsertOptions {
   trx?: knex.Transaction
   where?: string
   returning?: string[]
+  redisKey?: string
 }
 
 export function upsert (options: UpsertOptions): Promise<Object> {
+  if (options.table === 'versions') {
+    setVersionToRedis({
+      redisKey: options.redisKey,
+      insert: options.insert
+    })
+  }
+
   const insert = db(options.table)
     .insert(options.insert)
     .transacting(options.trx)
@@ -47,6 +59,7 @@ export interface VersionOptions {
   version: string
   compiler?: string
   location?: string
+  redisKey?: string
 }
 
 export interface EntryOptions {
@@ -105,7 +118,8 @@ export function createVersion (options: VersionOptions): Promise<{ id: string }>
     updates: ['version', 'location', 'updated', 'compiler', 'deprecated'],
     conflicts: ['entry_id', 'tag'],
     returning: ['id'],
-    where: 'versions.updated <= excluded.updated'
+    where: 'versions.updated <= excluded.updated',
+    redisKey: options.redisKey
   })
     .then((row: any) => {
       if (row) {
@@ -123,6 +137,7 @@ export function createVersion (options: VersionOptions): Promise<{ id: string }>
 
 export function createEntryAndVersion (options: EntryAndVersionOptions): Promise<{ id: string }> {
   const { name, source, updated, version, compiler, location } = options
+  const redisKey = `${source}:${name}`
 
   return createEntry(options)
     .then((row) => {
@@ -140,7 +155,8 @@ export function createEntryAndVersion (options: EntryAndVersionOptions): Promise
         updated,
         version,
         compiler,
-        location
+        location,
+        redisKey
       })
     })
 }
@@ -154,25 +170,28 @@ export interface VersionsOptions {
 export function deleteVersions (options: VersionsOptions) {
   const { name, source, updated } = options
 
-  return db.transaction(trx => {
-    return db('entries')
-      .transacting(trx)
-      .first('id')
-      .where({ name, source })
-      .then((row) => {
-        if (row == null) {
-          return
-        }
-
-        return db('versions')
-          .transacting(trx)
-          .update({ deprecated: updated })
-          .where('entry_id', '=', row.id)
-          .andWhere('updated', '<', updated)
-          .returning('id')
+  return deleteVersionsInRedis(source, name)
+    .then(() => {
+      return db.transaction(trx => {
+        return db('entries')
+        .transacting(trx)
+        .first('id')
+        .where({ name, source })
+        .then((row) => {
+          if (row == null) {
+            return
+          }
+
+          return db('versions')
+            .transacting(trx)
+            .update({ deprecated: updated })
+            .where('entry_id', '=', row.id)
+            .andWhere('updated', '<', updated)
+            .returning('id')
+        })
+        .then(trx.commit)
+        .catch(trx.rollback)
       })
-      .then(trx.commit)
-      .catch(trx.rollback)
   })
 }
 

src/worker/jobs/support/redis.ts

@@ -0,0 +1,18 @@
+import _ = require('lodash')
+import redis = require('redis')
+import Promise = require('any-promise')
+
+const client = redis.createClient()
+
+export function setVersion (options: any) {
+  const option = _.clone(options.insert)
+  delete option.entry_id
+
+  client.set(options.redisKey, JSON.stringify(option))
+}
+
+export function deleteVersion (source: string, name: string) {
+  return new Promise((resolve) => {
+    client.del(`${source}:${name}`, () => resolve())
+  })
+}

src/worker/jobs/typings.ts

@@ -118,13 +118,15 @@ export function indexTypingsFileChange (job: kue.Job) {
 
               const data: VersionOptions[] = Object.keys(versions).map((version) => {
                 const value = versions[version]
+                const redisKey = `${source}:${name}`
 
                 if (typeof value === 'string') {
                   return {
                     version,
                     entryId: row.id,
                     location: value,
-                    updated
+                    updated,
+                    redisKey
                   }
                 }
 
@@ -134,7 +136,8 @@ export function indexTypingsFileChange (job: kue.Job) {
                   compiler: value.compiler,
                   location: value.location,
                   description: value.description,
-                  updated
+                  updated,
+                  redisKey
                 }
               })
 

typings.json

@@ -20,6 +20,7 @@
     "es6-promise": "github:typings/typed-es6-promise#94aac67ef7a14a8de8e9e1d3c1f9a26caa0d9fb1",
     "http-errors": "github:typed-typings/npm-http-errors#6fa04b5f2cc2560f4547820f5d138dbb5789345d",
     "invariant": "github:typings/typed-invariant#58403cee078ebef52112c4227c4d23a97821ef5c",
+    "lodash": "registry:npm/lodash#4.0.0+20160305082308",
     "minimatch": "github:typed-typings/npm-minimatch#74f47de8acb42d668491987fc6bc144e7d9aa891",
     "ms": "github:typings/typed-ms#f40c81c7f45bc35e970de851117c29fc959220b2",
     "node-uuid": "github:rapropos/typed-node-uuid#a2ad36f2802416729eaad89559d76e0b03ba673c",