From 55e7fb8de7e325c2dc7178fdcfe7fead89cb1d77 Mon Sep 17 00:00:00 2001 From: abouthiroppy Date: Wed, 23 Mar 2016 08:27:23 +0900 Subject: [PATCH] Cache latest versions in Redis - issue https://github.com/typings/api/issues/35 - add files - `src/api/routes/support/redis.ts` - `src/worker/jobs/support/redis.ts` - add libraries - `lodash` - 'redis' - define `redisKey` - {source}:{name} acquire package info with precedence from Redis when `version/latest` --- package.json | 2 ++ src/api/routes/support/db.ts | 21 +++++++++--- src/api/routes/support/redis.ts | 18 ++++++++++ src/worker/jobs/support/db.ts | 59 +++++++++++++++++++++----------- src/worker/jobs/support/redis.ts | 18 ++++++++++ src/worker/jobs/typings.ts | 7 ++-- typings.json | 1 + 7 files changed, 100 insertions(+), 26 deletions(-) create mode 100644 src/api/routes/support/redis.ts create mode 100644 src/worker/jobs/support/redis.ts diff --git a/package.json b/package.json index 814f01a..5b8b599 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "invariant": "^2.2.0", "knex": "^0.10.0", "kue": "^0.10.4", + "lodash": "^4.6.1", "minimatch": "^3.0.0", "ms": "^0.7.1", "newrelic": "^1.25.1", @@ -78,6 +79,7 @@ "pad-left": "^2.0.1", "pg": "^4.4.3", "promise-finally": "^2.0.1", + "redis": "^2.5.3", "semver": "^5.1.0", "split": "^1.0.0", "thenify": "^3.1.1", diff --git a/src/api/routes/support/db.ts b/src/api/routes/support/db.ts index 98a75f4..9f7c742 100644 --- a/src/api/routes/support/db.ts +++ b/src/api/routes/support/db.ts @@ -4,6 +4,7 @@ import db from '../../../support/knex' import arrify = require('arrify') import createError = require('http-errors') import { AMBIENT_SOURCES, MAIN_SOURCES, ALL_SOURCES } from '../../../support/constants' +import { getVersion as getVersionInRedis } from './redis' export function getEntry (source: string, name: string) { return db('entries') @@ -121,6 +122,16 @@ export function getMatchingVersions (source: string, name: string, version: stri * Retrieve the latest available version. */ export function getLatest (source: string, name: string, version?: string) { + return getVersionInRedis(source, name).then((data) => { + if (data) { + return JSON.parse(data) + } else { + return getVersionInDB() + } + }).catch(() => { + return getVersionInDB() + }) + // Pick the first result. function result (versions: Version[]) { if (versions.length === 0) { @@ -130,11 +141,13 @@ export function getLatest (source: string, name: string, version?: string) { return Promise.resolve(versions[0]) } - if (version) { - return getMatchingVersions(source, name, version).then(result) - } + function getVersionInDB() { + if (version) { + return getMatchingVersions(source, name, version).then(result) + } - return getVersions(source, name).then(result) + return getVersions(source, name).then(result) + } } export interface SearchOptions { diff --git a/src/api/routes/support/redis.ts b/src/api/routes/support/redis.ts new file mode 100644 index 0000000..8660ce8 --- /dev/null +++ b/src/api/routes/support/redis.ts @@ -0,0 +1,18 @@ +import redis = require('redis') +import Promise = require('any-promise') + +const client = redis.createClient() + +export function getVersion (source: string, name: string): Promise { + const key = `${source}:${name}` + + return new Promise((resolve: Function, reject: Function) => { + client.get(key, (err, data) => { + if (err) { + reject(err) + } else { + resolve(data) + } + }) + }) +} diff --git a/src/worker/jobs/support/db.ts b/src/worker/jobs/support/db.ts index 71fb913..69d49fb 100644 --- a/src/worker/jobs/support/db.ts +++ b/src/worker/jobs/support/db.ts @@ -3,6 +3,10 @@ import Promise = require('any-promise') import semver = require('semver') import pad = require('pad-left') import db from '../../../support/knex' +import { + setVersion as setVersionToRedis, + deleteVersion as deleteVersionsInRedis +} from './redis' export interface UpsertOptions { table: string @@ -12,9 +16,17 @@ export interface UpsertOptions { trx?: knex.Transaction where?: string returning?: string[] + redisKey?: string } export function upsert (options: UpsertOptions): Promise { + if (options.table === 'versions') { + setVersionToRedis({ + redisKey: options.redisKey, + insert: options.insert + }) + } + const insert = db(options.table) .insert(options.insert) .transacting(options.trx) @@ -47,6 +59,7 @@ export interface VersionOptions { version: string compiler?: string location?: string + redisKey?: string } export interface EntryOptions { @@ -105,7 +118,8 @@ export function createVersion (options: VersionOptions): Promise<{ id: string }> updates: ['version', 'location', 'updated', 'compiler', 'deprecated'], conflicts: ['entry_id', 'tag'], returning: ['id'], - where: 'versions.updated <= excluded.updated' + where: 'versions.updated <= excluded.updated', + redisKey: options.redisKey }) .then((row: any) => { if (row) { @@ -123,6 +137,7 @@ export function createVersion (options: VersionOptions): Promise<{ id: string }> export function createEntryAndVersion (options: EntryAndVersionOptions): Promise<{ id: string }> { const { name, source, updated, version, compiler, location } = options + const redisKey = `${source}:${name}` return createEntry(options) .then((row) => { @@ -140,7 +155,8 @@ export function createEntryAndVersion (options: EntryAndVersionOptions): Promise updated, version, compiler, - location + location, + redisKey }) }) } @@ -154,25 +170,28 @@ export interface VersionsOptions { export function deleteVersions (options: VersionsOptions) { const { name, source, updated } = options - return db.transaction(trx => { - return db('entries') - .transacting(trx) - .first('id') - .where({ name, source }) - .then((row) => { - if (row == null) { - return - } - - return db('versions') - .transacting(trx) - .update({ deprecated: updated }) - .where('entry_id', '=', row.id) - .andWhere('updated', '<', updated) - .returning('id') + return deleteVersionsInRedis(source, name) + .then(() => { + return db.transaction(trx => { + return db('entries') + .transacting(trx) + .first('id') + .where({ name, source }) + .then((row) => { + if (row == null) { + return + } + + return db('versions') + .transacting(trx) + .update({ deprecated: updated }) + .where('entry_id', '=', row.id) + .andWhere('updated', '<', updated) + .returning('id') + }) + .then(trx.commit) + .catch(trx.rollback) }) - .then(trx.commit) - .catch(trx.rollback) }) } diff --git a/src/worker/jobs/support/redis.ts b/src/worker/jobs/support/redis.ts new file mode 100644 index 0000000..3d7811d --- /dev/null +++ b/src/worker/jobs/support/redis.ts @@ -0,0 +1,18 @@ +import _ = require('lodash') +import redis = require('redis') +import Promise = require('any-promise') + +const client = redis.createClient() + +export function setVersion (options: any) { + const option = _.clone(options.insert) + delete option.entry_id + + client.set(options.redisKey, JSON.stringify(option)) +} + +export function deleteVersion (source: string, name: string) { + return new Promise((resolve) => { + client.del(`${source}:${name}`, () => resolve()) + }) +} diff --git a/src/worker/jobs/typings.ts b/src/worker/jobs/typings.ts index d6bfa18..165bd62 100644 --- a/src/worker/jobs/typings.ts +++ b/src/worker/jobs/typings.ts @@ -118,13 +118,15 @@ export function indexTypingsFileChange (job: kue.Job) { const data: VersionOptions[] = Object.keys(versions).map((version) => { const value = versions[version] + const redisKey = `${source}:${name}` if (typeof value === 'string') { return { version, entryId: row.id, location: value, - updated + updated, + redisKey } } @@ -134,7 +136,8 @@ export function indexTypingsFileChange (job: kue.Job) { compiler: value.compiler, location: value.location, description: value.description, - updated + updated, + redisKey } }) diff --git a/typings.json b/typings.json index 97c5be6..fb45c4d 100644 --- a/typings.json +++ b/typings.json @@ -20,6 +20,7 @@ "es6-promise": "github:typings/typed-es6-promise#94aac67ef7a14a8de8e9e1d3c1f9a26caa0d9fb1", "http-errors": "github:typed-typings/npm-http-errors#6fa04b5f2cc2560f4547820f5d138dbb5789345d", "invariant": "github:typings/typed-invariant#58403cee078ebef52112c4227c4d23a97821ef5c", + "lodash": "registry:npm/lodash#4.0.0+20160305082308", "minimatch": "github:typed-typings/npm-minimatch#74f47de8acb42d668491987fc6bc144e7d9aa891", "ms": "github:typings/typed-ms#f40c81c7f45bc35e970de851117c29fc959220b2", "node-uuid": "github:rapropos/typed-node-uuid#a2ad36f2802416729eaad89559d76e0b03ba673c",