From f7a54473f75369dbd0f27e6d4c97b9c5b787b6ff Mon Sep 17 00:00:00 2001 From: at15 Date: Wed, 7 Sep 2016 17:49:52 -0700 Subject: [PATCH] [config] Support resolve external ref #4 - didn't use library for resolve $ref - all path are transformed to full path in Parser - add a Stack class to deal with relative path in $ref values, ie: in example/data/people.yml $ref : sway.yml means example/data/sway.yml instead of ./sway.yml --- example/data/people.yml | 2 +- lib/config/parser.js | 140 +++++++++++++++++++++++++++++++++++-- lib/util/fs.js | 21 +++++- lib/util/stack.js | 27 +++++++ package.json | 1 + test/config/parser-spec.js | 19 ++++- test/util/fs-spec.js | 17 +++++ test/util/stack-spec.js | 23 ++++++ 8 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 lib/util/stack.js create mode 100644 test/util/fs-spec.js create mode 100644 test/util/stack-spec.js diff --git a/example/data/people.yml b/example/data/people.yml index c7e18ef..9946f4d 100644 --- a/example/data/people.yml +++ b/example/data/people.yml @@ -11,4 +11,4 @@ ComMouse: github: ComMouse # TODO: use json pointer ie: https://github.com/whitlockjc/json-refs sway: - - $ref: 'sway.yml' \ No newline at end of file + $ref: 'sway.yml' \ No newline at end of file diff --git a/lib/config/parser.js b/lib/config/parser.js index 4a51b23..702d89c 100644 --- a/lib/config/parser.js +++ b/lib/config/parser.js @@ -4,9 +4,16 @@ * Enhanced yaml parser that support $ref */ 'use strict'; +const path = require('path'); + +const _ = require('lodash'); +const yaml = require('js-yaml'); + const logger = require('../logger'); const fsUtil = require('../util/fs'); -const yaml = require('js-yaml'); +const Stack = require('../util/stack'); + +// FIXME: use full path for all internal file path, input as relative path is still supported /** * @property files {Array} @@ -17,13 +24,57 @@ class Parser { // all the files this parser has loaded // TODO: may use object for faster lookup this.files = []; - // Set of file -> parsed pairs + // file -> parsed pairs, shallow parsed this.parsed = {}; + // file -> parsed pairs, $ref has been resolved + this.resolved = {}; + this.resolving = new Stack(); this.mainEntryFile = ''; } - /** @param filePath {string} */ + /** + * @param filePath {string} + */ + getParsed(filePath) { + filePath = fsUtil.fullPath(filePath); + if (!_.isEmpty(this.parsed[filePath])) { + return this.parsed[filePath]; + } + logger.warn('file not loaded or empty but get queried', {file: filePath}); + return {}; + } + + /** + * @param filePath {string} + */ + getResolved(filePath) { + filePath = fsUtil.fullPath(filePath); + if (!_.isEmpty(this.resolved[filePath])) { + return this.resolved[filePath]; + } + logger.warn('file not resolved or empty but get queried', {file: filePath}); + return {}; + } + + /** + * + * @param filePath {string} + * @param obj {Object} + */ + setResolved(filePath, obj) { + filePath = fsUtil.fullPath(filePath); + this.resolved[filePath] = obj; + } + + /** + * @param filePath {string} + */ addFile(filePath) { + filePath = fsUtil.fullPath(filePath); + // the file is already loaded + if (!_.isEmpty(this.parsed[filePath])) { + return true; + } if (!fsUtil.fileExists(filePath)) { logger.warn('file not found', {file: filePath}); return false; @@ -33,13 +84,83 @@ class Parser { return true; } + /** + * + * @param filePath {string} + * @returns {boolean} + */ + resolveFile(filePath) { + filePath = fsUtil.fullPath(filePath); + this.resolving.push(filePath); + // add this file if it does not exist + this.addFile(filePath); + let resolved = this.resolveObject(this.getParsed(filePath)); + this.setResolved(filePath, resolved); + // TODO: should assert the pop return same value as we pushed + this.resolving.pop(); + return true; + } + + /** + * + * @param obj {Object} + * @returns {Object} + */ + resolveObject(obj) { + let resolved = this.resolveExternal(obj); + return this.resolveInner(resolved); + } + + /** + * + * @param obj {Object} + * @returns {Object} + */ + resolveExternal(obj) { + // loop and find all the $ref + _.forEach(obj, (value, key) => { + if (!_.has(value, '$ref')) { + return; + } + let ref = value['$ref']; + if (Parser.isInnerRef(ref)) { + return; + } + // now load the file + // TODO: check result + // TODO: need to deal with file path + let dir = fsUtil.dir(this.resolving.top()); + ref = path.resolve(dir, ref); + this.addFile(ref); + // TODO: may use getResolved, should I resolve the external one + // TODO: this object is by reference right? + obj[key] = this.getParsed(ref); + }); + return obj; + } + + /** + * + * @param obj {Object} + * @return {Object} + */ + resolveInner(obj) { + logger.warn('resolve inner is not supported'); + return obj; + } + /** * Use specified file as main config entry, call this before call entry * * @param filePath {string} */ setMainEntry(filePath) { - + if (_.isEmpty(this.parsed[filePath])) { + logger.warn('must add file before set it as main entry', {file: filePath}); + return false; + } + this.mainEntryFile = filePath; + return this.resolveFile(filePath); } /** @@ -54,6 +175,17 @@ class Parser { return {}; } } + + /** + * isInnerRef only checks if the syntax is for inner reference in current document, + * it does NOT check if the reference exists + * + * @param ref {string} + * @returns {boolean} + */ + static isInnerRef(ref) { + return _.startsWith(ref, '#/'); + } } module.exports = Parser; \ No newline at end of file diff --git a/lib/util/fs.js b/lib/util/fs.js index 1d6b2e9..52040d4 100644 --- a/lib/util/fs.js +++ b/lib/util/fs.js @@ -1,12 +1,14 @@ /** * Created by at15 on 2016/9/6. */ -var fs = require('fs'); +const fs = require('fs'); +const path = require('path'); +const _ = require('lodash'); /** @param path {string} */ function fileExists(path) { try { - fs.accessSync(path, fs.constants.F_OK) + fs.accessSync(path, fs.constants.F_OK); return true; } catch (e) { return false; @@ -18,7 +20,20 @@ function readAsString(path) { return fs.readFileSync(path, 'utf8'); } +function fullPath(p) { + if (_.startsWith(p, process.cwd())) { + return p; + } + return path.resolve(process.cwd(), p); +} + +function dir(p) { + return path.dirname(p); +} + module.exports = { fileExists, - readAsString + readAsString, + fullPath, + dir }; \ No newline at end of file diff --git a/lib/util/stack.js b/lib/util/stack.js new file mode 100644 index 0000000..5608be6 --- /dev/null +++ b/lib/util/stack.js @@ -0,0 +1,27 @@ +/** + * Created by at15 on 2016/9/7. + */ +'use strict'; + +class Stack { + constructor() { + this.data = []; + this.len = 0; + } + + push(ele) { + this.data[this.len] = ele; + this.len++; + } + + pop() { + this.len--; + return this.data[this.len]; + } + + top() { + return this.data[this.len - 1]; + } +} + +module.exports = Stack; \ No newline at end of file diff --git a/package.json b/package.json index 9d52e21..4ee3b38 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "index.js", "scripts": { "lint": "eslint .", + "ft": "mocha test/**/*-spec.js", "test": "npm run lint && mocha test/**/*-spec.js", "cover": "istanbul cover node_modules/mocha/bin/_mocha test/**/*-spec.js -- -R spec && istanbul check-coverage", "coveralls": "istanbul cover ./node_modules/mocha/bin/_mocha test/**/*-spec.js --report lcovonly -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage" diff --git a/test/config/parser-spec.js b/test/config/parser-spec.js index fdcd96a..41ef1d9 100644 --- a/test/config/parser-spec.js +++ b/test/config/parser-spec.js @@ -3,6 +3,7 @@ */ 'use strict'; const expect = require('chai').expect; +const fsUtil = require('../../lib/util/fs'); const Parser = require('../../lib/config/parser'); describe('Parser', () => { @@ -15,11 +16,16 @@ describe('Parser', () => { }); }); + it('can check inner reference syntax', ()=> { + expect(Parser.isInnerRef('#/people/sway')).to.eql(true); + expect(Parser.isInnerRef('sway.yml')).to.eql(false); + }); + it('can add file', () => { let parser = new Parser(); let file = 'example/data/people.yml'; parser.addFile(file); - expect(parser.parsed).to.have.ownProperty(file); + expect(parser.parsed).to.have.ownProperty(fsUtil.fullPath(file)); }); it('can add several files', () => { @@ -28,7 +34,16 @@ describe('Parser', () => { let file2 = 'example/data/sway.yml'; parser.addFile(file1); parser.addFile((file2)); - expect(parser.parsed).to.have.all.keys(file1, file2); + expect(parser.parsed).to.have.all.keys(fsUtil.fullPath(file1), fsUtil.fullPath(file2)); }); + // FIXME: relative file path is not handled + it('can resolve external file', () => { + let parser = new Parser(); + let entry = 'example/data/people.yml'; + parser.addFile(entry); + parser.resolveFile(entry); + expect(parser.getResolved(entry).sway).to.eql(Parser.shallowParse('example/data/sway.yml')); + }) + }); \ No newline at end of file diff --git a/test/util/fs-spec.js b/test/util/fs-spec.js new file mode 100644 index 0000000..f137d74 --- /dev/null +++ b/test/util/fs-spec.js @@ -0,0 +1,17 @@ +'use strict'; +const expect = require('chai').expect; +const fsUtil = require('../../lib/util/fs'); + +describe('fs util', ()=> { + it('resolve partial path to current folder', () => { + expect(fsUtil.fullPath('test/util/fs-spec.js')).to.eql(__filename); + }); + + it('leave path with current folder prefix as it is', () => { + expect(fsUtil.fullPath(__filename)).to.eql(__filename); + }); + + it('has shortcut for get file directory', () => { + expect(fsUtil.dir(__filename)).to.eql(__dirname); + }) +}); \ No newline at end of file diff --git a/test/util/stack-spec.js b/test/util/stack-spec.js new file mode 100644 index 0000000..91f5c98 --- /dev/null +++ b/test/util/stack-spec.js @@ -0,0 +1,23 @@ +/** + * Created by at15 on 2016/9/7. + */ +'use strict'; + +const expect = require('chai').expect; +const Stack = require('../../lib/util/stack'); + +describe('Stack', ()=> { + it('can push and pop', () => { + let s = new Stack(); + s.push(1); + expect(s.pop()).to.eqls(1); + }); + + it('use top to return last value', () => { + let s = new Stack(); + s.push(1); + s.push(2); + expect(s.top()).to.eqls(2); + expect(s.top()).to.eqls(2); + }); +}); \ No newline at end of file