Skip to content

Commit

Permalink
fix nits
Browse files Browse the repository at this point in the history
  • Loading branch information
yorkie committed Jul 28, 2020
1 parent db525fd commit f52e50b
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 17 deletions.
62 changes: 45 additions & 17 deletions packages/daemon/templates/node/predict.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,63 @@
const { pipeline, output } = require('./metadata.json');

function _requirePlugin(name) {
const mod = require(pipeline[name]);
let modPath = pipeline[name];
let isInference = false;
if (name === 'modelDefine') {
try {
require.resolve(`${modPath}/dist/inference.js`);
modPath = `${modPath}/dist/inference.js`;
isInference = true;
} catch (e) {
console.warn(`${modPath}/dist/inference.js doesn't exist, use ${modPath}.`);
}
}
const mod = require(modPath);
const plugin = {
name,
isInference,
module: null
};
if (mod && typeof mod.default === 'function') {
return mod.default;
plugin.module = mod.default;
} else {
plugin.module = mod;
}
return mod;
return plugin;
}

const modelDefineModule = _requirePlugin('modelDefine');

let dataProcessModule;
// load runtime plugins
const modelDefine = _requirePlugin('modelDefine');
let dataProcess;
if (typeof pipeline.dataProcess === 'string') {
dataProcessModule = _requirePlugin('dataProcess');
dataProcess = _requirePlugin('dataProcess');
}

let model;
const recoverPath = `${__dirname}/model`;
const dataset = JSON.parse(output.dataset);
const modelDefine = modelDefineModule(null, {
recoverPath: __dirname + '/model',
dataset
});

let model;
async function loadModel() {
if (model) {
return model;
}
if (!modelDefine.isInference) {
// compatible with the mixed way.
model = await modelDefine.module(null, {
recoverPath,
dataset,
});
} else {
// modelPath, { labelMaps, feature }
model = await modelDefine.module(recoverPath, dataset);
}
}

async function predict(data) {
if (!model) {
model = await modelDefine;
}
model = await loadModel();
const sample = { data, label: null };
if (typeof dataProcessModule === 'function') {
await dataProcessModule(sample, {}, JSON.parse(pipeline.dataProcessParams));
if (dataProcess && typeof dataProcess.module === 'function') {
await dataProcess.module(sample, {}, JSON.parse(pipeline.dataProcessParams));
}
return await model.predict(sample);
};
Expand Down
5 changes: 5 additions & 0 deletions packages/daemon/test/outputgen/node.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
describe('test the outputgen for node project', () => {
it('#create simple predicting', async () => {
const simple = require('./simple');
});
});
67 changes: 67 additions & 0 deletions packages/daemon/test/outputgen/simple/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
'use strict';

const { pipeline, output } = require('./metadata.json');

function _requirePlugin(name) {
let modPath = pipeline[name];
let isInference = false;
if (name === 'modelDefine') {
try {
require.resolve(`${modPath}/dist/inference.js`);
modPath = `${modPath}/dist/inference.js`;
isInference = true;
} catch (e) {
console.warn(`${modPath}/dist/inference.js doesn't exist, use ${modPath}.`);
}
}
const mod = require(modPath);
const plugin = {
name,
isInference,
module: null
};
if (mod && typeof mod.default === 'function') {
plugin.module = mod.default;
} else {
plugin.module = mod;
}
return plugin;
}

// load runtime plugins
const modelDefine = _requirePlugin('modelDefine');
let dataProcess;
if (typeof pipeline.dataProcess === 'string') {
dataProcess = _requirePlugin('dataProcess');
}

let model;
const recoverPath = `${__dirname}/model`;
const dataset = JSON.parse(output.dataset);

async function loadModel() {
if (model) {
return model;
}
if (!modelDefine.isInference) {
// compatible with the mixed way.
model = await modelDefine.module(null, {
recoverPath,
dataset,
});
} else {
// modelPath, { labelMaps, feature }
model = await modelDefine.module(recoverPath, dataset);
}
}

async function predict(data) {
model = await loadModel();
const sample = { data, label: null };
if (dataProcess && typeof dataProcess.module === 'function') {
await dataProcess.module(sample, {}, JSON.parse(pipeline.dataProcessParams));
}
return await model.predict(sample);
};

module.exports = predict;
26 changes: 26 additions & 0 deletions packages/daemon/test/outputgen/simple/metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"cwd": "./",
"pipeline": {
"id": "foobar",
"name": null,
"dataProcess": "./own-data-process",
"dataProcessParams": null,
"modelDefine": "./own-model-define",
"modelDefineParams": "{}",
"createdAt": "2020-07-15T02:30:05.534Z",
"updatedAt": "2020-07-15T02:30:05.534Z"
},
"output": {
"id": "foobar",
"pipelineId": "foobar",
"specVersion": "1.0.3",
"status": 2,
"currentIndex": -1,
"updatedAt": "2020-07-15T02:30:29.466Z",
"createdAt": "2020-07-15T02:30:05.546Z",
"evaluateMap": "{\"pass\":true,\"accuracy\":0.927570093457944}",
"evaluatePass": true,
"endTime": 1594780229435,
"dataset": "{\"feature\":{\"names\":[\"test\"]}}"
}
}
Empty file.
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/**
* @file This is for the plugin to load Bayes Classifier model.
*/

import { CsvSample } from '@pipcook/pipcook-core';
import { join } from 'path';
import { processPredictData, loadModel } from './script';

const boa = require('@pipcook/boa');
const sys = boa.import('sys');

/**
* The inference model.
* @param recoverPath The model path to be recovered.
*/
export default async function inference (recoverPath: string): Promise<any> {
sys.path.insert(0, join(__dirname, 'assets'));
const model = await loadModel(join(recoverPath, 'model.pkl'));
const featurePath = join(recoverPath, 'feature_words.pkl');
const stopwordsPath = join(recoverPath, 'stopwords.txt');

return async (text: CsvSample) => {
const processData = await processPredictData(text.data, featurePath, stopwordsPath);
return model.predict(processData).toString();
};
};

0 comments on commit f52e50b

Please sign in to comment.