-
Notifications
You must be signed in to change notification settings - Fork 209
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
169 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
describe('test the outputgen for node project', () => { | ||
it('#create simple predicting', async () => { | ||
const simple = require('./simple'); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
'use strict'; | ||
|
||
const { pipeline, output } = require('./metadata.json'); | ||
|
||
function _requirePlugin(name) { | ||
let modPath = pipeline[name]; | ||
let isInference = false; | ||
if (name === 'modelDefine') { | ||
try { | ||
require.resolve(`${modPath}/dist/inference.js`); | ||
modPath = `${modPath}/dist/inference.js`; | ||
isInference = true; | ||
} catch (e) { | ||
console.warn(`${modPath}/dist/inference.js doesn't exist, use ${modPath}.`); | ||
} | ||
} | ||
const mod = require(modPath); | ||
const plugin = { | ||
name, | ||
isInference, | ||
module: null | ||
}; | ||
if (mod && typeof mod.default === 'function') { | ||
plugin.module = mod.default; | ||
} else { | ||
plugin.module = mod; | ||
} | ||
return plugin; | ||
} | ||
|
||
// load runtime plugins | ||
const modelDefine = _requirePlugin('modelDefine'); | ||
let dataProcess; | ||
if (typeof pipeline.dataProcess === 'string') { | ||
dataProcess = _requirePlugin('dataProcess'); | ||
} | ||
|
||
let model; | ||
const recoverPath = `${__dirname}/model`; | ||
const dataset = JSON.parse(output.dataset); | ||
|
||
async function loadModel() { | ||
if (model) { | ||
return model; | ||
} | ||
if (!modelDefine.isInference) { | ||
// compatible with the mixed way. | ||
model = await modelDefine.module(null, { | ||
recoverPath, | ||
dataset, | ||
}); | ||
} else { | ||
// modelPath, { labelMaps, feature } | ||
model = await modelDefine.module(recoverPath, dataset); | ||
} | ||
} | ||
|
||
async function predict(data) { | ||
model = await loadModel(); | ||
const sample = { data, label: null }; | ||
if (dataProcess && typeof dataProcess.module === 'function') { | ||
await dataProcess.module(sample, {}, JSON.parse(pipeline.dataProcessParams)); | ||
} | ||
return await model.predict(sample); | ||
}; | ||
|
||
module.exports = predict; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
{ | ||
"cwd": "./", | ||
"pipeline": { | ||
"id": "foobar", | ||
"name": null, | ||
"dataProcess": "./own-data-process", | ||
"dataProcessParams": null, | ||
"modelDefine": "./own-model-define", | ||
"modelDefineParams": "{}", | ||
"createdAt": "2020-07-15T02:30:05.534Z", | ||
"updatedAt": "2020-07-15T02:30:05.534Z" | ||
}, | ||
"output": { | ||
"id": "foobar", | ||
"pipelineId": "foobar", | ||
"specVersion": "1.0.3", | ||
"status": 2, | ||
"currentIndex": -1, | ||
"updatedAt": "2020-07-15T02:30:29.466Z", | ||
"createdAt": "2020-07-15T02:30:05.546Z", | ||
"evaluateMap": "{\"pass\":true,\"accuracy\":0.927570093457944}", | ||
"evaluatePass": true, | ||
"endTime": 1594780229435, | ||
"dataset": "{\"feature\":{\"names\":[\"test\"]}}" | ||
} | ||
} |
Empty file.
Empty file.
Empty file.
26 changes: 26 additions & 0 deletions
26
packages/plugins/model-define/bayesian-model-define/src/inference.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
/** | ||
* @file This is for the plugin to load Bayes Classifier model. | ||
*/ | ||
|
||
import { CsvSample } from '@pipcook/pipcook-core'; | ||
import { join } from 'path'; | ||
import { processPredictData, loadModel } from './script'; | ||
|
||
const boa = require('@pipcook/boa'); | ||
const sys = boa.import('sys'); | ||
|
||
/** | ||
* The inference model. | ||
* @param recoverPath The model path to be recovered. | ||
*/ | ||
export default async function inference (recoverPath: string): Promise<any> { | ||
sys.path.insert(0, join(__dirname, 'assets')); | ||
const model = await loadModel(join(recoverPath, 'model.pkl')); | ||
const featurePath = join(recoverPath, 'feature_words.pkl'); | ||
const stopwordsPath = join(recoverPath, 'stopwords.txt'); | ||
|
||
return async (text: CsvSample) => { | ||
const processData = await processPredictData(text.data, featurePath, stopwordsPath); | ||
return model.predict(processData).toString(); | ||
}; | ||
}; |