feat: first commit, publish pipcook

.eslintrc.js

@@ -0,0 +1,15 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  rules: {
+      'no-var': "error",
+      '@typescript-eslint/consistent-type-definitions': [
+          "error",
+          "interface"
+      ]  
+  },
+  "parserOptions": {
+    "ecmaVersion": 6,
+    "sourceType": "module",
+  },
+}

.gitignore

@@ -0,0 +1,13 @@
+node_modules
+/lerna-debug.log
+pipcook-pipeline-*
+.DS_STORE
+.vscode/
+dist
+tsconfig.tsbuildinfo
+samples/
+samples*/
+.pipcook-log/
+doc/
+pipcook_venv
+.temp

.travis.yml

@@ -0,0 +1,14 @@
+sudo: false
+language: node_js
+node_js:
+  - '8'
+  - '10'
+  - '12'
+before_install:
+  - npm i npminstall -g
+install:
+  - npminstall
+script:
+  - npm run ci
+after_script:
+  - npminstall codecov && codecov

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to Contribute
+We are excited that you are interested in contributing to Pipcook. Before submitting your contribution, please take a moment to read through a few small guidelines here
+
+## Reporting Issues
+- We use Github issues to manage our issues. We use status to mark the progress of our issues.
+
+- Try to search for your issue, it may have already been asked, answered or even fixed in the development branch.
+
+- Check if the issue is reproducible with the latest stable version of Pipcook. If you are using a pre-release, please indicate the specific version you are using.
+
+- It is required that you clearly describe the steps necessary to reproduce the issue you are running into. If the issues are asked to provide clear descriptions for more than 5 days, we will close it immediately.
+
+- If your issue is resolved but still open, don’t hesitate to close it. In case you found a solution by yourself, it could be helpful to explain how you fixed it.
+
+## Pull Request Guidelines
+- Only code that's ready for release should be committed to the master branch. All development should be done in dedicated branches.
+- Checkout a **new** topic branch from master branch, and merge back against master branch.
+- If adding new feature:
+  - Add accompanying test case.
+  - Provide convincing reason to add this feature. Ideally you should open a suggestion issue first and have it greenlighted before working on it.
+- If fixing a bug:
+  - If you are resolving a special issue, add `(fix #xxxx[,#xxx])` (#xxxx is the issue id) in your title for a better release log, 
+  - Provide detailed description of the bug in the PR. Live demo preferred.
+  - Add appropriate test coverage if applicable.
+
+
+## Git Commit Specific
+- Your commits message must follow our git commit specific.
+- We will check your commit message, if it does not conform to the specification, the commit will be automatically refused, make sure you have read the specification above.
+
+## Providing Feedback
+We are happy to hear any feedbacks or are delighted to ask any questions. You can join our Dingding Group or ask away on Stack Overflow using the tag Pipcook

Dockerfile

@@ -0,0 +1,29 @@
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
+LABEL version=1.0
+LABEL description="docker image for pipcook runtime"
+
+## will use 7778 port
+EXPOSE 7778
+
+## install pipcook dependencies
+
+RUN apt-get update || :
+RUN apt-get install -y python-software-properties software-properties-common
+RUN echo '\n' | add-apt-repository ppa:chris-lea/node.js
+RUN apt-get update || :
+RUN echo 'Y' | apt-get install nodejs
+RUN apt install nodejs-legacy
+RUN echo 'Y' | apt install npm
+RUN npm install n -g
+RUN apt install wget
+RUN n stable
+RUN PATH="$PATH"
+RUN npm install @pipcook/pipcook-cli -g
+RUN echo 'Y' | apt-get install python3-pip
+RUN mv /usr/bin/python /usr/bin/python.bak
+RUN ln -s /usr/bin/python3.5 /usr/bin/python
+RUN ln -s /usr/bin/pip3 /usr/bin/pip
+# RUN cd /home && pipcook-cli init
+
+ENV TF_FORCE_GPU_ALLOW_GROWTH=true
+ENV LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64

LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -198,4 +199,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

README.md

@@ -1,2 +1,55 @@
-# pipcook
-pipcook
+# Pipcook
+
+With the mission of enabling front-end engineers to utilize the power of machine learning without any prerequisites and the vision to lead front-end technical field to the intelligentization, Pipcook has became the one-step front-end algorithm platform from processing data to deploying models. Pipcook is focused on front-end area and developed from front-end developers view. With the principle of being frendly for web developer, Pipcook will push the whole area forward with the engine of machine learning.
+
+## Quick Start
+
+- Environment: Node.js >= 10.16, Npm >= 6.1
+- Python: python >= 3.6 with correct pip installed (This is required if you want to use pipcook-python-node. For more info, check [here](https://github.com/alibaba/pipcook/wiki/%E6%83%B3%E8%A6%81%E4%BD%BF%E7%94%A8python%EF%BC%9F))
+
+We recommend to install scafford tool for Pipcook to manage Pipcook projects.
+```
+sudo npm install -g pipcook-cli
+```
+
+You can initialize a Pipcook project with just a few commands
+```
+mkdir pipcook-example && cd pipcook-example
+pipcook init
+cd pipcook-example
+```
+
+## Documentation
+
+Please refer to [this link](https://github.com/alibaba/pipcook/wiki/Pipcook-%E6%98%AF%E4%BB%80%E4%B9%88%EF%BC%9F) to check the full documentation
+
+
+## Run your first Pipcook pipeline
+In the initialized folder, we have prepared several samples for you, They are :
+
+- pipeline-mnist-image-classification: pipeline for classific Mnist image classification problem.
+- pipeline-databinding-image-classification: pipeline example to train the iamge classification task which is to classifify [imgcook](https://www.imgcook.com/) databinding pictures.
+- pipeline-object-detection: pipeline example to train object detection task which is for component recognition used by imgcook
+- python-keras: example to use Python Keras library to train deep leraning network in js syntax and runtime
+
+For example, You can quickly run the pipeline to do a mnist image classification. To start the pipeline, just run
+```
+node examples/pipeline-mnist-image-classification.js
+```
+
+## How to contribute
+Please you have installed typescript and lerna. To check if you have installed correcly, run following commands
+```
+lerna -v
+tsc -v
+```
+
+First, clone the repository
+Then, to bootstrap the lerna project (install all dependencies for npm packages), run
+```
+lerna bootstrap
+```
+Please focus on the codes in src directory, each time after you change something, run below command to compile codes
+```
+lerna run compile
+```

appveyor.yml

@@ -0,0 +1,16 @@
+environment:
+  matrix:
+    - nodejs_version: '8'
+    - nodejs_version: '10'
+    - nodejs_version: '12'
+
+install:
+  - ps: Install-Product node $env:nodejs_version
+  - npm i npminstall && node_modules\.bin\npminstall
+
+test_script:
+  - node --version
+  - npm --version
+  - npm run test
+
+build: off

example/pipeline-databinding-image-classification.js

@@ -0,0 +1,62 @@
+/**
+ * This is an example Pipcook file. This pipeline is used to train the image classification task. We have several plugins used in this pipeline:
+ * 
+ * - imageClassDataCollect: used to collect the image classification data. This is a general-purpose image classification collect plugin.
+ *   The user can specify the url of their own dataset instead of ours as long as the dataset conforms to the plugin's expectation. For more information
+ *   of this plugin, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-image-class-data-collect
+ * 
+ * - imageClassDataAccess: used to access the expected image dataset format (PASCOL VOC) and access these images into the pipeline. This is the uniform data
+ *   access plugin for image and make sure that pipcook has uniform dataset that can be published and communicated later in data world. For more information
+ *   about this plugin, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-image-class-data-access
+ * 
+ * - mobileNetLoad: used to load the mobile net specifically. For more information about this plguin ,Please refer to
+ *   https://github.com/alibaba/pipcook/wiki/pipcook-plugins-local-mobilenet-model-load
+ * 
+ * - modelTrainPlugin: uesd to train the model. Currently it supports models of tf.LayersModel. For more information, Please refer to 
+ *   https://github.com/alibaba/pipcook/wiki/pipcook-plugins-model-train
+ * 
+ * - modelEvaluatePlugin: used to evaluate the model. Currently it supports models of tf.LayersModel and classification model which implements
+ *   predict interface we defined for model. For more information, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-model-evaluate
+ * 
+ */
+
+let {DataCollect, DataAccess, ModelLoad, ModelTrain, ModelEvaluate, PipcookRunner} = require('../packages/pipcook-core');
+
+let imageClassDataAccess = require('../packages/pipcook-plugins-image-class-data-access').default;
+let mobileNetLoad = require('../packages/pipcook-plugins-local-mobileNet-model-load').default;
+let modelTrainPlugin = require('../packages/pipcook-plugins-model-train').default;
+let modelEvaluatePlugin = require('../packages/pipcook-plugins-model-evaluate').default;
+let imageClassDataCollect = require('../packages/pipcook-plugins-image-class-data-collect').default
+
+async function startPipeline() {
+  // collect mnist data
+  const dataCollect = DataCollect(imageClassDataCollect, {
+    url: 'http://ai-sample.oss-cn-hangzhou.aliyuncs.com/image_classification/datasets/eCommerceImageClassification.zip'
+  });
+
+  // access mnist data into our specifiction
+  const dataAccess = DataAccess(imageClassDataAccess, {
+    imgSize:[256, 256],
+  });
+
+  // load mobile net model
+  const modelLoad = ModelLoad(mobileNetLoad, {
+    modelName: 'test1'
+  });
+
+  // train the model
+  const modelTrain = ModelTrain(modelTrainPlugin, {
+    epochs: 15
+  });
+
+  // evaluate the model
+  const modelEvaluate = ModelEvaluate(modelEvaluatePlugin);
+
+  const runner = new PipcookRunner('test1', {
+    predictServer: true
+  });
+
+  runner.run([dataCollect, dataAccess, modelLoad, modelTrain, modelEvaluate])
+}
+
+startPipeline();

example/pipeline-load-image-classification.js

@@ -0,0 +1,39 @@
+/**
+ * This is an example Pipcook file. This pipeline is used to load an previous trained model and only run for prediction (deployment)
+ * 
+ * - imageClassDataAccess: used to access the expected image dataset format (PASCOL VOC) and access these images into the pipeline. This is the uniform data
+ *   access plugin for image and make sure that pipcook has uniform dataset that can be published and communicated later in data world. For more information
+ *   about this plugin, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-image-class-data-access
+ * 
+ * - simpleCnnModelLoad: used to load the a 5-conv CNN network specifically. For more information about this plguin ,Please refer to
+ *   https://github.com/alibaba/pipcook/wiki/pipcook-plugins-simple-cnn-model-load
+ * 
+ */
+let {DataAccess, ModelLoad, PipcookRunner} = require('../packages/pipcook-core');
+
+let imageClassDataAccess = require('../packages/pipcook-plugins-image-class-data-access').default;
+let simpleCnnModelLoad = require('../packages/pipcook-plugins-simple-cnn-model-load').default;
+
+
+async function startPipeline() {
+  // access mnist data into our specifiction
+  const dataAccess = DataAccess(imageClassDataAccess, {
+    imgSize:[28, 28],
+  });
+
+  // load mobile net model
+  const modelLoad = ModelLoad(simpleCnnModelLoad, {
+    modelName: 'test1',
+    modelId: '1576029886695-test1'
+  });
+
+  const runner = new PipcookRunner('test1', {
+    onlyPredict: true
+  });
+
+  runner.run([dataAccess, modelLoad])
+}
+
+startPipeline();
+
+

example/pipeline-mnist-image-classfication.js

@@ -0,0 +1,62 @@
+/**
+ * This is an example Pipcook file. This pipeline is used to train the image classification task of mnist image data. 
+ * We have several plugins used in this pipeline:
+ * 
+ * - imageMnistDataCollection: used to collect the mnist data specifically. For more information
+ *   of this plugin, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-image-mnist-data-collect
+ * 
+ * - imageClassDataAccess: used to access the expected image dataset format (PASCOL VOC) and access these images into the pipeline. This is the uniform data
+ *   access plugin for image and make sure that pipcook has uniform dataset that can be published and communicated later in data world. For more information
+ *   about this plugin, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-image-class-data-access
+ * 
+ * - mobileNetLoad: used to load the mobile net specifically. For more information about this plguin ,Please refer to
+ *   https://github.com/alibaba/pipcook/wiki/pipcook-plugins-local-mobilenet-model-load
+ * 
+ * - modelTrainPlugin: uesd to train the model. Currently it supports models of tf.LayersModel. For more information, Please refer to 
+ *   https://github.com/alibaba/pipcook/wiki/pipcook-plugins-model-train
+ * 
+ * - modelEvaluatePlugin: used to evaluate the model. Currently it supports models of tf.LayersModel and classification model which implements
+ *   predict interface we defined for model. For more information, Please refer to https://github.com/alibaba/pipcook/wiki/pipcook-plugins-model-evaluate
+ * 
+ */
+let {DataCollect, DataAccess, ModelLoad, ModelTrain, ModelEvaluate, PipcookRunner} = require('../packages/pipcook-core');
+
+let imageClassDataAccess = require('../packages/pipcook-plugins-image-class-data-access').default;
+let simpleCnnModelLoad = require('../packages/pipcook-plugins-simple-cnn-model-load').default;
+let imageClassModelTrain = require('../packages/pipcook-plugins-model-train').default;
+let classModelEvalute = require('../packages/pipcook-plugins-model-evaluate').default;
+let imageMnistDataCollection = require('../packages/pipcook-plugins-image-mnist-data-collect').default
+
+async function startPipeline() {
+  // collect mnist data
+  const dataCollect = DataCollect(imageMnistDataCollection, {
+    trainingCount:200,
+    testCount: 200
+  });
+
+  // access mnist data into our specifiction
+  const dataAccess = DataAccess(imageClassDataAccess, {
+    imgSize:[28, 28],
+  });
+
+  // load mobile net model
+  const modelLoad = ModelLoad(simpleCnnModelLoad, {
+    modelName: 'test1'
+  });
+
+  // train the model
+  const modelTrain = ModelTrain(imageClassModelTrain, {
+    epochs: 1
+  });
+
+  // evaluate the model
+  const modelEvaluate = ModelEvaluate(classModelEvalute);
+
+  const runner = new PipcookRunner('test1', {
+    predictServer: true
+  });
+
+  runner.run([dataCollect, dataAccess, modelLoad, modelTrain, modelEvaluate])
+}
+
+startPipeline();