refactor: coordinator and gsctl under FLEX architecture (#3481)

- refactor and reorg of gsctl to support FLEX Interactive
- extra coordinator to unify the code paths from gsctl and WebUI by OpenApi specification
- add `setup_sdk.py` under `python/graphscope` directory to build and release FLEX python SDK

## Related issue number

- Fixes #3218
- Fixes #3214

.gitignore

@@ -18,6 +18,9 @@ python/proto
 # dot file
 *.dot
 
+# swp file
+*.swp
+
 # Protobuf GRPC
 *.pb.*
 *_pb2.py
@@ -94,4 +97,4 @@ flex/interactive/data/*
 flex/interactive/logs/*
 flex/interactive/examples/sf0.1-raw/
 flex/interactive/.running
-flex/interactive/.env
+flex/interactive/.env

flex/coordinator/.dockerignore

@@ -0,0 +1,72 @@
+.travis.yaml
+.openapi-generator-ignore
+README.md
+tox.ini
+git_push.sh
+test-requirements.txt
+setup.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.python-version
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints

flex/coordinator/.gitignore

@@ -0,0 +1,66 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.venv/
+.python-version
+.pytest_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints

flex/coordinator/.openapi-generator-ignore

@@ -0,0 +1,26 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md
+
+gs_flex_coordinator/controllers/*
+setup.py

flex/coordinator/.openapi-generator/FILES

@@ -0,0 +1,59 @@
+.dockerignore
+.gitignore
+.travis.yml
+Dockerfile
+README.md
+git_push.sh
+gs_flex_coordinator/__init__.py
+gs_flex_coordinator/__main__.py
+gs_flex_coordinator/encoder.py
+gs_flex_coordinator/models/__init__.py
+gs_flex_coordinator/models/alert_message.py
+gs_flex_coordinator/models/alert_receiver.py
+gs_flex_coordinator/models/alert_rule.py
+gs_flex_coordinator/models/base_model.py
+gs_flex_coordinator/models/column_mapping.py
+gs_flex_coordinator/models/connection.py
+gs_flex_coordinator/models/connection_status.py
+gs_flex_coordinator/models/deployment_info.py
+gs_flex_coordinator/models/deployment_status.py
+gs_flex_coordinator/models/edge_mapping.py
+gs_flex_coordinator/models/edge_mapping_destination_vertex_mappings_inner.py
+gs_flex_coordinator/models/edge_mapping_source_vertex_mappings_inner.py
+gs_flex_coordinator/models/edge_mapping_source_vertex_mappings_inner_column.py
+gs_flex_coordinator/models/edge_mapping_type_triplet.py
+gs_flex_coordinator/models/edge_type.py
+gs_flex_coordinator/models/edge_type_vertex_type_pair_relations_inner.py
+gs_flex_coordinator/models/edge_type_vertex_type_pair_relations_inner_x_csr_params.py
+gs_flex_coordinator/models/graph.py
+gs_flex_coordinator/models/graph_stored_procedures.py
+gs_flex_coordinator/models/groot_edge_type.py
+gs_flex_coordinator/models/groot_edge_type_relations_inner.py
+gs_flex_coordinator/models/groot_graph.py
+gs_flex_coordinator/models/groot_graph_gremlin_interface.py
+gs_flex_coordinator/models/groot_property.py
+gs_flex_coordinator/models/groot_schema.py
+gs_flex_coordinator/models/groot_vertex_type.py
+gs_flex_coordinator/models/model_property.py
+gs_flex_coordinator/models/model_schema.py
+gs_flex_coordinator/models/node_status.py
+gs_flex_coordinator/models/procedure.py
+gs_flex_coordinator/models/procedure_params_inner.py
+gs_flex_coordinator/models/property_property_type.py
+gs_flex_coordinator/models/schema_mapping.py
+gs_flex_coordinator/models/schema_mapping_loading_config.py
+gs_flex_coordinator/models/schema_mapping_loading_config_data_source.py
+gs_flex_coordinator/models/schema_mapping_loading_config_format.py
+gs_flex_coordinator/models/service_status.py
+gs_flex_coordinator/models/service_status_sdk_endpoints.py
+gs_flex_coordinator/models/start_service_request.py
+gs_flex_coordinator/models/update_alert_messages_request.py
+gs_flex_coordinator/models/vertex_mapping.py
+gs_flex_coordinator/models/vertex_type.py
+gs_flex_coordinator/openapi/openapi.yaml
+gs_flex_coordinator/test/__init__.py
+gs_flex_coordinator/typing_utils.py
+gs_flex_coordinator/util.py
+requirements.txt
+test-requirements.txt
+tox.ini

flex/coordinator/.openapi-generator/VERSION

@@ -0,0 +1 @@
+7.2.0

flex/coordinator/.travis.yml

@@ -0,0 +1,14 @@
+# ref: https://docs.travis-ci.com/user/languages/python
+language: python
+python:
+  - "3.2"
+  - "3.3"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7"
+  - "3.8"
+# command to install dependencies
+install: "pip install -r requirements.txt"
+# command to run tests
+script: nosetests

flex/coordinator/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3-alpine
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+COPY requirements.txt /usr/src/app/
+
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+COPY . /usr/src/app
+
+EXPOSE 8080
+
+ENTRYPOINT ["python3"]
+
+CMD ["-m", "gs_flex_coordinator"]

flex/coordinator/README.md

@@ -0,0 +1,49 @@
+# OpenAPI generated server
+
+## Overview
+This server was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using the
+[OpenAPI-Spec](https://openapis.org) from a remote server, you can easily generate a server stub.  This
+is an example of building a OpenAPI-enabled Flask server.
+
+This example uses the [Connexion](https://github.com/zalando/connexion) library on top of Flask.
+
+## Requirements
+Python 3.5.2+
+
+## Usage
+To run the server, please execute the following from the root directory:
+
+```
+pip3 install -r requirements.txt
+python3 -m gs_flex_coordinator
+```
+
+and open your browser to here:
+
+```
+http://localhost:8080/ui/
+```
+
+Your OpenAPI definition lives here:
+
+```
+http://localhost:8080/openapi.json
+```
+
+To launch the integration tests, use tox:
+```
+sudo pip install tox
+tox
+```
+
+## Running with Docker
+
+To run the server on a Docker container, please execute the following from the root directory:
+
+```bash
+# building the image
+docker build -t gs_flex_coordinator .
+
+# starting up a container
+docker run -p 8080:8080 gs_flex_coordinator
+```

flex/coordinator/git_push.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/
+#
+# Usage example: /bin/sh ./git_push.sh wing328 openapi-petstore-perl "minor update" "gitlab.com"
+
+git_user_id=$1
+git_repo_id=$2
+release_note=$3
+git_host=$4
+
+if [ "$git_host" = "" ]; then
+    git_host="github.com"
+    echo "[INFO] No command line input provided. Set \$git_host to $git_host"
+fi
+
+if [ "$git_user_id" = "" ]; then
+    git_user_id="GIT_USER_ID"
+    echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id"
+fi
+
+if [ "$git_repo_id" = "" ]; then
+    git_repo_id="GIT_REPO_ID"
+    echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id"
+fi
+
+if [ "$release_note" = "" ]; then
+    release_note="Minor update"
+    echo "[INFO] No command line input provided. Set \$release_note to $release_note"
+fi
+
+# Initialize the local directory as a Git repository
+git init
+
+# Adds the files in the local repository and stages them for commit.
+git add .
+
+# Commits the tracked changes and prepares them to be pushed to a remote repository.
+git commit -m "$release_note"
+
+# Sets the new remote
+git_remote=$(git remote)
+if [ "$git_remote" = "" ]; then # git remote not defined
+
+    if [ "$GIT_TOKEN" = "" ]; then
+        echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment."
+        git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git
+    else
+        git remote add origin https://${git_user_id}:"${GIT_TOKEN}"@${git_host}/${git_user_id}/${git_repo_id}.git
+    fi
+
+fi
+
+git pull origin master
+
+# Pushes (Forces) the changes in the local repository up to the remote repository
+echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git"
+git push origin master 2>&1 | grep -v 'To https'

flex/coordinator/gs_flex_coordinator/VERSION

@@ -0,0 +1 @@
+../../../VERSION