repo-sync-2023-12-11T18:42:13+0800

zhangxingmeng · Dec 11, 2023 · 3d0cb29 · 3d0cb29
1 parent ee4d526
commit 3d0cb29
Show file tree

Hide file tree

Showing 62 changed files with 3,397 additions and 244 deletions.
diff --git a/.bazeliskrc b/.bazeliskrc
@@ -0,0 +1,2 @@
+USE_BAZEL_VERSION=5.1.1
+BAZELISK_BASE_URL=https://github.com/bazelbuild/bazel/releases/download
diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml
@@ -24,7 +24,7 @@ jobs:
           name: Cancel build after set time
           background: true
           command: |
-            sleep 7200
+            sleep 3600
             echo "Canceling workflow as too much time has elapsed"
             curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
       - checkout

diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml
@@ -20,7 +20,7 @@ parameters:
 jobs:
   linux_publish:
     docker:
-      - image: registry.hub.docker.com/secretflow/release-ci:0.12
+      - image: registry.hub.docker.com/secretflow/release-ci:0.7
     resource_class: 2xlarge
     parameters:
       python_ver:

diff --git a/.circleci/testpypi-config.yml b/.circleci/testpypi-config.yml
@@ -21,7 +21,7 @@ parameters:
 jobs:
   linux_publish:
     docker:
-      - image: registry.hub.docker.com/secretflow/release-ci:0.12
+      - image: registry.hub.docker.com/secretflow/release-ci:0.7
     resource_class: 2xlarge
     parameters:
       python_ver:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,20 +12,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `Fixed` for any bug fixes.
 `Security` in case of vulnerabilities.
 
-
-## [1.3.0.dev231205] - 2023-12-05
-
-### Changed
-- Add feature selection in all model predict comps.
-
-### Fixed
-- Fix pvalue & more readable assert msg.
+## [1.3.0.dev231211] - 2023-12-11
+### Added
+- Add IO component including read, write and identity.
+- Change groupby component to by-query style.
 
 
 ## [1.3.0.dev231128] - 2023-11-28
 
 ### Added
 - Add secretflow tuner for automl and autoattack.
+- Add IO component including read, write and identity.
+- Change groupby component to by-query style.
 
 ## [1.3.0.dev231120] - 2023-11-20
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -4,7 +4,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "yacl",
-    commit = "8492bdf8c39444262fdb53b5fdf19a0b75f0356a",
+    commit = "6ba8bd5f02035176ec4daaca1c1269195a1b1b4e",
     remote = "https://github.com/secretflow/yacl.git",
 )
 

diff --git a/docker/comp_list.json b/docker/comp_list.json
@@ -244,6 +244,120 @@
         }
       ]
     },
+    {
+      "domain": "io",
+      "name": "identity",
+      "desc": "map any input to output",
+      "version": "0.0.1",
+      "inputs": [
+        {
+          "name": "input_data",
+          "desc": "Input dist data",
+          "types": [
+            "sf.model.ss_glm",
+            "sf.model.ss_glm",
+            "sf.model.sgb",
+            "sf.model.ss_xgb",
+            "sf.model.ss_sgd",
+            "sf.rule.binning",
+            "sf.rule.preprocessing",
+            "sf.read_data"
+          ]
+        }
+      ],
+      "outputs": [
+        {
+          "name": "output_data",
+          "desc": "Output dist data",
+          "types": [
+            "sf.model.ss_glm",
+            "sf.model.ss_glm",
+            "sf.model.sgb",
+            "sf.model.ss_xgb",
+            "sf.model.ss_sgd",
+            "sf.rule.binning",
+            "sf.rule.preprocessing",
+            "sf.read_data"
+          ]
+        }
+      ]
+    },
+    {
+      "domain": "io",
+      "name": "read_data",
+      "desc": "read model or rules from sf cluster",
+      "version": "0.0.1",
+      "inputs": [
+        {
+          "name": "input_dd",
+          "desc": "Input dist data",
+          "types": [
+            "sf.rule.binning",
+            "sf.model.ss_glm"
+          ]
+        }
+      ],
+      "outputs": [
+        {
+          "name": "output_data",
+          "desc": "Output rules or models in DistData.meta",
+          "types": [
+            "sf.read_data"
+          ]
+        }
+      ]
+    },
+    {
+      "domain": "io",
+      "name": "write_data",
+      "desc": "write model or rules back to sf cluster",
+      "version": "0.0.1",
+      "attrs": [
+        {
+          "name": "write_data",
+          "desc": "rule or model protobuf by json formate",
+          "type": "AT_STRING",
+          "atomic": {}
+        },
+        {
+          "name": "write_data_type",
+          "desc": "which rule or model is writing",
+          "type": "AT_STRING",
+          "atomic": {
+            "isOptional": true,
+            "defaultValue": {
+              "s": "sf.rule.binning"
+            },
+            "allowedValues": {
+              "ss": [
+                "sf.rule.binning",
+                "sf.model.ss_glm"
+              ]
+            }
+          }
+        }
+      ],
+      "inputs": [
+        {
+          "name": "input_dd",
+          "desc": "Input dist data. Rule reconstructions may need hidden info in original rule for security considerations.",
+          "types": [
+            "sf.rule.binning",
+            "sf.model.ss_glm"
+          ]
+        }
+      ],
+      "outputs": [
+        {
+          "name": "output_model",
+          "desc": "Output rules or models in sf cluster format",
+          "types": [
+            "sf.rule.binning",
+            "sf.model.ss_glm"
+          ]
+        }
+      ]
+    },
     {
       "domain": "ml.eval",
       "name": "biclassification_eval",
@@ -538,7 +652,9 @@
           "type": "AT_BOOL",
           "atomic": {
             "isOptional": true,
-            "defaultValue": {}
+            "defaultValue": {
+              "b": true
+            }
           }
         }
       ],
@@ -597,7 +713,9 @@
           "type": "AT_BOOL",
           "atomic": {
             "isOptional": true,
-            "defaultValue": {}
+            "defaultValue": {
+              "b": true
+            }
           }
         },
         {
@@ -678,7 +796,9 @@
           "type": "AT_BOOL",
           "atomic": {
             "isOptional": true,
-            "defaultValue": {}
+            "defaultValue": {
+              "b": true
+            }
           }
         },
         {
@@ -746,7 +866,9 @@
           "type": "AT_BOOL",
           "atomic": {
             "isOptional": true,
-            "defaultValue": {}
+            "defaultValue": {
+              "b": true
+            }
           }
         },
         {
@@ -1894,7 +2016,7 @@
           "name": "out_rules",
           "desc": "feature gen rule",
           "types": [
-            "sf.rule.proprocessing"
+            "sf.rule.preprocessing"
           ]
         }
       ]
@@ -1933,7 +2055,7 @@
           "name": "out_rules",
           "desc": "case when substitution rule",
           "types": [
-            "sf.rule.proprocessing"
+            "sf.rule.preprocessing"
           ]
         }
       ]
@@ -1963,13 +2085,13 @@
         },
         {
           "name": "value_type",
-          "desc": "Type of the value to compare with. Must be one of 'FLOAT', 'STR'",
+          "desc": "Type of the value to compare with. Must be one of ['STRING', 'FLOAT']",
           "type": "AT_STRING",
           "atomic": {
             "allowedValues": {
               "ss": [
-                "FLOAT",
-                "STR"
+                "STRING",
+                "FLOAT"
               ]
             }
           }
@@ -1982,7 +2104,7 @@
         },
         {
           "name": "float_epsilon",
-          "desc": "Epsilon value for floating point comparison.",
+          "desc": "Epsilon value for floating point comparison. WARNING: due to floating point representation in computers, set this number slightly larger if you want filter out the values exactly at desired boundary. for example, abs(1.001 - 1.002) is slightly larger than 0.001, and therefore may not be filter out using == and epsilson = 0.001",
           "type": "AT_FLOAT",
           "atomic": {
             "lowerBoundEnabled": true,
@@ -2165,7 +2287,7 @@
           "name": "out_rules",
           "desc": "fill value rule",
           "types": [
-            "sf.rule.proprocessing"
+            "sf.rule.preprocessing"
           ]
         }
       ]
@@ -2222,7 +2344,7 @@
           ],
           "attrs": [
             {
-              "name": "encode_features",
+              "name": "features",
               "desc": "Features to encode."
             }
           ]
@@ -2240,7 +2362,7 @@
           "name": "out_rules",
           "desc": "onehot rule",
           "types": [
-            "sf.rule.proprocessing"
+            "sf.rule.preprocessing"
           ]
         },
         {
@@ -2278,7 +2400,7 @@
         },
         {
           "name": "sort",
-          "desc": "Sort the output.",
+          "desc": "Sort the output. Warning: disable this feature may lead to errors in the following components. DO NOT CHOOSE FALSE if you want to append other components",
           "type": "AT_BOOL",
           "atomic": {
             "isOptional": true,
@@ -2377,7 +2499,7 @@
           "name": "input_rules",
           "desc": "Input preprocessing rules",
           "types": [
-            "sf.rule.proprocessing"
+            "sf.rule.preprocessing"
           ]
         }
       ],
@@ -2489,23 +2611,10 @@
       "version": "0.0.2",
       "attrs": [
         {
-          "name": "aggs",
-          "desc": "What kind of aggregation statistics we want to do? Currently only supports min, max, mean, sum, var, count(number of elements in each group). No repeatitions allowed",
-          "type": "AT_STRINGS",
-          "atomic": {
-            "listMinLengthInclusive": "1",
-            "listMaxLengthInclusive": "6",
-            "allowedValues": {
-              "ss": [
-                "min",
-                "max",
-                "mean",
-                "sum",
-                "var",
-                "count"
-              ]
-            }
-          }
+          "name": "aggregation_config",
+          "desc": "input groupby aggregation config",
+          "type": "AT_CUSTOM_PROTOBUF",
+          "customProtobufCls": "groupby_aggregation_config_pb2.GroupbyAggregationConfig"
         },
         {
           "name": "max_group_size",
@@ -2539,11 +2648,6 @@
               "desc": "by what columns should we group the values",
               "colMinCntInclusive": "1",
               "colMaxCntInclusive": "4"
-            },
-            {
-              "name": "values",
-              "desc": "on which columns should we calculate the statistics, only numerical columns are allowed.",
-              "colMinCntInclusive": "1"
             }
           ]
         }

diff --git a/docker/dev/build.sh b/docker/dev/build.sh
@@ -60,7 +60,8 @@ DOCKER_REG="secretflow"
 IMAGE_TAG=${DOCKER_REG}/sf-dev-anolis8:${VERSION}
 LATEST_TAG=${DOCKER_REG}/sf-dev-anolis8:latest
 echo -e "Building ${GREEN}${IMAGE_TAG}${NO_COLOR}"
-(cd ../.. && rm -rf dist/ && python setup.py bdist_wheel --lite && cp -r dist/* docker/dev/)
+(cd ../.. && rm -rf dist/)
+docker run -it  --rm   --mount type=bind,source="$(pwd)/../../../secretflow",target=/home/admin/src -w /home/admin  --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:0.12 /home/admin/src/docker/dev/entry.sh
 (cd ../ && cp -r release/.nsjail dev/ && cp release/.condarc dev/ && cp *.yml dev/ && cp *.json dev/)
 docker build . -f Dockerfile -t ${IMAGE_TAG} --build-arg config_templates="$(cat config_templates.yml)" --build-arg deploy_templates="$(cat deploy_templates.yml)" --build-arg comp_list="$(cat comp_list.json)" --build-arg translation="$(cat translation.json)"
 echo -e "Finish building ${GREEN}${IMAGE_TAG}${NO_COLOR}"

diff --git a/docker/dev/entry.sh b/docker/dev/entry.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -ex
+
+cp -r src src_copied
+cd src_copied
+
+
+
+
+conda create -n build python=3.8 -y
+conda activate build
+
+python3 setup.py bdist_wheel --lite
+
+cp dist/* ../src/docker/dev/