Repo sync (#532)

# Pull Request ## What problem does this PR solve? Issue Number: Fixed #528 ## Possible side effects? - Performance: - Backward compatibility:
secretflow · Feb 2, 2024 · 15b63f6 · 15b63f6
1 parent a124700
commit 15b63f6
Show file tree

Hide file tree

Showing 103 changed files with 6,451 additions and 3,006 deletions.
diff --git a/.circleci/continue-config.yml b/.circleci/continue-config.yml
@@ -141,11 +141,11 @@ jobs:
           command: git clone https://github.com/secretflow/devtools.git ../devtools
       - run:
           name: "Install homebrew dependencies"
-          command: brew install bazelisk cmake ninja libomp wget go md5sha1sum
+          command: brew install bazelisk cmake ninja libomp wget md5sha1sum
       - run:
           name: "Install Miniconda"
           command: |
-            wget https://repo.anaconda.com/miniconda/Miniconda3-py38_23.1.0-1-MacOSX-arm64.sh -O ~/miniconda.sh
+            wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-MacOSX-arm64.sh -O ~/miniconda.sh
             bash ~/miniconda.sh -b -p $HOME/miniconda
             source $HOME/miniconda/bin/activate
             conda init bash zsh

diff --git a/.circleci/dev-release-config.yml b/.circleci/dev-release-config.yml
@@ -61,7 +61,7 @@ jobs:
       - run:
           name: "build package and publish"
           command: |
-            conda create -n build python=3.9 -y
+            conda create -n build python=3.10 -y
             conda activate build
 
             sh ./build_wheel_entrypoint.sh
@@ -78,19 +78,18 @@ jobs:
       - run:
           name: "Install homebrew dependencies"
           command: |
-            brew install bazelisk cmake ninja libomp wget go
+            brew install bazelisk cmake ninja libomp wget
       - run:
           name: "Install Miniconda"
           command: |
-            wget https://repo.anaconda.com/miniconda/Miniconda3-py38_23.1.0-1-MacOSX-arm64.sh -O ~/miniconda.sh
+            wget https://repo.anaconda.com/miniconda/Miniconda3-py310_23.11.0-2-MacOSX-arm64.sh -O ~/miniconda.sh
             bash ~/miniconda.sh -b -p $HOME/miniconda
             source $HOME/miniconda/bin/activate
             conda init zsh bash
       - run:
           name: "build package and publish"
           command: |
-            conda update -n base -c defaults conda -y
-            conda create -n build python=3.8 -y
+            conda create -n build python=3.10 -y
             conda activate build
 
             sh ./build_wheel_entrypoint.sh

diff --git a/.circleci/release-config.yml b/.circleci/release-config.yml
@@ -145,7 +145,7 @@ workflows:
       - linux_x64_publish:
           matrix:
             parameters:
-              python_ver: ["3.8", "3.9", "3.10", "3.11"]
+              python_ver: ["3.9", "3.10", "3.11"]
           # This is mandatory to trigger a pipeline when pushing a tag
           filters:
             tags:
@@ -161,15 +161,15 @@ workflows:
       - macOS_x64_publish:
           matrix:
             parameters:
-              python_ver: ["3.8", "3.9", "3.10", "3.11"]
+              python_ver: ["3.9", "3.10", "3.11"]
           # This is mandatory to trigger a pipeline when pushing a tag
           filters:
             tags:
               only: /.*/
       - macOS_arm64_publish:
           matrix:
             parameters:
-              python_ver: ["3.8", "3.9", "3.10", "3.11"]
+              python_ver: ["3.9", "3.10", "3.11"]
           # This is mandatory to trigger a pipeline when pushing a tag
           filters:
             tags:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@
 - [Feature] Support more generic Torch model inference
 - [Improvement] Optimize one-time setup for yacl ot
 - [Improvement] Optimize sort performance
+- [Infra] Python 3.8 is no longer suppoted
 
 ## 20240105
 

diff --git a/README.md b/README.md
@@ -50,7 +50,7 @@ Please follow [Installation Guidelines](INSTALLATION.md) to install SPU.
 
 ## Citing SPU
 
-If you think SPU helpful for your research or development, please consider citing our [paper](https://www.usenix.org/conference/atc23/presentation/ma):
+If you think SPU is helpful for your research or development, please consider citing our [paper](https://www.usenix.org/conference/atc23/presentation/ma):
 
 ```text
 @inproceedings {spu,

diff --git a/bazel/patches/xla.patch b/bazel/patches/xla.patch
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -140,8 +140,8 @@ def _com_github_xtensor_xtl():
     )
 
 def _com_github_openxla_xla():
-    OPENXLA_COMMIT = "a0d4632c93451c685ce422d087388a5c624a5ee9"
-    OPENXLA_SHA256 = "3c990717bb64d7e27097f059b727e8e6e131a21c006ac16006f471fb8740b139"
+    OPENXLA_COMMIT = "f0946d01ef4cd9ecb1a27b4adb41ce6bcc846634"
+    OPENXLA_SHA256 = "0af44c6e621da42a87c68746d343b0400ed6ca4b7dc0a7c7efc32f32c83d6be2"
 
     maybe(
         http_archive,
@@ -159,10 +159,6 @@ def _com_github_openxla_xla():
         sha256 = OPENXLA_SHA256,
         strip_prefix = "xla-" + OPENXLA_COMMIT,
         type = ".tar.gz",
-        patch_args = ["-p1"],
-        patches = [
-            "@spulib//bazel:patches/xla.patch",
-        ],
         urls = [
             "https://github.com/openxla/xla/archive/{commit}.tar.gz".format(commit = OPENXLA_COMMIT),
         ],

diff --git a/docs/conf.py b/docs/conf.py
@@ -94,7 +94,7 @@
 # global variables
 extlinks = {
     'spu_doc_host': ('https://www.secretflow.org.cn/docs/spu/en/', 'doc '),
-    'spu_code_host': ('https://github.com/secretflow', 'code '),
+    'spu_code_host': ('https://github.com/secretflow/%s', 'code '),
 }
 
 html_favicon = '_static/favicon.ico'

diff --git a/docs/development/ir_dump.rst b/docs/development/ir_dump.rst
@@ -2,7 +2,7 @@ Dump IR to DAG
 ========================
 
 .. warning::
-    The configuration for dump IR to DAG maybe unstable. Please refer to the :spu_code_host:`spu.proto </spu/blob/main/libspu/spu.proto>` for latest configurations.
+    The configuration for dump IR to DAG maybe unstable. Please refer to the :spu_code_host:`spu.proto <spu/blob/main/libspu/spu.proto>` for latest configurations.
 
 Introduction
 ------------
@@ -16,14 +16,14 @@ With the aid of visualized DAG, the execution logic and required operators will
 
 TL;DR
 ------------
-In case you just want to have a try on obtaining the DAG for executed code, we provide a :spu_code_host:`demo </spu/blob/main/examples/python/ir_dump/ir_dump.py>` that demonstrates the required code modifications to enable dumping the IR of executed code to a custom path.
+In case you just want to have a try on obtaining the DAG for executed code, we provide a :spu_code_host:`demo <spu/blob/main/examples/python/ir_dump/ir_dump.py>` that demonstrates the required code modifications to enable dumping the IR of executed code to a custom path.
 
 For those users who have designated requirements (e.g., dump to txt, dot, html), we recommend to read the following detailed step-by-step elaborations.
 
 
 Configuration
 -------------
-Please first have a look at the :spu_code_host:`spu.proto </spu/blob/main/libspu/spu.proto>` for SPU. To dump the IR, we should modify the compiler options, which is shown in the following code snippet.
+Please first have a look at the :spu_code_host:`spu.proto <spu/blob/main/libspu/spu.proto>` for SPU. To dump the IR, we should modify the compiler options, which is shown in the following code snippet.
 
 .. code-block:: protobuf
     :caption: Compiler Options
@@ -120,7 +120,7 @@ to
     res_spu = ppd.device("SPU")(func, copts=copts)(x_spu, y_spu)
 
 .. Note::
-    Here, `func` is a Python function. Please refer to the :spu_code_host:`demo </spu/blob/main/examples/python/ir_dump/ir_dump.py>` for the context.
+    Here, `func` is a Python function. Please refer to the :spu_code_host:`demo <spu/blob/main/examples/python/ir_dump/ir_dump.py>` for the context.
 
 In the end, you can just run the target code and the output (e.g., DOT) can be found in **your custom path**.
 

diff --git a/docs/reference/complexity.json b/docs/reference/complexity.json
@@ -1 +1 @@
-{"reports":[{"protocol":"Semi2k","entries":[{"kernel":"a2b","latency":"(log(K)+1)*log(N)","comm":"(2*log(K)+1)*2*K*(N-1)*(N-1)"},{"kernel":"b2a","latency":"1","comm":"K*(N-1)"},{"kernel":"a2p","latency":"1","comm":"K*(N-1)"},{"kernel":"b2p","latency":"1","comm":"K*(N-1)"},{"kernel":"add_bb","latency":"log(K)+1","comm":"log(K)*K*2+K"},{"kernel":"add_aa","latency":"0","comm":"0"},{"kernel":"add_ap","latency":"0","comm":"0"},{"kernel":"mul_aa","latency":"1","comm":"K*2*(N-1)"},{"kernel":"mul_ap","latency":"0","comm":"0"},{"kernel":"mmul_aa","latency":"1","comm":"K*2*(N-1)*m*n"},{"kernel":"mmul_ap","latency":"0","comm":"0"},{"kernel":"trunc_a","latency":"1","comm":"K*(N-1)"},{"kernel":"xor_bb","latency":"0","comm":"0"},{"kernel":"xor_bp","latency":"0","comm":"0"},{"kernel":"and_bb","latency":"1","comm":"K*2*(N-1)"},{"kernel":"and_bp","latency":"0","comm":"0"}]},{"protocol":"Aby3","entries":[{"kernel":"a2b","latency":"log(K)+1+1","comm":"log(K)*K+K*2"},{"kernel":"b2a","latency":"TODO","comm":"TODO"},{"kernel":"a2p","latency":"1","comm":"K"},{"kernel":"b2p","latency":"1","comm":"K"},{"kernel":"add_bb","latency":"log(K)+1","comm":"log(K)*K*2+K"},{"kernel":"add_aa","latency":"0","comm":"0"},{"kernel":"add_ap","latency":"0","comm":"0"},{"kernel":"mul_aa","latency":"1","comm":"K"},{"kernel":"mul_ap","latency":"0","comm":"0"},{"kernel":"mmul_aa","latency":"1","comm":"K*m*n"},{"kernel":"mmul_ap","latency":"0","comm":"0"},{"kernel":"trunc_a","latency":"3","comm":"4*K"},{"kernel":"xor_bb","latency":"0","comm":"0"},{"kernel":"xor_bp","latency":"0","comm":"0"},{"kernel":"and_bb","latency":"1","comm":"K"},{"kernel":"and_bp","latency":"0","comm":"0"}]}]}
+{"reports":[{"protocol":"Semi2k","entries":[{"kernel":"a2b","latency":"(log(K)+1)*log(N)","comm":"(2*log(K)+1)*2*K*(N-1)*(N-1)"},{"kernel":"b2a","latency":"1","comm":"K*(N-1)"},{"kernel":"a2p","latency":"1","comm":"K*(N-1)"},{"kernel":"b2p","latency":"1","comm":"K*(N-1)"},{"kernel":"add_aa","latency":"0","comm":"0"},{"kernel":"add_ap","latency":"0","comm":"0"},{"kernel":"mul_aa","latency":"1","comm":"K*2*(N-1)"},{"kernel":"mul_ap","latency":"0","comm":"0"},{"kernel":"mmul_aa","latency":"1","comm":"K*(N-1)*(m+n)*k"},{"kernel":"mmul_ap","latency":"0","comm":"0"},{"kernel":"trunc_a","latency":"1","comm":"K*(N-1)"},{"kernel":"xor_bb","latency":"0","comm":"0"},{"kernel":"xor_bp","latency":"0","comm":"0"},{"kernel":"and_bb","latency":"1","comm":"K*2*(N-1)"},{"kernel":"and_bp","latency":"0","comm":"0"}]},{"protocol":"Aby3","entries":[{"kernel":"a2b","latency":"log(K)+1+1","comm":"2*log(K)*K+K*2"},{"kernel":"b2a","latency":"TODO","comm":"TODO"},{"kernel":"a2p","latency":"1","comm":"K"},{"kernel":"b2p","latency":"1","comm":"K"},{"kernel":"add_aa","latency":"0","comm":"0"},{"kernel":"add_ap","latency":"0","comm":"0"},{"kernel":"mul_aa","latency":"1","comm":"K"},{"kernel":"mul_ap","latency":"0","comm":"0"},{"kernel":"mmul_aa","latency":"1","comm":"K*m*n"},{"kernel":"mmul_ap","latency":"0","comm":"0"},{"kernel":"trunc_a","latency":"3","comm":"4*K"},{"kernel":"xor_bb","latency":"0","comm":"0"},{"kernel":"xor_bp","latency":"0","comm":"0"},{"kernel":"and_bb","latency":"1","comm":"K"},{"kernel":"and_bp","latency":"0","comm":"0"}]}]}
diff --git a/docs/reference/complexity.md b/docs/reference/complexity.md
@@ -5,34 +5,32 @@
 |b2a    |1                 |K\*(N-1)                         |
 |a2p    |1                 |K\*(N-1)                         |
 |b2p    |1                 |K\*(N-1)                         |
-|add_bb |log(K)+1          |log(K)\*K\*2+K                   |
 |add_aa |0                 |0                                |
 |add_ap |0                 |0                                |
 |mul_aa |1                 |K\*2\*(N-1)                      |
 |mul_ap |0                 |0                                |
-|mmul_aa|1                 |K\*2\*(N-1)\*m\*n                |
+|mmul_aa|1                 |K\*(N-1)\*(m+n)\*k               |
 |mmul_ap|0                 |0                                |
 |trunc_a|1                 |K\*(N-1)                         |
 |xor_bb |0                 |0                                |
 |xor_bp |0                 |0                                |
 |and_bb |1                 |K\*2\*(N-1)                      |
 |and_bp |0                 |0                                |
 # Aby3
-|kernel | latency  |     comm     |
-|-------|----------|--------------|
-|a2b    |log(K)+1+1|log(K)\*K+K\*2|
-|b2a    |TODO      |TODO          |
-|a2p    |1         |K             |
-|b2p    |1         |K             |
-|add_bb |log(K)+1  |log(K)\*K\*2+K|
-|add_aa |0         |0             |
-|add_ap |0         |0             |
-|mul_aa |1         |K             |
-|mul_ap |0         |0             |
-|mmul_aa|1         |K\*m\*n       |
-|mmul_ap|0         |0             |
-|trunc_a|3         |4\*K          |
-|xor_bb |0         |0             |
-|xor_bp |0         |0             |
-|and_bb |1         |K             |
-|and_bp |0         |0             |
+|kernel | latency  |      comm       |
+|-------|----------|-----------------|
+|a2b    |log(K)+1+1|2\*log(K)\*K+K\*2|
+|b2a    |TODO      |TODO             |
+|a2p    |1         |K                |
+|b2p    |1         |K                |
+|add_aa |0         |0                |
+|add_ap |0         |0                |
+|mul_aa |1         |K                |
+|mul_ap |0         |0                |
+|mmul_aa|1         |K\*m\*n          |
+|mmul_ap|0         |0                |
+|trunc_a|3         |4\*K             |
+|xor_bb |0         |0                |
+|xor_bp |0         |0                |
+|and_bb |1         |K                |
+|and_bp |0         |0                |
diff --git a/docs/reference/pphlo_doc.rst b/docs/reference/pphlo_doc.rst
@@ -3,7 +3,7 @@ PPHlo API reference
 
 PPHlo is short for (SPU High level ops), it's the assembly language of SPU.
 
-PPHlo is built on `MLIR <https://mlir.llvm.org/>`_ infrastructure, the concrete ops definition could be found :spu_code_host:`here </spu/blob/master/libspu%2Fdialect%2Fpphlo_ops.td>`.
+PPHlo is built on `MLIR <https://mlir.llvm.org/>`_ infrastructure, the concrete ops definition could be found :spu_code_host:`here <spu/blob/main/libspu/dialect/pphlo/ops.td>`.
 
 Op List
 ~~~~~~~