Merge branch 'master' into update-ray-setup

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,6 +12,10 @@ Happy contributing!
 
 <!-- Example: Closes #31 -->
 
+## Docs link
+
+<!-- Add documentation link built by CI jobs here, and specify the changed place -->
+
 ## Describe your changes
 
 <!-- List all the proposed changes in your PR -->
@@ -29,10 +33,19 @@ Happy contributing!
 - [ ] All new and existing tests passed.
 - [ ] All commits are signed-off.
 
+## Setup Process
+
+<!-- Describe how you set up this PR's environment to help maintainers reproduce your results more easily -->
+
 ## Screenshots
 
 <!-- Add all the screenshots which support your changes -->
 
 ## Note to reviewers
 
 <!-- Add notes to reviewers if applicable -->
+
+## Related PRs
+
+<!-- Add related pull requests for reviewers to check -->
+

.github/workflows/checks.yml

@@ -6,24 +6,9 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
   push:
     branches:
       - master
-    paths:
-      - 'datacatalog/**'
-      - 'flyteadmin/**'
-      - 'flytecopilot/**'
-      - 'flyteidl/**'
-      - 'flyteplugins/**'
-      - 'flytepropeller/**'
-      - 'flytestdlib/**'
 env:
   GO_VERSION: "1.19"
   PRIORITIES: "P0"
@@ -134,47 +119,11 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
 
-  bump-tags:
-    name: Bump git tags
-    # TODO(monorepo): skip this if author is flyte-bot?
-    if: ${{ github.event_name != 'pull_request' }}
-    needs:
-      - integration
-      - lint
-      - unit-tests
-      - generate
-    uses: ./.github/workflows/bump-tags.yml
-    secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  # TODO(monorepo): we are not going to release any binary
-  # goreleaser:
-  #   name: Goreleaser
-  #   needs: [ bump_version ] # Only to ensure it can successfully build
-  #   uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master
-  #   with:
-  #     go-version: "1.19"
-  #   secrets:
-  #     FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
-
-  push_docker_image:
-    name: Build & Push Image
-    needs: [ bump-tags ]
-    strategy:
-      fail-fast: false
-      matrix:
-        component:
-          - datacatalog
-          - flyteadmin
-          - flytecopilot
-          - flytepropeller
-          - flytescheduler
-    uses: ./.github/workflows/publish.yml
+  build_docker_images:
+    name: Build Images
+    uses: ./.github/workflows/publish-images.yml
     with:
-      version: ${{ needs.bump-tags.outputs.next_version }}
-      component: ${{ matrix.component }}
-      dockerfile: Dockerfile.${{ matrix.component }}
-      push: true
+      push: false
     secrets:
       FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
       FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}

.github/workflows/create_release.yml

@@ -7,29 +7,57 @@ on:
         required: true
 
 jobs:
-  bump-version:
-    name: bump-version
+  generate-tags:
+    name: Generate git tags
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.bump-version.outputs.tag }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: "0"
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
+            const components = [
+              "datacatalog",
+              "flyteadmin",
+              "flytecopilot",
+              "flyteidl",
+              "flyteplugins",
+              "flytepropeller",
+              "flytestdlib",
+            ];
+            for (const c of components) {
+              github.rest.git.createRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: `refs/tags/${c}/${{ github.event.inputs.version }}`,
+                sha: context.sha
+              })
+            }
 
-      - name: Bump version and push tag
-        id: bump-version
-        uses: anothrNick/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
-          WITH_V: true
-          CUSTOM_TAG: ${{ github.event.inputs.version }}
-          RELEASE_BRANCHES: master
+  build-docker-images:
+    needs:
+      - generate-tags
+    uses: ./.github/workflows/publish-images.yml
+    with:
+      version: ${{ github.event.inputs.version }}
+      push: true
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
 
   publish-flyte-binary-image:
     name: Publish flyte binary image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - generate-tags
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,24 +73,25 @@ jobs:
 
       - name: Tag image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
           done
 
   publish-flyte-component-image:
     name: Publish flyte component image for the release version
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     strategy:
       matrix:
         component:
           [
+              datacatalog,
+              flyteadmin,
               flyteconsole,
+              flytecopilot,
               flytepropeller,
-              flyteadmin,
-              datacatalog,
               flytescheduler,
-              flytecopilot,
           ]
     steps:
       - name: Checkout
@@ -76,11 +105,7 @@ jobs:
       - name: Get Latest Version of component
         id: set_version
         run: |
-          if [ ${{ matrix.component }} = "flytecopilot" ]; then
-             echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
-          else
-             echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
-          fi
+          echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
         shell: bash
 
       - name: Login to GitHub Container Registry
@@ -92,14 +117,15 @@ jobs:
 
       - name: Tag Image to release version
         run: |
-          for release in latest ${{ needs.bump-version.outputs.version }}; do
+          for release in latest ${{ github.event.inputs.version }}; do
             docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
           done
 
   helm-release:
     name: Flyte helm release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -118,7 +144,7 @@ jobs:
           git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
       - name: Prepare Flyte Helm Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
           REPOSITORY: "https://flyteorg.github.io/flyte"
         run: |
           make prepare_artifacts
@@ -131,7 +157,8 @@ jobs:
   manifest-release:
     name: Flyte manifest release
     runs-on: ubuntu-latest
-    needs: [bump-version]
+    needs:
+      - build-docker-images
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -140,7 +167,7 @@ jobs:
 
       - name: Prepare Flyte Release
         env:
-          VERSION: ${{ needs.bump-version.outputs.version }}
+          VERSION: ${{ github.event.inputs.version }}
         run: |
           make prepare_artifacts
           git stash

.github/workflows/end2end.yml

@@ -87,9 +87,8 @@ jobs:
                   advanced_composition/advanced_composition/dynamics.py \
                   advanced_composition/advanced_composition/map_task.py \
                   advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/custom_objects.py \
-                  data_types_and_io/data_types_and_io/schema.py \
-                  data_types_and_io/data_types_and_io/typed_schema.py ;
+                  data_types_and_io/data_types_and_io/dataclass.py \
+                  data_types_and_io/data_types_and_io/structured_dataset.py ;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \

.github/workflows/flyteidl-checks.yml

@@ -6,13 +6,9 @@ concurrency:
 
 on:
   pull_request:
-    paths:
-      - 'flyteidl/**'
   push:
     branches:
       - master
-    paths:
-      - 'flyteidl/**'
 env:
   GO_VERSION: "1.19"
 jobs:

.github/workflows/flyteidl-release.yml

@@ -5,7 +5,7 @@ on:
     types: [created]
 
 jobs:
-  deploy:
+  deploy-to-pypi:
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -20,11 +20,11 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install setuptools wheel twine
-      - name: Autobump version
+      - name: Set version in python package
         run: |
           # from refs/tags/v1.2.3 get 1.2.3
           VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
-          VERSION=$VERSION make -C flyteidl update_pyversion
+          VERSION=$VERSION make update_pyversion
         shell: bash
       - name: Build and publish
         env:
@@ -33,11 +33,17 @@ jobs:
         run: |
           python setup.py sdist bdist_wheel
           twine upload dist/*
+  deploy-to-npm:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: flyteidl
+    steps:
       - uses: actions/setup-node@v1
         with:
           node-version: "12.x"
           registry-url: "https://registry.npmjs.org"
-      - name: Autobump version
+      - name: Set version in npm package
         run: |
           # from refs/tags/v1.2.3 get 1.2.3
           VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')