From db61e2283bfcaefb4fe6bc8620ed1cfe81ed0bc6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 27 Nov 2023 10:41:59 -0800 Subject: [PATCH 1/3] Remove .github folder from each subdirectory (#4480) Signed-off-by: Kevin Su Co-authored-by: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> --- datacatalog/.github/config.yml | 15 -- datacatalog/.github/workflows/checks.yml | 70 ----- .../.github/workflows/upgrade_automation.yml | 21 -- flyteadmin/.github/workflows/checks.yml | 90 ------- .../.github/workflows/upgrade_automation.yml | 20 -- flytecopilot/.github/config.yml | 15 -- flytecopilot/.github/workflows/checks.yml | 84 ------ .../.github/workflows/upgrade_automation.yml | 20 -- flyteidl/.github/config.yml | 16 -- .../workflows/boilerplate-automation.yml | 36 --- flyteidl/.github/workflows/buf_publish.yaml | 16 -- flyteidl/.github/workflows/master.yml | 46 ---- flyteidl/.github/workflows/npmpublish.yaml | 26 -- flyteidl/.github/workflows/pythonpublish.yaml | 32 --- .../.github/workflows/release-automation.yaml | 20 -- flyteidl/.github/workflows/verification.yml | 252 ------------------ .../workflows/boilerplate-automation.yml | 37 --- flyteplugins/.github/workflows/master.yml | 76 ------ .../.github/workflows/pull_request.yml | 46 ---- .../.github/workflows/upgrade_automation.yml | 20 -- flytepropeller/.github/workflows/checks.yml | 86 ------ .../.github/workflows/codeql-analysis.yml | 78 ------ flytepropeller/.github/workflows/krew.yaml | 19 -- .../.github/workflows/upgrade_automation.yml | 21 -- flytestdlib/.github/workflows/master.yml | 72 ----- .../.github/workflows/pull_request.yml | 45 ---- .../.github/workflows/upgrade_automation.yml | 12 - 27 files changed, 1291 deletions(-) delete mode 100644 datacatalog/.github/config.yml delete mode 100644 datacatalog/.github/workflows/checks.yml delete mode 100644 datacatalog/.github/workflows/upgrade_automation.yml delete mode 100644 flyteadmin/.github/workflows/checks.yml delete mode 100644 flyteadmin/.github/workflows/upgrade_automation.yml delete mode 100644 flytecopilot/.github/config.yml delete mode 100644 flytecopilot/.github/workflows/checks.yml delete mode 100644 flytecopilot/.github/workflows/upgrade_automation.yml delete mode 100644 flyteidl/.github/config.yml delete mode 100644 flyteidl/.github/workflows/boilerplate-automation.yml delete mode 100644 flyteidl/.github/workflows/buf_publish.yaml delete mode 100644 flyteidl/.github/workflows/master.yml delete mode 100644 flyteidl/.github/workflows/npmpublish.yaml delete mode 100644 flyteidl/.github/workflows/pythonpublish.yaml delete mode 100644 flyteidl/.github/workflows/release-automation.yaml delete mode 100644 flyteidl/.github/workflows/verification.yml delete mode 100644 flyteplugins/.github/workflows/boilerplate-automation.yml delete mode 100644 flyteplugins/.github/workflows/master.yml delete mode 100644 flyteplugins/.github/workflows/pull_request.yml delete mode 100644 flyteplugins/.github/workflows/upgrade_automation.yml delete mode 100644 flytepropeller/.github/workflows/checks.yml delete mode 100644 flytepropeller/.github/workflows/codeql-analysis.yml delete mode 100644 flytepropeller/.github/workflows/krew.yaml delete mode 100644 flytepropeller/.github/workflows/upgrade_automation.yml delete mode 100644 flytestdlib/.github/workflows/master.yml delete mode 100644 flytestdlib/.github/workflows/pull_request.yml delete mode 100644 flytestdlib/.github/workflows/upgrade_automation.yml diff --git a/datacatalog/.github/config.yml b/datacatalog/.github/config.yml deleted file mode 100644 index f99bcd78f1..0000000000 --- a/datacatalog/.github/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: > - Thank you for opening this pull request! πŸ™Œ - - These tips will help get your PR across the finish line: - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! πŸŽ‰ - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! πŸ›  diff --git a/datacatalog/.github/workflows/checks.yml b/datacatalog/.github/workflows/checks.yml deleted file mode 100644 index 82efe6c675..0000000000 --- a/datacatalog/.github/workflows/checks.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Datacatalog Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: "1.19" - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker_build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - generate: - name: Check Go Gennerate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: "1.19" - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ docker_build, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - with: - go-version: "1.19" - - push_docker_image: - name: Build & Push Datacatalog Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/datacatalog/.github/workflows/upgrade_automation.yml b/datacatalog/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 02cf6cde61..0000000000 --- a/datacatalog/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteadmin/.github/workflows/checks.yml b/flyteadmin/.github/workflows/checks.yml deleted file mode 100644 index 6451811c53..0000000000 --- a/flyteadmin/.github/workflows/checks.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Flyteadmin Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker-build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - endtoend: - name: End2End Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/end2end.yml@master - with: - priorities: "P0" - cache_key: ${{ needs.docker-build.outputs.cache_key }} - - integration: - name: Integration Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/integration.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - - generate: - name: Check Go Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ endtoend, integration, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flyteadmin Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - - push-docker-image-flytescheduler: - name: Build & Push Flytescheduler Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - push: true - repository: flyteorg/flytescheduler - dockerfile: scheduler.Dockerfile - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/flyteadmin/.github/workflows/upgrade_automation.yml b/flyteadmin/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c2..0000000000 --- a/flyteadmin/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytecopilot/.github/config.yml b/flytecopilot/.github/config.yml deleted file mode 100644 index f99bcd78f1..0000000000 --- a/flytecopilot/.github/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: > - Thank you for opening this pull request! πŸ™Œ - - These tips will help get your PR across the finish line: - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! πŸŽ‰ - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! πŸ›  diff --git a/flytecopilot/.github/workflows/checks.yml b/flytecopilot/.github/workflows/checks.yml deleted file mode 100644 index a7e624f772..0000000000 --- a/flytecopilot/.github/workflows/checks.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Flyte copilot Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - dry_run: - name: Dry run goreleaser - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser dry run - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: --snapshot --skip-publish --rm-dist - - build_docker_image: - name: Build & Push Flyte copilot Image - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: v1 - dockerfile: Dockerfile - push: false - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - - generate: - name: Check Go Gennerate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ lint, tests, generate, dry_run, build_docker_image ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flyte copilot Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} diff --git a/flytecopilot/.github/workflows/upgrade_automation.yml b/flytecopilot/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c2..0000000000 --- a/flytecopilot/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteidl/.github/config.yml b/flyteidl/.github/config.yml deleted file mode 100644 index 73da252e52..0000000000 --- a/flyteidl/.github/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: | - Thank you for opening this pull request! πŸ™Œ - - These tips will help get your PR across the finish line: - - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! πŸŽ‰ - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! πŸ›  diff --git a/flyteidl/.github/workflows/boilerplate-automation.yml b/flyteidl/.github/workflows/boilerplate-automation.yml deleted file mode 100644 index 7a257af06a..0000000000 --- a/flyteidl/.github/workflows/boilerplate-automation.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Update Boilerplate Automation -on: - workflow_dispatch: - -jobs: - update-boilerplate: - name: Update Boilerplate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Update Boilerplate - run: | - make update_boilerplate - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.FLYTE_BOT_PAT }} - commit-message: Update Boilerplate - committer: Flyte-Bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: flyte-bot-update-boilerplate - delete-branch: true - title: 'Update Boilerplate' - body: | - Update Boilerplate - - Auto-generated by [flyte-bot] - labels: | - boilerplate - team-reviewers: | - owners - maintainers - draft: false \ No newline at end of file diff --git a/flyteidl/.github/workflows/buf_publish.yaml b/flyteidl/.github/workflows/buf_publish.yaml deleted file mode 100644 index 17ecac2423..0000000000 --- a/flyteidl/.github/workflows/buf_publish.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Publish Buf Package - -on: - release: - types: [created] - -jobs: - buf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: bufbuild/buf-setup-action@v1 - - uses: bufbuild/buf-push-action@v1 - with: - buf_token: ${{ secrets.BUF_TOKEN }} - input: 'protos' diff --git a/flyteidl/.github/workflows/master.yml b/flyteidl/.github/workflows/master.yml deleted file mode 100644 index 2fc19bbdab..0000000000 --- a/flyteidl/.github/workflows/master.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - if: github.repository == 'flyteorg/flyteidl' - name: Bump Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump-version - uses: anothrNick/github-tag-action@1.36.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - - goreleaser: - if: github.repository == 'flyteorg/flyteidl' - name: Goreleaser - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteidl/.github/workflows/npmpublish.yaml b/flyteidl/.github/workflows/npmpublish.yaml deleted file mode 100644 index 1101ca0a20..0000000000 --- a/flyteidl/.github/workflows/npmpublish.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish NPM Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # Setup .npmrc file to publish to npm - - uses: actions/setup-node@v1 - with: - node-version: "12.x" - registry-url: "https://registry.npmjs.org" - - name: Autobump version - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - VERSION=$VERSION make update_npmversion - shell: bash - - run: npm install - - run: npm publish --access=public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/flyteidl/.github/workflows/pythonpublish.yaml b/flyteidl/.github/workflows/pythonpublish.yaml deleted file mode 100644 index b418b35c1b..0000000000 --- a/flyteidl/.github/workflows/pythonpublish.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Upload PyPi Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Autobump version - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - VERSION=$VERSION make update_pyversion - shell: bash - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/flyteidl/.github/workflows/release-automation.yaml b/flyteidl/.github/workflows/release-automation.yaml deleted file mode 100644 index b0388223da..0000000000 --- a/flyteidl/.github/workflows/release-automation.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Invoke Workflow - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - strategy: - matrix: - COMPONENT: [flyteadmin, flytepropeller, flyteconsole, flytecopilot, flyteplugins, datacatalog, flytectl] - fail-fast: false - steps: - - uses: actions/checkout@v2 - - name: Update flyteidl version - run: | - WORKFLOW_ID=$(curl -sS https://api.github.com/repos/flyteorg/${{matrix.COMPONENT}}/actions/workflows | jq '.workflows[] | select(.path == ".github/workflows/upgrade_automation.yml") | {id}' | jq .id) - curl -X POST -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/flyteorg/${{matrix.COMPONENT}}/actions/workflows/$WORKFLOW_ID/dispatches -H "Authorization: token ${{ secrets.FLYTE_BOT_PAT }}" -d '{"ref":"master", "inputs": {"component": "flyteidl"}}' - shell: bash diff --git a/flyteidl/.github/workflows/verification.yml b/flyteidl/.github/workflows/verification.yml deleted file mode 100644 index bb8270a475..0000000000 --- a/flyteidl/.github/workflows/verification.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: Verification Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - pull_request: - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: 1.19 - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - with: - go-version: 1.19 - generate-protos: - runs-on: ubuntu-latest - name: Generate Protos - container: - image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - # GO Protos - - name: Proto-Service-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service --with_gateway -l go --go_source_relative - - name: Proto-Admin-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Core-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Event-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Plugins-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l go --go_source_relative --validate_out - - name: Proto-Datacatalog-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out - - # Python is going to be generated using buf - # Python - # - name: Proto-Service-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python - # - name: Proto-Admin-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python - # - name: Proto-Core-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python - # - name: Proto-Event-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python - # - name: Proto-Plugins-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python - # - name: Proto-Datacatalog-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python - - # Cpp - - name: Proto-Service-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l cpp - - name: Proto-Admin-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l cpp - - name: Proto-Core-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l cpp - - name: Proto-Event-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l cpp - - name: Proto-Plugins-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l cpp - - name: Proto-Datacatalog-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l cpp - - # Java - - name: Proto-Service-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l java - - name: Proto-Admin-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l java - - name: Proto-Core-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l java - - name: Proto-Event-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l java - - name: Proto-Plugins-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l java - - name: Proto-Datacatalog-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l java - - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - generate-protos-buf: - name: Generate python protos using buf - runs-on: ubuntu-latest - needs: [ generate-protos ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: bufbuild/buf-setup-action@v1 - - name: "Clean Generated" - run: rm -rf ./gen - - name: "Run buf generate" - run: buf generate - - name: "Include __init__.py files" - run: find gen/pb_python -type d -exec touch {}/__init__.py \; - - uses: actions/upload-artifact@master - with: - name: generated-buf - path: ./gen - - generate-admin-swagger-code: - runs-on: ubuntu-latest - name: Generate Admin Swagger Code - needs: - - generate-protos - - generate-protos-buf - container: - image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2 - options: --cpus 1 - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - uses: actions/download-artifact@master - with: - name: generated-buf - path: ./gen - - run: ls gen/pb-go/flyteidl/service/ - - run: cp -R gen/pb-go/flyteidl/service/ ../ - - run: ls ../service/ - # Open API 2 - - name: OpenAPI-Binary - run: go-bindata -pkg service -o ../service/openapi.go -prefix ../service/ -modtime 1562572800 ../service/admin.swagger.json - - run: rm -rf gen/pb-go/flyteidl/service - - run: mkdir -p gen/pb-go/flyteidl/service - - run: cp -R ../service gen/pb-go/flyteidl/ - - run: ls gen/pb-go/flyteidl/ - - run: ls gen/pb-go/flyteidl/service/ - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - generate-js-code: - runs-on: ubuntu-latest - name: Generate JS Code - needs: [ generate-admin-swagger-code ] - container: - image: schottra/docker-protobufjs:v0.0.2 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - run: mkdir -p /defs - - run: mkdir -p /gen - - run: cp -R . /defs - - run: node /code/generate-protobuf.js --module-name flyteidl -d protos/flyteidl/core -d protos/flyteidl/event -d protos/flyteidl/admin -d protos/flyteidl/service -- --root flyteidl -t static-module -w default --no-delimited --force-long --no-convert -p /defs/protos - - run: ls /defs/gen/ - - uses: actions/upload-artifact@master - with: - name: generated - path: /defs/gen - - generate-swagger-code: - runs-on: ubuntu-latest - name: Generate Swagger Code - needs: [ generate-js-code ] - container: - image: docker.io/lyft/swagger-codegen-cli:dc5ce6ec6d7d4d980fa882d6bd13a83cba3be3c3 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l go -o ./gen/pb-go/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin - - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l python -o ./gen/pb_python/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - test-proto-changes: - runs-on: ubuntu-latest - name: Test Proto Changes - needs: [ generate-swagger-code ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - name: Test - run: ./scripts/test_diff.sh - - generate: - name: Check Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: 1.19 diff --git a/flyteplugins/.github/workflows/boilerplate-automation.yml b/flyteplugins/.github/workflows/boilerplate-automation.yml deleted file mode 100644 index d4a76239be..0000000000 --- a/flyteplugins/.github/workflows/boilerplate-automation.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Update Boilerplate Automation -on: - workflow_dispatch: - -jobs: - update-boilerplate: - name: Update Boilerplate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Update Boilerplate - run: | - make update_boilerplate - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.FLYTE_BOT_PAT }} - commit-message: Update Boilerplate - committer: Flyte-Bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: flyte-bot-update-boilerplate - delete-branch: true - title: 'Update Boilerplate' - body: | - Update Boilerplate - - Auto-generated by [flyte-bot] - labels: | - boilerplate - team-reviewers: | - owners - maintainers - draft: false - diff --git a/flyteplugins/.github/workflows/master.yml b/flyteplugins/.github/workflows/master.yml deleted file mode 100644 index 0dc07253b4..0000000000 --- a/flyteplugins/.github/workflows/master.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - name: Bump Version - if: github.event.commits[0].author.name != 'goreleaserbot' - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Bump version and push tag - id: bump-version - uses: anothrNick/github-tag-action@1.36.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - goreleaser: - name: Goreleaser - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} - tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - lint: - name: Run lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint diff --git a/flyteplugins/.github/workflows/pull_request.yml b/flyteplugins/.github/workflows/pull_request.yml deleted file mode 100644 index 52f6e89c30..0000000000 --- a/flyteplugins/.github/workflows/pull_request.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Pull Request - -on: pull_request - -jobs: - tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - lint: - name: Run lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint - - generate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Go generate and diff - run: DELTA_CHECK=true make generate diff --git a/flyteplugins/.github/workflows/upgrade_automation.yml b/flyteplugins/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c2..0000000000 --- a/flyteplugins/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytepropeller/.github/workflows/checks.yml b/flytepropeller/.github/workflows/checks.yml deleted file mode 100644 index acb7bde43d..0000000000 --- a/flytepropeller/.github/workflows/checks.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Flytepropeller Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: "1.19" - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker-build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - endtoend: - name: End2End Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/end2end.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - priorities: "P0" - - integration: - name: Integration Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/integration.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - go-version: "1.19" - - generate: - name: Check Go Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: "1.19" - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ endtoend, integration, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flytepropeller Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/flytepropeller/.github/workflows/codeql-analysis.yml b/flytepropeller/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 4ddece498b..0000000000 --- a/flytepropeller/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,78 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '23 3 * * 6' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - uses: actions/setup-go@v3 - with: - go-version: '1.19' - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # πŸ“š https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/flytepropeller/.github/workflows/krew.yaml b/flytepropeller/.github/workflows/krew.yaml deleted file mode 100644 index a8742237af..0000000000 --- a/flytepropeller/.github/workflows/krew.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: releaser -on: - release: - types: - - published - -jobs: - release-cli-via-krew: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Unshallow - run: git fetch --prune --unshallow - - - name: Update new version in krew-index - if: "!github.event.release.prerelease" - uses: rajatjindal/krew-release-bot@v0.0.38 diff --git a/flytepropeller/.github/workflows/upgrade_automation.yml b/flytepropeller/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 6d1ea41639..0000000000 --- a/flytepropeller/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - go-version: 1.19 - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytestdlib/.github/workflows/master.yml b/flytestdlib/.github/workflows/master.yml deleted file mode 100644 index 3c25d004e4..0000000000 --- a/flytestdlib/.github/workflows/master.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - name: Bump Version - if: github.event.commits[0].author.name != 'goreleaserbot' - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump-version - uses: flyteorg/github-tag-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - - goreleaser: - name: Goreleaser - runs-on: ubuntu-latest - needs: [ bump-version ] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - tests-lint: - name: Run tests and lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make mod_download && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint - - name: Bench tests - run: make install && make test_benchmark diff --git a/flytestdlib/.github/workflows/pull_request.yml b/flytestdlib/.github/workflows/pull_request.yml deleted file mode 100644 index 39c22142a3..0000000000 --- a/flytestdlib/.github/workflows/pull_request.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Pull Request - -on: pull_request - -jobs: - tests-lint: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make mod_download && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - name: Bench tests - run: make install && make test_benchmark - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: '1.19' - test-generate-integrity: - name: Ensure go generate has run - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Go generate and diff - run: DELTA_CHECK=true make generate - diff --git a/flytestdlib/.github/workflows/upgrade_automation.yml b/flytestdlib/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 94fceadd27..0000000000 --- a/flytestdlib/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Boilerplate Upgrade Automation -on: - workflow_dispatch: - -jobs: - trigger-upgrade: - name: Boilerplate Upgrade Automation - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: boilerplate - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file From 98dd50588c120a134db7f4545ab5b36b2f02ff8a Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Mon, 27 Nov 2023 11:07:53 -0800 Subject: [PATCH 2/3] Fix the loop variable scheduler issue (#4468) * Fix the loop variable scheduler issue Signed-off-by: pmahindrakar-oss * lint Signed-off-by: pmahindrakar-oss * renamed variable Signed-off-by: pmahindrakar-oss * Removed the debuggger file Signed-off-by: pmahindrakar-oss * Added comments Signed-off-by: pmahindrakar-oss --------- Signed-off-by: pmahindrakar-oss --- flyteadmin/scheduler/core/gocron_scheduler.go | 11 ++- .../scheduler/core/gocron_scheduler_test.go | 96 +++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/flyteadmin/scheduler/core/gocron_scheduler.go b/flyteadmin/scheduler/core/gocron_scheduler.go index f569b7c335..c9386746c6 100644 --- a/flyteadmin/scheduler/core/gocron_scheduler.go +++ b/flyteadmin/scheduler/core/gocron_scheduler.go @@ -54,23 +54,26 @@ func (g *GoCronScheduler) GetTimedFuncWithSchedule() TimedFuncWithSchedule { func (g *GoCronScheduler) BootStrapSchedulesFromSnapShot(ctx context.Context, schedules []models.SchedulableEntity, snapshot snapshoter.Snapshot) { for _, s := range schedules { + // Copy the object to save to a new pointer since the pointer is saved later + // Issue due to https://github.com/golang/go/discussions/56010 + schedule := s if *s.Active { funcRef := g.GetTimedFuncWithSchedule() nameOfSchedule := identifier.GetScheduleName(ctx, s) // Initialize the lastExectime as the updatedAt time // Assumption here that schedule was activated and that the 0th execution of the schedule // which will be used as a reference - lastExecTime := &s.UpdatedAt + lastExecTime := &schedule.UpdatedAt fromSnapshot := snapshot.GetLastExecutionTime(nameOfSchedule) // Use the latest time if available in the snapshot - if fromSnapshot != nil && fromSnapshot.After(s.UpdatedAt) { + if fromSnapshot != nil && fromSnapshot.After(schedule.UpdatedAt) { lastExecTime = fromSnapshot } - err := g.ScheduleJob(ctx, s, funcRef, lastExecTime) + err := g.ScheduleJob(ctx, schedule, funcRef, lastExecTime) if err != nil { g.metrics.JobScheduledFailedCounter.Inc() - logger.Errorf(ctx, "unable to register the schedule %+v due to %v", s, err) + logger.Errorf(ctx, "unable to register the schedule %+v due to %v", schedule, err) } } } diff --git a/flyteadmin/scheduler/core/gocron_scheduler_test.go b/flyteadmin/scheduler/core/gocron_scheduler_test.go index d19436ddd5..6732eab09d 100644 --- a/flyteadmin/scheduler/core/gocron_scheduler_test.go +++ b/flyteadmin/scheduler/core/gocron_scheduler_test.go @@ -296,3 +296,99 @@ func TestCatchUpAllSchedule(t *testing.T) { catchupSuccess := g.CatchupAll(ctx, toTime) assert.True(t, catchupSuccess) } + +func TestGoCronScheduler_BootStrapSchedulesFromSnapShot(t *testing.T) { + g := setupWithSchedules(t, "testing", []models.SchedulableEntity{}, true) + True := true + False := false + scheduleActive1 := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(1000, time.October, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "schedule_active_1", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &True, + } + scheduleActive2 := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(2000, time.November, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "schedule_active_2", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &True, + } + scheduleInactive := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(3000, time.December, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "cron3", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &False, + } + + schedule1SnapshotTime := time.Date(5000, time.December, 19, 10, 0, 0, 0, time.UTC) + schedule2SnapshotTime := time.Date(6000, time.December, 19, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + schedules []models.SchedulableEntity + snapshoter snapshoter.Snapshot + expectedCatchUpTimes map[string]*time.Time + }{ + { + name: "two active", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2}, + snapshoter: &snapshoter.SnapshotV1{}, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &scheduleActive1.UpdatedAt, "1420107156943834850": &scheduleActive2.UpdatedAt}, + }, + { + name: "two active one inactive", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2, scheduleInactive}, + snapshoter: &snapshoter.SnapshotV1{}, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &scheduleActive1.UpdatedAt, "1420107156943834850": &scheduleActive2.UpdatedAt}, + }, + { + name: "two active one inactive with snapshot populated", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2, scheduleInactive}, + snapshoter: &snapshoter.SnapshotV1{ + LastTimes: map[string]*time.Time{ + "11407394263542327059": &schedule1SnapshotTime, + "1420107156943834850": &schedule2SnapshotTime, + }, + }, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &schedule1SnapshotTime, "1420107156943834850": &schedule2SnapshotTime}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g.BootStrapSchedulesFromSnapShot(context.Background(), tt.schedules, tt.snapshoter) + g.jobStore.Range(func(key, value interface{}) bool { + jobID := key.(string) + job := value.(*GoCronJob) + if !*job.schedule.Active { + return true + } + assert.Equal(t, job.catchupFromTime, tt.expectedCatchUpTimes[jobID]) + return true + }) + for _, schedule := range tt.schedules { + g.DeScheduleJob(context.TODO(), schedule) + } + }) + } +} From c300fa4ebd363ac28a09e1e5ae64f7c8ebdde666 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 28 Nov 2023 08:59:23 +0800 Subject: [PATCH 3/3] Databricks Plugin Setup Doc Enhancement (#4445) --------- Signed-off-by: Future Outlier Signed-off-by: Kevin Su Co-authored-by: Future Outlier Co-authored-by: Kevin Su --- rsts/deployment/plugins/webapi/databricks.rst | 110 ++++++++++++++---- 1 file changed, 89 insertions(+), 21 deletions(-) diff --git a/rsts/deployment/plugins/webapi/databricks.rst b/rsts/deployment/plugins/webapi/databricks.rst index 671fdb4e18..ee38a481df 100644 --- a/rsts/deployment/plugins/webapi/databricks.rst +++ b/rsts/deployment/plugins/webapi/databricks.rst @@ -42,30 +42,99 @@ Databricks workspace To set up your Databricks account, follow these steps: 1. Create a `Databricks account `__. + +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_workspace.png + :alt: A screenshot of Databricks workspace creation. + 2. Ensure that you have a Databricks workspace up and running. + +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/open_workspace.png + :alt: A screenshot of Databricks workspace. + 3. Generate a `personal access token `__ to be used in the Flyte configuration. - You can find the personal access token in the user settings within the workspace. + You can find the personal access token in the user settings within the workspace. ``User settings`` -> ``Developer`` -> ``Access tokens`` -.. note:: +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/databricks_access_token.png + :alt: A screenshot of access token. + +4. Enable custom containers on your Databricks cluster before you trigger the workflow. + +.. code-block:: bash + + curl -X PATCH -n -H "Authorization: Bearer " \ + https:///api/2.0/workspace-conf \ + -d '{"enableDcs": "true"}' - When testing the Databricks plugin on the demo cluster, create an S3 bucket because the local demo - cluster utilizes MinIO. Follow the `AWS instructions - `__ - to generate access and secret keys, which can be used to access your preferred S3 bucket. +For more detail, check `custom containers `__. -Create an `instance profile +5. Create an `instance profile `__ for the Spark cluster. This profile enables the Spark job to access your data in the S3 bucket. -Please follow all four steps specified in the documentation. -Upload the following entrypoint.py file to either +Create an instance profile using the AWS console (For AWS Users) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. In the AWS console, go to the IAM service. +2. Click the Roles tab in the sidebar. +3. Click Create role. + + a. Under Trusted entity type, select AWS service. + b. Under Use case, select **EC2**. + c. Click Next. + d. At the bottom of the page, click Next. + e. In the Role name field, type a role name. + f. Click Create role. + +4. In the role list, click the **AmazonS3FullAccess** role. +5. Click Create role button. + +In the role summary, copy the Role ARN. + +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/s3_arn.png + :alt: A screenshot of s3 arn. + +Locate the IAM role that created the Databricks deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you don’t know which IAM role created the Databricks deployment, do the following: + +1. As an account admin, log in to the account console. +2. Go to ``Workspaces`` and click your workspace name. +3. In the Credentials box, note the role name at the end of the Role ARN + +For example, in the Role ARN ``arn:aws:iam::123456789123:role/finance-prod``, the role name is finance-prod + +Edit the IAM role that created the Databricks deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +1. In the AWS console, go to the IAM service. +2. Click the Roles tab in the sidebar. +3. Click the role that created the Databricks deployment. +4. On the Permissions tab, click the policy. +5. Click Edit Policy. +6. Append the following block to the end of the Statement array. Ensure that you don’t overwrite any of the existing policy. Replace with the role you created in Configure S3 access with instance profiles. + +.. code-block:: bash + + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam:::role/" + } + + +6. Upload the following ``entrypoint.py`` file to either `DBFS `__ -(the final path can be ``dbfs:///FileStore/tables/entrypoint.py``) or S3. -This file will be executed by the Spark driver node, overriding the default command in the -`dbx `__ job. +(the final path will be ``dbfs:///FileStore/tables/entrypoint.py``) or S3. +This file will be executed by the Spark driver node, overriding the default command of the +`Databricks `__ job. This entrypoint file will + +1. Download the inputs from S3 to the local filesystem. +2. Execute the spark task. +3. Upload the outputs from the local filesystem to S3 for the downstream tasks to consume. + -.. TODO: A quick-and-dirty workaround to resolve https://github.com/flyteorg/flyte/issues/3853 issue is to import pandas. +.. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/flyte/deployment/plugins/databricks/dbfs.png + :alt: A screenshot of dbfs. .. code-block:: python @@ -101,9 +170,7 @@ This file will be executed by the Spark driver node, overriding the default comm def main(): - args = sys.argv - click_ctx = click.Context(click.Command("dummy")) if args[1] == "pyflyte-fast-execute": parser = _fast_execute_task_cmd.make_parser(click_ctx) @@ -122,6 +189,12 @@ This file will be executed by the Spark driver node, overriding the default comm Specify plugin configuration ---------------------------- +.. note:: + + Demo cluster saves the data to minio, but Databricks job saves the data to S3. + Therefore, you need to update the AWS credentials for the single binary deployment, so the pod can + access the S3 bucket that DataBricks job writes to. + .. tabs:: @@ -330,7 +403,6 @@ Add the Databricks access token to FlytePropeller: apiVersion: v1 data: FLYTE_DATABRICKS_API_TOKEN: - client_secret: Zm9vYmFy kind: Secret ... @@ -376,8 +448,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod kubectl get pods -n flyte -.. note:: - - Make sure you enable `custom containers - `__ - on your Databricks cluster before you trigger the workflow. \ No newline at end of file +For databricks plugin on the Flyte cluster, please refer to `Databricks Plugin Example `_