From db61e2283bfcaefb4fe6bc8620ed1cfe81ed0bc6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 27 Nov 2023 10:41:59 -0800 Subject: [PATCH 1/2] Remove .github folder from each subdirectory (#4480) Signed-off-by: Kevin Su Co-authored-by: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> --- datacatalog/.github/config.yml | 15 -- datacatalog/.github/workflows/checks.yml | 70 ----- .../.github/workflows/upgrade_automation.yml | 21 -- flyteadmin/.github/workflows/checks.yml | 90 ------- .../.github/workflows/upgrade_automation.yml | 20 -- flytecopilot/.github/config.yml | 15 -- flytecopilot/.github/workflows/checks.yml | 84 ------ .../.github/workflows/upgrade_automation.yml | 20 -- flyteidl/.github/config.yml | 16 -- .../workflows/boilerplate-automation.yml | 36 --- flyteidl/.github/workflows/buf_publish.yaml | 16 -- flyteidl/.github/workflows/master.yml | 46 ---- flyteidl/.github/workflows/npmpublish.yaml | 26 -- flyteidl/.github/workflows/pythonpublish.yaml | 32 --- .../.github/workflows/release-automation.yaml | 20 -- flyteidl/.github/workflows/verification.yml | 252 ------------------ .../workflows/boilerplate-automation.yml | 37 --- flyteplugins/.github/workflows/master.yml | 76 ------ .../.github/workflows/pull_request.yml | 46 ---- .../.github/workflows/upgrade_automation.yml | 20 -- flytepropeller/.github/workflows/checks.yml | 86 ------ .../.github/workflows/codeql-analysis.yml | 78 ------ flytepropeller/.github/workflows/krew.yaml | 19 -- .../.github/workflows/upgrade_automation.yml | 21 -- flytestdlib/.github/workflows/master.yml | 72 ----- .../.github/workflows/pull_request.yml | 45 ---- .../.github/workflows/upgrade_automation.yml | 12 - 27 files changed, 1291 deletions(-) delete mode 100644 datacatalog/.github/config.yml delete mode 100644 datacatalog/.github/workflows/checks.yml delete mode 100644 datacatalog/.github/workflows/upgrade_automation.yml delete mode 100644 flyteadmin/.github/workflows/checks.yml delete mode 100644 flyteadmin/.github/workflows/upgrade_automation.yml delete mode 100644 flytecopilot/.github/config.yml delete mode 100644 flytecopilot/.github/workflows/checks.yml delete mode 100644 flytecopilot/.github/workflows/upgrade_automation.yml delete mode 100644 flyteidl/.github/config.yml delete mode 100644 flyteidl/.github/workflows/boilerplate-automation.yml delete mode 100644 flyteidl/.github/workflows/buf_publish.yaml delete mode 100644 flyteidl/.github/workflows/master.yml delete mode 100644 flyteidl/.github/workflows/npmpublish.yaml delete mode 100644 flyteidl/.github/workflows/pythonpublish.yaml delete mode 100644 flyteidl/.github/workflows/release-automation.yaml delete mode 100644 flyteidl/.github/workflows/verification.yml delete mode 100644 flyteplugins/.github/workflows/boilerplate-automation.yml delete mode 100644 flyteplugins/.github/workflows/master.yml delete mode 100644 flyteplugins/.github/workflows/pull_request.yml delete mode 100644 flyteplugins/.github/workflows/upgrade_automation.yml delete mode 100644 flytepropeller/.github/workflows/checks.yml delete mode 100644 flytepropeller/.github/workflows/codeql-analysis.yml delete mode 100644 flytepropeller/.github/workflows/krew.yaml delete mode 100644 flytepropeller/.github/workflows/upgrade_automation.yml delete mode 100644 flytestdlib/.github/workflows/master.yml delete mode 100644 flytestdlib/.github/workflows/pull_request.yml delete mode 100644 flytestdlib/.github/workflows/upgrade_automation.yml diff --git a/datacatalog/.github/config.yml b/datacatalog/.github/config.yml deleted file mode 100644 index f99bcd78f1f..00000000000 --- a/datacatalog/.github/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: > - Thank you for opening this pull request! 🙌 - - These tips will help get your PR across the finish line: - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! 🎉 - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! 🛠 diff --git a/datacatalog/.github/workflows/checks.yml b/datacatalog/.github/workflows/checks.yml deleted file mode 100644 index 82efe6c6757..00000000000 --- a/datacatalog/.github/workflows/checks.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Datacatalog Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: "1.19" - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker_build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - generate: - name: Check Go Gennerate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: "1.19" - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ docker_build, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - with: - go-version: "1.19" - - push_docker_image: - name: Build & Push Datacatalog Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/datacatalog/.github/workflows/upgrade_automation.yml b/datacatalog/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 02cf6cde613..00000000000 --- a/datacatalog/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteadmin/.github/workflows/checks.yml b/flyteadmin/.github/workflows/checks.yml deleted file mode 100644 index 6451811c535..00000000000 --- a/flyteadmin/.github/workflows/checks.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Flyteadmin Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker-build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - endtoend: - name: End2End Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/end2end.yml@master - with: - priorities: "P0" - cache_key: ${{ needs.docker-build.outputs.cache_key }} - - integration: - name: Integration Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/integration.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - - generate: - name: Check Go Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ endtoend, integration, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flyteadmin Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - - push-docker-image-flytescheduler: - name: Build & Push Flytescheduler Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - push: true - repository: flyteorg/flytescheduler - dockerfile: scheduler.Dockerfile - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/flyteadmin/.github/workflows/upgrade_automation.yml b/flyteadmin/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c22..00000000000 --- a/flyteadmin/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytecopilot/.github/config.yml b/flytecopilot/.github/config.yml deleted file mode 100644 index f99bcd78f1f..00000000000 --- a/flytecopilot/.github/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: > - Thank you for opening this pull request! 🙌 - - These tips will help get your PR across the finish line: - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! 🎉 - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! 🛠 diff --git a/flytecopilot/.github/workflows/checks.yml b/flytecopilot/.github/workflows/checks.yml deleted file mode 100644 index a7e624f772c..00000000000 --- a/flytecopilot/.github/workflows/checks.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Flyte copilot Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - dry_run: - name: Dry run goreleaser - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser dry run - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: --snapshot --skip-publish --rm-dist - - build_docker_image: - name: Build & Push Flyte copilot Image - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: v1 - dockerfile: Dockerfile - push: false - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - - generate: - name: Check Go Gennerate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ lint, tests, generate, dry_run, build_docker_image ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flyte copilot Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} diff --git a/flytecopilot/.github/workflows/upgrade_automation.yml b/flytecopilot/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c22..00000000000 --- a/flytecopilot/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteidl/.github/config.yml b/flyteidl/.github/config.yml deleted file mode 100644 index 73da252e523..00000000000 --- a/flyteidl/.github/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Comment to be posted on PRs from first-time contributors in your repository -newPRWelcomeComment: | - Thank you for opening this pull request! 🙌 - - These tips will help get your PR across the finish line: - - - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. - - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). - -# Comment to be posted to on pull requests merged by a first time user -firstPRMergeComment: > - Congrats on merging your first pull request! 🎉 - -# Comment to be posted on first-time issues -newIssueWelcomeComment: > - Thank you for opening your first issue here! 🛠 diff --git a/flyteidl/.github/workflows/boilerplate-automation.yml b/flyteidl/.github/workflows/boilerplate-automation.yml deleted file mode 100644 index 7a257af06a7..00000000000 --- a/flyteidl/.github/workflows/boilerplate-automation.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Update Boilerplate Automation -on: - workflow_dispatch: - -jobs: - update-boilerplate: - name: Update Boilerplate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Update Boilerplate - run: | - make update_boilerplate - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.FLYTE_BOT_PAT }} - commit-message: Update Boilerplate - committer: Flyte-Bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: flyte-bot-update-boilerplate - delete-branch: true - title: 'Update Boilerplate' - body: | - Update Boilerplate - - Auto-generated by [flyte-bot] - labels: | - boilerplate - team-reviewers: | - owners - maintainers - draft: false \ No newline at end of file diff --git a/flyteidl/.github/workflows/buf_publish.yaml b/flyteidl/.github/workflows/buf_publish.yaml deleted file mode 100644 index 17ecac24231..00000000000 --- a/flyteidl/.github/workflows/buf_publish.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Publish Buf Package - -on: - release: - types: [created] - -jobs: - buf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: bufbuild/buf-setup-action@v1 - - uses: bufbuild/buf-push-action@v1 - with: - buf_token: ${{ secrets.BUF_TOKEN }} - input: 'protos' diff --git a/flyteidl/.github/workflows/master.yml b/flyteidl/.github/workflows/master.yml deleted file mode 100644 index 2fc19bbdab8..00000000000 --- a/flyteidl/.github/workflows/master.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - if: github.repository == 'flyteorg/flyteidl' - name: Bump Version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump-version - uses: anothrNick/github-tag-action@1.36.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - - goreleaser: - if: github.repository == 'flyteorg/flyteidl' - name: Goreleaser - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.19 - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flyteidl/.github/workflows/npmpublish.yaml b/flyteidl/.github/workflows/npmpublish.yaml deleted file mode 100644 index 1101ca0a208..00000000000 --- a/flyteidl/.github/workflows/npmpublish.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish NPM Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # Setup .npmrc file to publish to npm - - uses: actions/setup-node@v1 - with: - node-version: "12.x" - registry-url: "https://registry.npmjs.org" - - name: Autobump version - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - VERSION=$VERSION make update_npmversion - shell: bash - - run: npm install - - run: npm publish --access=public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/flyteidl/.github/workflows/pythonpublish.yaml b/flyteidl/.github/workflows/pythonpublish.yaml deleted file mode 100644 index b418b35c1b5..00000000000 --- a/flyteidl/.github/workflows/pythonpublish.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Upload PyPi Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Autobump version - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - VERSION=$VERSION make update_pyversion - shell: bash - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/flyteidl/.github/workflows/release-automation.yaml b/flyteidl/.github/workflows/release-automation.yaml deleted file mode 100644 index b0388223da0..00000000000 --- a/flyteidl/.github/workflows/release-automation.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Invoke Workflow - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - strategy: - matrix: - COMPONENT: [flyteadmin, flytepropeller, flyteconsole, flytecopilot, flyteplugins, datacatalog, flytectl] - fail-fast: false - steps: - - uses: actions/checkout@v2 - - name: Update flyteidl version - run: | - WORKFLOW_ID=$(curl -sS https://api.github.com/repos/flyteorg/${{matrix.COMPONENT}}/actions/workflows | jq '.workflows[] | select(.path == ".github/workflows/upgrade_automation.yml") | {id}' | jq .id) - curl -X POST -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/flyteorg/${{matrix.COMPONENT}}/actions/workflows/$WORKFLOW_ID/dispatches -H "Authorization: token ${{ secrets.FLYTE_BOT_PAT }}" -d '{"ref":"master", "inputs": {"component": "flyteidl"}}' - shell: bash diff --git a/flyteidl/.github/workflows/verification.yml b/flyteidl/.github/workflows/verification.yml deleted file mode 100644 index bb8270a4759..00000000000 --- a/flyteidl/.github/workflows/verification.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: Verification Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - pull_request: - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: 1.19 - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - with: - go-version: 1.19 - generate-protos: - runs-on: ubuntu-latest - name: Generate Protos - container: - image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - # GO Protos - - name: Proto-Service-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service --with_gateway -l go --go_source_relative - - name: Proto-Admin-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Core-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Event-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event --with_gateway -l go --go_source_relative --validate_out - - name: Proto-Plugins-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l go --go_source_relative --validate_out - - name: Proto-Datacatalog-Go - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l go --go_source_relative --validate_out - - # Python is going to be generated using buf - # Python - # - name: Proto-Service-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l python - # - name: Proto-Admin-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l python - # - name: Proto-Core-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l python - # - name: Proto-Event-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l python - # - name: Proto-Plugins-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l python - # - name: Proto-Datacatalog-Python - # run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l python - - # Cpp - - name: Proto-Service-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l cpp - - name: Proto-Admin-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l cpp - - name: Proto-Core-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l cpp - - name: Proto-Event-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l cpp - - name: Proto-Plugins-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l cpp - - name: Proto-Datacatalog-Cpp - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l cpp - - # Java - - name: Proto-Service-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l java - - name: Proto-Admin-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l java - - name: Proto-Core-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l java - - name: Proto-Event-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l java - - name: Proto-Plugins-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l java - - name: Proto-Datacatalog-Java - run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l java - - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - generate-protos-buf: - name: Generate python protos using buf - runs-on: ubuntu-latest - needs: [ generate-protos ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: bufbuild/buf-setup-action@v1 - - name: "Clean Generated" - run: rm -rf ./gen - - name: "Run buf generate" - run: buf generate - - name: "Include __init__.py files" - run: find gen/pb_python -type d -exec touch {}/__init__.py \; - - uses: actions/upload-artifact@master - with: - name: generated-buf - path: ./gen - - generate-admin-swagger-code: - runs-on: ubuntu-latest - name: Generate Admin Swagger Code - needs: - - generate-protos - - generate-protos-buf - container: - image: lyft/protocgenerator:8167e11d3b3439373c2f033080a4b550078884a2 - options: --cpus 1 - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - uses: actions/download-artifact@master - with: - name: generated-buf - path: ./gen - - run: ls gen/pb-go/flyteidl/service/ - - run: cp -R gen/pb-go/flyteidl/service/ ../ - - run: ls ../service/ - # Open API 2 - - name: OpenAPI-Binary - run: go-bindata -pkg service -o ../service/openapi.go -prefix ../service/ -modtime 1562572800 ../service/admin.swagger.json - - run: rm -rf gen/pb-go/flyteidl/service - - run: mkdir -p gen/pb-go/flyteidl/service - - run: cp -R ../service gen/pb-go/flyteidl/ - - run: ls gen/pb-go/flyteidl/ - - run: ls gen/pb-go/flyteidl/service/ - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - generate-js-code: - runs-on: ubuntu-latest - name: Generate JS Code - needs: [ generate-admin-swagger-code ] - container: - image: schottra/docker-protobufjs:v0.0.2 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - run: mkdir -p /defs - - run: mkdir -p /gen - - run: cp -R . /defs - - run: node /code/generate-protobuf.js --module-name flyteidl -d protos/flyteidl/core -d protos/flyteidl/event -d protos/flyteidl/admin -d protos/flyteidl/service -- --root flyteidl -t static-module -w default --no-delimited --force-long --no-convert -p /defs/protos - - run: ls /defs/gen/ - - uses: actions/upload-artifact@master - with: - name: generated - path: /defs/gen - - generate-swagger-code: - runs-on: ubuntu-latest - name: Generate Swagger Code - needs: [ generate-js-code ] - container: - image: docker.io/lyft/swagger-codegen-cli:dc5ce6ec6d7d4d980fa882d6bd13a83cba3be3c3 - options: --cpus 1 - volumes: - - /github/workspace:/defs - env: - REPO_BLOB_SHA: master - PROJECT_ANNOTATION_PREFIX: flyte.interface - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l go -o ./gen/pb-go/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin - - run: java -jar /opt/swagger-codegen-cli/swagger-codegen-cli.jar generate -i ./gen/pb-go/flyteidl/service/admin.swagger.json -l python -o ./gen/pb_python/flyteidl/service/flyteadmin --additional-properties=packageName=flyteadmin - - uses: actions/upload-artifact@master - with: - name: generated - path: ./gen - - test-proto-changes: - runs-on: ubuntu-latest - name: Test Proto Changes - needs: [ generate-swagger-code ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: "Clean Generated" - run: rm -rf ./gen - - uses: actions/download-artifact@master - with: - name: generated - path: ./gen - - name: Test - run: ./scripts/test_diff.sh - - generate: - name: Check Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: 1.19 diff --git a/flyteplugins/.github/workflows/boilerplate-automation.yml b/flyteplugins/.github/workflows/boilerplate-automation.yml deleted file mode 100644 index d4a76239bea..00000000000 --- a/flyteplugins/.github/workflows/boilerplate-automation.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Update Boilerplate Automation -on: - workflow_dispatch: - -jobs: - update-boilerplate: - name: Update Boilerplate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Update Boilerplate - run: | - make update_boilerplate - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.FLYTE_BOT_PAT }} - commit-message: Update Boilerplate - committer: Flyte-Bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: flyte-bot-update-boilerplate - delete-branch: true - title: 'Update Boilerplate' - body: | - Update Boilerplate - - Auto-generated by [flyte-bot] - labels: | - boilerplate - team-reviewers: | - owners - maintainers - draft: false - diff --git a/flyteplugins/.github/workflows/master.yml b/flyteplugins/.github/workflows/master.yml deleted file mode 100644 index 0dc07253b47..00000000000 --- a/flyteplugins/.github/workflows/master.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - name: Bump Version - if: github.event.commits[0].author.name != 'goreleaserbot' - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Bump version and push tag - id: bump-version - uses: anothrNick/github-tag-action@1.36.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - goreleaser: - name: Goreleaser - runs-on: ubuntu-latest - needs: [bump-version] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} - tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - lint: - name: Run lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint diff --git a/flyteplugins/.github/workflows/pull_request.yml b/flyteplugins/.github/workflows/pull_request.yml deleted file mode 100644 index 52f6e89c30f..00000000000 --- a/flyteplugins/.github/workflows/pull_request.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Pull Request - -on: pull_request - -jobs: - tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make install && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - lint: - name: Run lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint - - generate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Go generate and diff - run: DELTA_CHECK=true make generate diff --git a/flyteplugins/.github/workflows/upgrade_automation.yml b/flyteplugins/.github/workflows/upgrade_automation.yml deleted file mode 100644 index b94163b1c22..00000000000 --- a/flyteplugins/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytepropeller/.github/workflows/checks.yml b/flytepropeller/.github/workflows/checks.yml deleted file mode 100644 index acb7bde43d0..00000000000 --- a/flytepropeller/.github/workflows/checks.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Flytepropeller Checks - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - pull_request: - branches: - - master - push: - branches: - - master - -jobs: - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: "1.19" - - tests: - name: Unit Tests - uses: flyteorg/flytetools/.github/workflows/tests.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - docker-build: - name: Docker Build Images - uses: flyteorg/flytetools/.github/workflows/docker_build.yml@master - - endtoend: - name: End2End Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/end2end.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - priorities: "P0" - - integration: - name: Integration Test - needs: [ docker-build ] - uses: flyteorg/flytetools/.github/workflows/integration.yml@master - with: - cache_key: ${{ needs.docker-build.outputs.cache_key }} - go-version: "1.19" - - generate: - name: Check Go Generate - uses: flyteorg/flytetools/.github/workflows/go_generate.yml@master - with: - go-version: "1.19" - - bump_version: - name: Bump Version - if: ${{ github.event_name != 'pull_request' }} - needs: [ endtoend, integration, lint, tests, generate ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/bump_version.yml@master - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - goreleaser: - name: Goreleaser - needs: [ bump_version ] # Only to ensure it can successfully build - uses: flyteorg/flytetools/.github/workflows/goreleaser.yml@master - with: - go-version: "1.19" - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - - push_docker_image: - name: Build & Push Flytepropeller Image - needs: [ bump_version ] - uses: flyteorg/flytetools/.github/workflows/publish.yml@master - with: - version: ${{ needs.bump_version.outputs.version }} - dockerfile: Dockerfile - push: true - repository: ${{ github.repository }} - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} - FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }} - diff --git a/flytepropeller/.github/workflows/codeql-analysis.yml b/flytepropeller/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 4ddece498b4..00000000000 --- a/flytepropeller/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,78 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '23 3 * * 6' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - uses: actions/setup-go@v3 - with: - go-version: '1.19' - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/flytepropeller/.github/workflows/krew.yaml b/flytepropeller/.github/workflows/krew.yaml deleted file mode 100644 index a8742237af3..00000000000 --- a/flytepropeller/.github/workflows/krew.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: releaser -on: - release: - types: - - published - -jobs: - release-cli-via-krew: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Unshallow - run: git fetch --prune --unshallow - - - name: Update new version in krew-index - if: "!github.event.release.prerelease" - uses: rajatjindal/krew-release-bot@v0.0.38 diff --git a/flytepropeller/.github/workflows/upgrade_automation.yml b/flytepropeller/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 6d1ea416394..00000000000 --- a/flytepropeller/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Upgrade Automation -on: - workflow_dispatch: - inputs: - component: - description: "Name of Flyte component" - required: true - default: "boilerplate" - type: choice - options: - - boilerplate - - flyteidl -jobs: - trigger-upgrade: - name: ${{ github.event.inputs.component }} Upgrade - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: ${{ github.event.inputs.component }} - go-version: 1.19 - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} diff --git a/flytestdlib/.github/workflows/master.yml b/flytestdlib/.github/workflows/master.yml deleted file mode 100644 index 3c25d004e4b..00000000000 --- a/flytestdlib/.github/workflows/master.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Master - -on: - push: - branches: - - master - -jobs: - bump-version: - name: Bump Version - if: github.event.commits[0].author.name != 'goreleaserbot' - runs-on: ubuntu-latest - outputs: - version: ${{ steps.bump-version.outputs.tag }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump-version - uses: flyteorg/github-tag-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true - DEFAULT_BUMP: patch - - goreleaser: - name: Goreleaser - runs-on: ubuntu-latest - needs: [ bump-version ] - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 - with: - version: latest - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - tests-lint: - name: Run tests and lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make mod_download && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Lint - run: make install && make lint - - name: Bench tests - run: make install && make test_benchmark diff --git a/flytestdlib/.github/workflows/pull_request.yml b/flytestdlib/.github/workflows/pull_request.yml deleted file mode 100644 index 39c22142a3c..00000000000 --- a/flytestdlib/.github/workflows/pull_request.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Pull Request - -on: pull_request - -jobs: - tests-lint: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Unit Tests - run: make mod_download && make test_unit_codecov - - name: Push CodeCov - uses: codecov/codecov-action@v3.1.1 - with: - file: coverage.txt - flags: unittests - fail_ci_if_error: false - - name: Bench tests - run: make install && make test_benchmark - lint: - name: Lint - uses: flyteorg/flytetools/.github/workflows/lint.yml@master - with: - go-version: '1.19' - test-generate-integrity: - name: Ensure go generate has run - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - uses: actions/setup-go@v2 - with: - go-version: '1.19' - - name: Go generate and diff - run: DELTA_CHECK=true make generate - diff --git a/flytestdlib/.github/workflows/upgrade_automation.yml b/flytestdlib/.github/workflows/upgrade_automation.yml deleted file mode 100644 index 94fceadd27a..00000000000 --- a/flytestdlib/.github/workflows/upgrade_automation.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Boilerplate Upgrade Automation -on: - workflow_dispatch: - -jobs: - trigger-upgrade: - name: Boilerplate Upgrade Automation - uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master - with: - component: boilerplate - secrets: - FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file From 98dd50588c120a134db7f4545ab5b36b2f02ff8a Mon Sep 17 00:00:00 2001 From: Prafulla Mahindrakar Date: Mon, 27 Nov 2023 11:07:53 -0800 Subject: [PATCH 2/2] Fix the loop variable scheduler issue (#4468) * Fix the loop variable scheduler issue Signed-off-by: pmahindrakar-oss * lint Signed-off-by: pmahindrakar-oss * renamed variable Signed-off-by: pmahindrakar-oss * Removed the debuggger file Signed-off-by: pmahindrakar-oss * Added comments Signed-off-by: pmahindrakar-oss --------- Signed-off-by: pmahindrakar-oss --- flyteadmin/scheduler/core/gocron_scheduler.go | 11 ++- .../scheduler/core/gocron_scheduler_test.go | 96 +++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/flyteadmin/scheduler/core/gocron_scheduler.go b/flyteadmin/scheduler/core/gocron_scheduler.go index f569b7c335e..c9386746c64 100644 --- a/flyteadmin/scheduler/core/gocron_scheduler.go +++ b/flyteadmin/scheduler/core/gocron_scheduler.go @@ -54,23 +54,26 @@ func (g *GoCronScheduler) GetTimedFuncWithSchedule() TimedFuncWithSchedule { func (g *GoCronScheduler) BootStrapSchedulesFromSnapShot(ctx context.Context, schedules []models.SchedulableEntity, snapshot snapshoter.Snapshot) { for _, s := range schedules { + // Copy the object to save to a new pointer since the pointer is saved later + // Issue due to https://github.com/golang/go/discussions/56010 + schedule := s if *s.Active { funcRef := g.GetTimedFuncWithSchedule() nameOfSchedule := identifier.GetScheduleName(ctx, s) // Initialize the lastExectime as the updatedAt time // Assumption here that schedule was activated and that the 0th execution of the schedule // which will be used as a reference - lastExecTime := &s.UpdatedAt + lastExecTime := &schedule.UpdatedAt fromSnapshot := snapshot.GetLastExecutionTime(nameOfSchedule) // Use the latest time if available in the snapshot - if fromSnapshot != nil && fromSnapshot.After(s.UpdatedAt) { + if fromSnapshot != nil && fromSnapshot.After(schedule.UpdatedAt) { lastExecTime = fromSnapshot } - err := g.ScheduleJob(ctx, s, funcRef, lastExecTime) + err := g.ScheduleJob(ctx, schedule, funcRef, lastExecTime) if err != nil { g.metrics.JobScheduledFailedCounter.Inc() - logger.Errorf(ctx, "unable to register the schedule %+v due to %v", s, err) + logger.Errorf(ctx, "unable to register the schedule %+v due to %v", schedule, err) } } } diff --git a/flyteadmin/scheduler/core/gocron_scheduler_test.go b/flyteadmin/scheduler/core/gocron_scheduler_test.go index d19436ddd54..6732eab09d2 100644 --- a/flyteadmin/scheduler/core/gocron_scheduler_test.go +++ b/flyteadmin/scheduler/core/gocron_scheduler_test.go @@ -296,3 +296,99 @@ func TestCatchUpAllSchedule(t *testing.T) { catchupSuccess := g.CatchupAll(ctx, toTime) assert.True(t, catchupSuccess) } + +func TestGoCronScheduler_BootStrapSchedulesFromSnapShot(t *testing.T) { + g := setupWithSchedules(t, "testing", []models.SchedulableEntity{}, true) + True := true + False := false + scheduleActive1 := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(1000, time.October, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "schedule_active_1", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &True, + } + scheduleActive2 := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(2000, time.November, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "schedule_active_2", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &True, + } + scheduleInactive := models.SchedulableEntity{ + BaseModel: adminModels.BaseModel{ + UpdatedAt: time.Date(3000, time.December, 19, 10, 0, 0, 0, time.UTC), + }, + SchedulableEntityKey: models.SchedulableEntityKey{ + Project: "project", + Domain: "domain", + Name: "cron3", + Version: "version1", + }, + CronExpression: "0 19 * * *", + Active: &False, + } + + schedule1SnapshotTime := time.Date(5000, time.December, 19, 10, 0, 0, 0, time.UTC) + schedule2SnapshotTime := time.Date(6000, time.December, 19, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + schedules []models.SchedulableEntity + snapshoter snapshoter.Snapshot + expectedCatchUpTimes map[string]*time.Time + }{ + { + name: "two active", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2}, + snapshoter: &snapshoter.SnapshotV1{}, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &scheduleActive1.UpdatedAt, "1420107156943834850": &scheduleActive2.UpdatedAt}, + }, + { + name: "two active one inactive", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2, scheduleInactive}, + snapshoter: &snapshoter.SnapshotV1{}, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &scheduleActive1.UpdatedAt, "1420107156943834850": &scheduleActive2.UpdatedAt}, + }, + { + name: "two active one inactive with snapshot populated", + schedules: []models.SchedulableEntity{scheduleActive1, scheduleActive2, scheduleInactive}, + snapshoter: &snapshoter.SnapshotV1{ + LastTimes: map[string]*time.Time{ + "11407394263542327059": &schedule1SnapshotTime, + "1420107156943834850": &schedule2SnapshotTime, + }, + }, + expectedCatchUpTimes: map[string]*time.Time{"11407394263542327059": &schedule1SnapshotTime, "1420107156943834850": &schedule2SnapshotTime}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g.BootStrapSchedulesFromSnapShot(context.Background(), tt.schedules, tt.snapshoter) + g.jobStore.Range(func(key, value interface{}) bool { + jobID := key.(string) + job := value.(*GoCronJob) + if !*job.schedule.Active { + return true + } + assert.Equal(t, job.catchupFromTime, tt.expectedCatchUpTimes[jobID]) + return true + }) + for _, schedule := range tt.schedules { + g.DeScheduleJob(context.TODO(), schedule) + } + }) + } +}