From e1ae6b80cd1b8cd685765db7c1520e2be0fe028f Mon Sep 17 00:00:00 2001 From: Ryan Liang Date: Tue, 29 Nov 2022 13:27:12 -0500 Subject: [PATCH 1/4] Reuse of Opensearch core and Opensearch Dashboard setup for CI workflow Signed-off-by: Ryan Liang --- .github/actions/install-dashboards/action.yml | 5 +- .github/actions/install-opensearch/action.yml | 61 +++++++++++++ .github/workflows/cypress-test.yml | 88 +++---------------- .github/workflows/integration-test.yml | 20 ++++- .github/workflows/unit-test.yml | 3 +- 5 files changed, 93 insertions(+), 84 deletions(-) create mode 100644 .github/actions/install-opensearch/action.yml diff --git a/.github/actions/install-dashboards/action.yml b/.github/actions/install-dashboards/action.yml index 7dae184e8..a5493a1b1 100644 --- a/.github/actions/install-dashboards/action.yml +++ b/.github/actions/install-dashboards/action.yml @@ -15,7 +15,6 @@ outputs: description: "The directory where the plugin has been configured" value: ${{ steps.determine-plugin-directory.outputs.plugin-directory }} - runs: using: "composite" steps: @@ -25,7 +24,6 @@ runs: - id: determine-plugin-directory run: echo "::set-output name=plugin-directory::./OpenSearch-Dashboards/plugins/${{ inputs.plugin_name }}" - # run: echo "plugin-directory=./OpenSearch-Dashboards/plugins/${{ inputs.plugin_name }}" >> $GITHUB_OUTPUT shell: bash - uses: actions/checkout@v2 @@ -42,7 +40,6 @@ runs: - uses: actions/checkout@v2 with: path: ${{ steps.determine-plugin-directory.outputs.plugin-directory }} - # ref: ${{ github.ref }} - id: osd-version continue-on-error: true @@ -53,7 +50,7 @@ runs: shell: bash - id: branch-switch-if-possible - continue-on-error: true # Defaults onto main if the branch switch doesn't work + continue-on-error: true if: ${{ steps.osd-version.outputs.osd-version }} run: git checkout ${{ steps.osd-version.outputs.osd-version }} || git checkout ${{ steps.osd-version.outputs.osd-x-version }}x working-directory: ./OpenSearch-Dashboards diff --git a/.github/actions/install-opensearch/action.yml b/.github/actions/install-opensearch/action.yml new file mode 100644 index 000000000..1adcc1867 --- /dev/null +++ b/.github/actions/install-opensearch/action.yml @@ -0,0 +1,61 @@ +name: 'Install OpenSearch with Plugin' +description: 'Installs and run OpenSearch docker' + +inputs: + opensearch_version: + description: 'The version of OpenSearch core' + required: true + + security_plugin_version: + description: 'The version of security plugin' + required: true + + plugin-name: + description: 'The the name of the plugin to use, such as opensearch-security' + required: true + + plugin-zip: + description: 'The name of the zip file for the plugin hosted on docker-host i.e. security-plugin.zip' + required: true + + plugin-start-script: + description: 'The file name for the configuration script for the plugin such as install_demo_configurations -- may not be needed for every plugin' + required: false + +runs: + using: "composite" + steps: + - name: Download OpenSearch Core + run: | + wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch_version }}/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-${{ inputs.opensearch_version }}-linux-x64.tar.gz + tar -xzf opensearch-*.tar.gz + rm -f opensearch-*.tar.gz + shell: bash + + - name: Download OpenSearch Security Plugin + run: wget -O ${{ inputs.plugin-name }}.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch_version }}/latest/linux/x64/tar/builds/opensearch/plugins/${{ inputs.plugin-name }}-${{ inputs.security_plugin_version }}.zip + shell: bash + + - name: Run OpenSearch with plugin + run: | + cat > os-ep.sh <> /opensearch/config/opensearch.yml + chown 1001:1001 -R /opensearch + su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch + EOF + docker build -t opensearch-test:latest -f- . < os-ep.sh <> /opensearch/config/opensearch.yml - chown 1001:1001 -R /opensearch - su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch - EOF - docker build -t opensearch-test:latest -f- . <> ./config/opensearch_dashboards.yml echo 'opensearch.hosts: ["https://localhost:9200"]' >> ./config/opensearch_dashboards.yml echo 'opensearch.ssl.verificationMode: none' >> ./config/opensearch_dashboards.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 2861fdbb8..3a9ef663c 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -17,6 +17,9 @@ jobs: - uses: browser-actions/setup-firefox@latest - run: firefox --version + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Download OpenSearch Core run: | wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-3.0.0-linux-x64.tar.gz @@ -26,7 +29,6 @@ jobs: - name: Download OpenSearch Security Plugin run: wget -O opensearch-security.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-3.0.0.0.zip - - name: Run OpenSearch with plugin run: | cat > os-ep.sh < Date: Thu, 1 Dec 2022 15:49:04 -0500 Subject: [PATCH 2/4] Reuse of Security repo action Signed-off-by: Ryan Liang --- .github/workflows/integration-test.yml | 101 +++++++++++++++---------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 3a9ef663c..f5018c9c7 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -5,64 +5,81 @@ on: [push, pull_request] env: TEST_BROWSER_HEADLESS: 1 CI: 1 + OPENSEARCH_VERSION: 3.0.0 + PLUGIN_NAME: opensearch-security jobs: tests: name: Run integration tests runs-on: ubuntu-latest steps: + - name: Checkout Branch + uses: actions/checkout@v2 + - uses: browser-actions/setup-geckodriver@latest - run: geckodriver --version - uses: browser-actions/setup-firefox@latest - run: firefox --version - - name: Checkout Branch - uses: actions/checkout@v2 - - - name: Download OpenSearch Core - run: | - wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-3.0.0-linux-x64.tar.gz - tar -xzf opensearch-*.tar.gz - rm -f opensearch-*.tar.gz + #- name: Download OpenSearch Core + # run: | + # wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-3.0.0-linux-x64.tar.gz + # tar -xzf opensearch-*.tar.gz + # rm -f opensearch-*.tar.gz - name: Download OpenSearch Security Plugin - run: wget -O opensearch-security.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-3.0.0.0.zip - - - name: Run OpenSearch with plugin + run: wget --progress=bar:force:noscroll -O opensearch-security.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-3.0.0.0.zip + + # + #- name: Run OpenSearch with plugin + # run: | + # cat > os-ep.sh <> /opensearch/config/opensearch.yml + # chown 1001:1001 -R /opensearch + # su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch + # EOF + # docker build -t opensearch-test:latest -f- . < os-ep.sh <> /opensearch/config/opensearch.yml - chown 1001:1001 -R /opensearch - su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch - EOF - docker build -t opensearch-test:latest -f- . < setup.sh <<'EOF' + chmod +x ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh + /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh" + echo "plugins.security.unsupported.restapi.allow_securityconfig_modification: true" >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/opensearch.yml EOF - docker run -d --network=host -i opensearch-test:latest - - # - name: Checkout Branch - # uses: actions/checkout@v2 - # - # - name: Run OpenSearch with Security Plugin - # uses: ./.github/actions/install-opensearch - # with: - # opensearch_version: 3.0.0 - # security_plugin_version: 3.0.0.0 - # plugin-name: opensearch-security - # plugin-start-script: install_demo_configuration - # plugin-zip: security-plugin.zip + + - name: Run Opensearch with A Single Plugin + #uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main + uses: RyanL1997/security/.github/actions/start-opensearch-with-one-plugin@security-reusable-action + with: + opensearch-version: ${{ env.OPENSEARCH_VERSION }} + plugin-name: ${{ env.PLUGIN_NAME }} + setup-script-name: setup + + #- name: Run OpenSearch with Security Plugin + # uses: opensearch-project/security/.github/actions/install-opensearch + # with: + # opensearch_version: 3.0.0 + # security_plugin_version: 3.0.0.0 + # plugin-name: opensearch-security + # plugin-start-script: install_demo_configuration + # plugin-zip: security-plugin.zip - id: install-dashboards uses: ./.github/actions/install-dashboards From 30aa018a55122066781166a1110bfa583080ce57 Mon Sep 17 00:00:00 2001 From: Ryan Liang Date: Thu, 1 Dec 2022 16:17:52 -0500 Subject: [PATCH 3/4] Apply reusable workflow for both integtation and cyress tests Signed-off-by: Ryan Liang --- .github/actions/install-dashboards/action.yml | 2 +- .github/actions/install-opensearch/action.yml | 61 ------------------- .github/workflows/cypress-test.yml | 27 +++++--- .github/workflows/integration-test.yml | 42 +------------ 4 files changed, 23 insertions(+), 109 deletions(-) delete mode 100644 .github/actions/install-opensearch/action.yml diff --git a/.github/actions/install-dashboards/action.yml b/.github/actions/install-dashboards/action.yml index a5493a1b1..d9b7d9eb0 100644 --- a/.github/actions/install-dashboards/action.yml +++ b/.github/actions/install-dashboards/action.yml @@ -50,7 +50,7 @@ runs: shell: bash - id: branch-switch-if-possible - continue-on-error: true + continue-on-error: true # Defaults onto main if the branch switch doesn't work if: ${{ steps.osd-version.outputs.osd-version }} run: git checkout ${{ steps.osd-version.outputs.osd-version }} || git checkout ${{ steps.osd-version.outputs.osd-x-version }}x working-directory: ./OpenSearch-Dashboards diff --git a/.github/actions/install-opensearch/action.yml b/.github/actions/install-opensearch/action.yml deleted file mode 100644 index 1adcc1867..000000000 --- a/.github/actions/install-opensearch/action.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: 'Install OpenSearch with Plugin' -description: 'Installs and run OpenSearch docker' - -inputs: - opensearch_version: - description: 'The version of OpenSearch core' - required: true - - security_plugin_version: - description: 'The version of security plugin' - required: true - - plugin-name: - description: 'The the name of the plugin to use, such as opensearch-security' - required: true - - plugin-zip: - description: 'The name of the zip file for the plugin hosted on docker-host i.e. security-plugin.zip' - required: true - - plugin-start-script: - description: 'The file name for the configuration script for the plugin such as install_demo_configurations -- may not be needed for every plugin' - required: false - -runs: - using: "composite" - steps: - - name: Download OpenSearch Core - run: | - wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch_version }}/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-${{ inputs.opensearch_version }}-linux-x64.tar.gz - tar -xzf opensearch-*.tar.gz - rm -f opensearch-*.tar.gz - shell: bash - - - name: Download OpenSearch Security Plugin - run: wget -O ${{ inputs.plugin-name }}.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch_version }}/latest/linux/x64/tar/builds/opensearch/plugins/${{ inputs.plugin-name }}-${{ inputs.security_plugin_version }}.zip - shell: bash - - - name: Run OpenSearch with plugin - run: | - cat > os-ep.sh <> /opensearch/config/opensearch.yml - chown 1001:1001 -R /opensearch - su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch - EOF - docker build -t opensearch-test:latest -f- . < setup.sh <<'EOF' + chmod +x ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh + /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh" + EOF + + - name: Run Opensearch with A Single Plugin + #uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main + uses: RyanL1997/security/.github/actions/start-opensearch-with-one-plugin@security-reusable-action with: - opensearch_version: 3.0.0 - security_plugin_version: 3.0.0.0 - plugin-name: opensearch-security - plugin-start-script: install_demo_configuration - plugin-zip: security-plugin.zip + opensearch-version: ${{ env.OPENSEARCH_VERSION }} + plugin-name: ${{ env.PLUGIN_NAME }} + setup-script-name: setup - name: Run Dashboard with Security Dashboards Plugin uses: ./.github/actions/install-dashboards diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index f5018c9c7..65d99be7e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -7,6 +7,7 @@ env: CI: 1 OPENSEARCH_VERSION: 3.0.0 PLUGIN_NAME: opensearch-security + PLUGIN_VERSION: 3.0.0.0 jobs: tests: @@ -21,39 +22,9 @@ jobs: - uses: browser-actions/setup-firefox@latest - run: firefox --version - - #- name: Download OpenSearch Core - # run: | - # wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-3.0.0-linux-x64.tar.gz - # tar -xzf opensearch-*.tar.gz - # rm -f opensearch-*.tar.gz - name: Download OpenSearch Security Plugin - run: wget --progress=bar:force:noscroll -O opensearch-security.zip https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-3.0.0.0.zip - - # - #- name: Run OpenSearch with plugin - # run: | - # cat > os-ep.sh <> /opensearch/config/opensearch.yml - # chown 1001:1001 -R /opensearch - # su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch - # EOF - # docker build -t opensearch-test:latest -f- . < Date: Fri, 2 Dec 2022 14:35:09 -0500 Subject: [PATCH 4/4] Reference to security repo Signed-off-by: Ryan Liang --- .github/workflows/cypress-test.yml | 3 +-- .github/workflows/integration-test.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cypress-test.yml b/.github/workflows/cypress-test.yml index 212445a3d..51cf78bbc 100644 --- a/.github/workflows/cypress-test.yml +++ b/.github/workflows/cypress-test.yml @@ -33,8 +33,7 @@ jobs: EOF - name: Run Opensearch with A Single Plugin - #uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main - uses: RyanL1997/security/.github/actions/start-opensearch-with-one-plugin@security-reusable-action + uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main with: opensearch-version: ${{ env.OPENSEARCH_VERSION }} plugin-name: ${{ env.PLUGIN_NAME }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 65d99be7e..2b692ef5d 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -36,8 +36,7 @@ jobs: EOF - name: Run Opensearch with A Single Plugin - #uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main - uses: RyanL1997/security/.github/actions/start-opensearch-with-one-plugin@security-reusable-action + uses: opensearch-project/security/.github/actions/start-opensearch-with-one-plugin@main with: opensearch-version: ${{ env.OPENSEARCH_VERSION }} plugin-name: ${{ env.PLUGIN_NAME }}