add log column burst check #3574
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| # Run this workflow every time a new commit pushed to your repository | |
| on: push | |
| env: | |
| DOTNET_NOLOGO: 1 | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| jobs: | |
| # Set the job key. The key is displayed as the job name | |
| # when a job name is not provided | |
| super-lint: | |
| # Name the Job | |
| name: Build, test, package and sign release | |
| # Set the type of machine to run on | |
| runs-on: windows-latest | |
| steps: | |
| - name: Stub Node dependencies | |
| shell: bash | |
| run: touch package-lock.json | |
| - name: Install Node for coverage reporting | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '16.x' | |
| cache: 'npm' | |
| - name: LCov merger tool | |
| run: npm install -g lcov-result-merger | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Determine RDMP build version | |
| id: version | |
| shell: bash | |
| run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT | |
| - name: Check output | |
| run: echo ${{ steps.version.outputs.rdmpversion }} | |
| - name: Setup .NET Core | |
| uses: actions/[email protected] | |
| with: | |
| dotnet-version: 6.0.x | |
| - name: Install MS SQL 2019 Express LocalDB | |
| uses: crazy-max/ghaction-chocolatey@v2 | |
| with: | |
| args: install -r sqllocaldb --no-progress | |
| - name: Initialise LocalDB | |
| shell: bash | |
| run: | | |
| SqlLocalDB.exe create MSSQLLocalDB -s | |
| sqlcmd -l 180 -S '(localdb)\MSSQLLocalDB' -Q "SELECT @@VERSION;" | |
| sed -i'' -e 's/localhost/\(localdb\)\\MSSQLLocalDB/' Tests.Common/TestDatabases.txt | |
| - uses: shogo82148/actions-setup-mysql@v1 | |
| with: | |
| mysql-version: '8.0' | |
| root-password: 'YourStrong!Passw0rd' | |
| auto-start: true | |
| - name: Build | |
| run: dotnet build --configuration Release --verbosity minimal | |
| - name: Create MySql Logging, DQE and Cohort Building Cache Db | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ | |
| - name: Initialise RDMP | |
| run: dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e | |
| - name: Populate Databases.yaml | |
| shell: bash | |
| run: | | |
| cat > ./Tools/rdmp/Databases.yaml << EOF | |
| CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; | |
| DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; | |
| EOF | |
| - name: Run integration test scripts | |
| run: | | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_list_destroy_catalogue.yaml && | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_cohort.yaml && | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/create_dataload.yaml && | |
| dotnet run -c Release --no-build --project Tools/rdmp/rdmp.csproj -- -f ./scripts/orphan_extractable_column.yaml && | |
| - name: Test (DB) | |
| shell: bash | |
| run: | | |
| rm -rf coverage | |
| dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` db-ui.lcov | |
| dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` db-core.lcov | |
| - name: Test with local file system | |
| shell: bash | |
| run: | | |
| echo "UseFileSystemRepo: true" >> Tests.Common/TestDatabases.txt | |
| dotnet test Rdmp.UI.Tests/Rdmp.UI.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` fs-ui.lcov | |
| dotnet test Rdmp.Core.Tests/Rdmp.Core.Tests.csproj --nologo --collect:"XPlat Code Coverage" --no-build --verbosity minimal -c Release --results-directory coverage -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=lcov | |
| mv `find coverage -type f` fs-core.lcov | |
| - name: Merge LCovs | |
| run: lcov-result-merger "{db,fs}-{ui,core}.lcov" all.lcov | |
| - name: Coveralls | |
| uses: coverallsapp/github-action@master | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| path-to-lcov: all.lcov | |
| flag-name: unit tests | |
| - name: Package | |
| run: | | |
| dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo | |
| dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo | |
| dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo | |
| - name: BundleSource | |
| shell: bash | |
| run: | | |
| mkdir -p Tools/BundleUpSourceIntoZip/output | |
| echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd | |
| perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt | |
| echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd | |
| - name: Sign | |
| shell: bash | |
| run: | | |
| signtool=(/c/Program\ Files\ \(x86\)/Windows\ Kits/10/bin/*/x64/signtool.exe) | |
| signtool=${signtool[${#signtool[@]}-1]} | |
| signtool=`echo $signtool | sed -e 's#^/c#c:#' | tr / \\\\` | |
| echo ${{ secrets.DIGICERT_PFX }} | base64 --decode > GitHubActionsWorkflow.pfx | |
| echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} PublishWindows/*.exe PublishWinForms/*.exe' | cmd | |
| mkdir -p dist | |
| cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }} | |
| echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} dist/rdmp.msi' | cmd | |
| (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd) | |
| (cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd) | |
| mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | |
| echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd | |
| (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd) | |
| - name: Install Perl dependencies | |
| uses: shogo82148/[email protected] | |
| with: | |
| install-modules-with: cpanm | |
| install-modules: Archive::Zip Archive::Tar | |
| - name: Fix execute permissions | |
| shell: perl {0} | |
| run: | | |
| use strict; | |
| use warnings; | |
| use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); | |
| use Archive::Tar; | |
| my ($tarname,$zipname)=('dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar','dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip'); | |
| # Find rdmp in .tar, set permissions 0755 | |
| my $tar=Archive::Tar->new($tarname) || die "read $tarname:$!\n"; | |
| my @tarbits=$tar->get_files(); | |
| foreach my $bit (@tarbits) { | |
| $bit->chmod('0755') if $bit->name =~ /\/rdmp$/; | |
| } | |
| $tar->write( $tarname ); | |
| # Find rdmp in .zip, set permissions 0755 | |
| my $srczip = Archive::Zip->new($zipname); | |
| my $zip=Archive::Zip->new(); | |
| foreach my $bit ($srczip->members()) { | |
| print "Adding ",$bit->fileName(),"\n"; | |
| my $m=$zip->addString($bit->contents(),$bit->fileName()); | |
| $m->fileName($bit->fileName()); | |
| my $iszip=$bit->fileName() =~ /\.zip$/i; | |
| $m->desiredCompressionMethod($iszip ? COMPRESSION_STORED : COMPRESSION_DEFLATED); | |
| $m->desiredCompressionLevel($iszip ? 0 : 9 ); | |
| } | |
| $zip->memberNamed('rdmp')->unixFileAttributes( 0755 ); | |
| $zip->overwriteAs($zipname); | |
| - name: Compress tar | |
| run: | | |
| 7z a -txz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar.xz dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar | |
| rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar | |
| - name: Build Nuget packages | |
| shell: bash | |
| run: | | |
| for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj | |
| do | |
| dotnet pack $i -c Release --include-symbols --nologo -o . -v:m -p:Version=${{ steps.version.outputs.rdmpversion }} | |
| done | |
| - name: Upload Nuget packages | |
| if: contains(github.ref, 'refs/tags/v') | |
| run: dotnet nuget push HIC.RDMP.Plugin*${{ steps.version.outputs.rdmpversion }}.nupkg -s https://api.nuget.org/v3/index.json --skip-duplicate -k ${{ secrets.NUGET_KEY }} | |
| - name: Calculate SHA256SUMS | |
| run: '&{foreach ($i in Get-ChildItem dist -Exclude *SUMS|Get-FileHash) { echo "$($i.Hash) $(echo $i | Split-Path -Leaf)" }} > dist/SHA256SUMS' | |
| - name: Archive production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: | | |
| dist | |
| - name: Upload binaries to release | |
| uses: svenstaro/[email protected] | |
| if: contains(github.ref, 'refs/tags/v') | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: dist/* | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| file_glob: true |