diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..657dccb --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,31 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu +{ + "name": "Ubuntu", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "ghcr.io/devcontainers/features/go:1": { + "version": "1.20" + }, + "ghcr.io/eitsupi/devcontainer-features/jq-likes:1": { + "jqVersion": "os-provided", + "yqVersion": "latest", + "gojqVersion": "latest" + } + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "sudo mkdir /data;sudo chmod a+rw /data; cp -r ${containerWorkspaceFolder}/data/* /data/." + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/test.Dockerfile b/.devcontainer/test.Dockerfile new file mode 100644 index 0000000..be4f28e --- /dev/null +++ b/.devcontainer/test.Dockerfile @@ -0,0 +1,17 @@ +ARG GO_VERSION=1.20.4 +ARG ALPINE_VERSION=3.16 + +FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=1000 + +# Setup user +RUN adduser $USERNAME -s /bin/sh -D -u $USER_UID $USER_GID && \ + mkdir -p /etc/sudoers.d && \ + echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \ + chmod 0440 /etc/sudoers.d/$USERNAME + +# Install packages and Go language server +RUN apk add -q --update --progress --no-cache git sudo openssh-client zsh +RUN go get -u -v golang.org/x/tools/cmd/gopls 2>&1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..01fffd3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/push-to-container-registry.yml b/.github/workflows/push-to-container-registry.yml new file mode 100644 index 0000000..8b20384 --- /dev/null +++ b/.github/workflows/push-to-container-registry.yml @@ -0,0 +1,45 @@ +name: Push to Container Registry + +on: + push: + branches: + - main + tags: + - 'v*' + +env: + IMAGE_NAME: protocol-verifier-http-server + DOCKERFILE_PATH: ./Dockerfile + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set lowercase image name + run: echo "repository_lowercase=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ env.repository_lowercase }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: ${{ env.DOCKERFILE_PATH }} + push: true + # tags: ghcr.io/${{ github.repository }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..19c9363 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,74 @@ +variables: + GL_GOLANG_VERSION: 1.20.4 + +cache: + paths: + - /apt-cache + - /go/src/github.com + - /go/src/golang.org + - /go/src/google.golang.org + - /go/src/gopkg.in + +stages: +- test +- build +- push + +include: +- template: Security/SAST.gitlab-ci.yml + +sast: + stage: test + +.go_before_script: + before_script: + - mkdir -p /go/src/gitlab.com/${CI_PROJECT_PATH} /go/src/_/builds + - cp -r $CI_PROJECT_DIR /go/src/gitlab.com/CI_PROJECT_PATH + - ln -s /go/src/gitlab.com/${CI_PROJECT_PATH} /go/src/_/builds/${CI_PROJECT_NAME} + - make dep + +# unit_tests: +# image: golang:${GL_GOLANG_VERSION} +# extends: +# - .go_before_script +# stage: test +# script: +# - make test + +# lint_code: +# image: golang:${GL_GOLANG_VERSION} +# extends: +# - .go_before_script +# stage: test +# script: +# - go get -u golang.org/x/lint/golint +# - make lint + +build: + image: golang:${GL_GOLANG_VERSION} + extends: + - .go_before_script + stage: build + script: + - make build + - ls + artifacts: + when: on_success + paths: + - protocol-verifier-http-server + expire_in: "1 days" + rules: + - if: $CI_COMMIT_TAG + +push: + stage: push + image: + name: gcr.io/kaniko-project/executor:v1.9.0-debug + entrypoint: [""] + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" + rules: + - if: $CI_COMMIT_TAG \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..78cfa1b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# first stage to build the go binary +FROM golang:bullseye as firstStage +RUN apt-get update && apt-get -y install make +RUN mkdir /tmp/http_server +WORKDIR /tmp/http_server +COPY . /tmp/http_server/ +RUN make dep && make build + +# second stage to build the app +FROM ubuntu:latest + +RUN mkdir -p /server/{bin,templates} +RUN mkdir -p /data/{aeo_svdc_config/job_definitions,aerconfig,events,logs/verifier,logs/reception,verifier_processed,verifier_incoming,invariant_store,job_id_store} + +COPY --from=firstStage /tmp/http_server/protocol-verifier-http-server /server/bin/protocol-verifier-http-server +WORKDIR /server +RUN chmod +x /server/bin/protocol-verifier-http-server +COPY /templates/index.html /server/templates/index.html + +EXPOSE 9000 + +ENV PATH=$PATH:/server/bin + +ENTRYPOINT ["protocol-verifier-http-server"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9b571da --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +PROJECT_NAME := "protocol-verifier-http-server" +PKG := gitlab.com/smartdcs1/cdsdt/$(PROJECT_NAME) +PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/) +GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go) + +.PHONY: all dep build #test lint + +# lint: ## Lint the files +# @golint -set_exit_status ./... + +# test: ## Run unittests +# @go test -short ${PKG_LIST} + +dep: ## Get the dependencies + @go get -v -d ./... + +build: dep ## Build the binary file + @go build -v $(PKG) + +help: ## Display this help screen + @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' \ No newline at end of file diff --git a/README.adoc b/README.adoc new file mode 100644 index 0000000..10a0455 --- /dev/null +++ b/README.adoc @@ -0,0 +1,3 @@ += Charon + +HTTP endpoint utility to ferry files into and out of a test configuration diff --git a/README.md b/README.md new file mode 100644 index 0000000..fc43b1b --- /dev/null +++ b/README.md @@ -0,0 +1,185 @@ +# protocol verifier http server + +## Paths + +### / + +This will be for a webpage front end which is still in development. + +### /health + +The health endpoint will return a `200 OK` when the application is running. + +Example curl +```bash +curl --location --request GET '127.0.0.1:9000/health' +``` + +### /upload + +#### /upload/aer-config + +- Single file upload only + +This path allows you to upload a single `config.json` to define the configuration for the AEReception application + +Example curl +```bash +curl --location --request POST '127.0.0.1:9000/upload/aer-config' \ +--form 'upload=@"/Users/tomgibbs/repo/smartdcs/cdsdt/protocol-verifier-http-server/testFiles/aereception/config.json"' +``` + +#### /upload/aeo-svdc-config + +- Multi file upload + +This path allows you to upload multiple files following the Protocol Verifiers guide to setting AEO_SVDC configuration. + +Typically this will require a `config.json`. + +Example curl +```bash +curl --location --request POST '127.0.0.1:9000/upload/aeo-svdc-config' \ +--form 'upload=@"/Users/tomgibbs/testFiles/aeo_svdc/config/config.json"' +``` + +#### /upload/job-definitions + +- Multi file upload + +This path allows you to upload multiple files following the Protocol Verifiers guide to setting job definitions. + +Typically this will require job definition json files. + +Example curl +```bash +curl --location --request POST '127.0.0.1:9000/upload/aeo-svdc-config' \ +--form 'upload=@"./FileRequest.json"' \ +--form 'upload=@"./ANDFork_ANDFork_a.json"' +``` + +#### /upload/events + +- Multi file upload + +Allows posting events to AERecption incoming directory for consumption by the protocol verifier. + +Example curl +```bash +curl --location --request POST '127.0.0.1:9000/upload/events' \ +--form 'upload=@"/Users/tomgibbs/repo/smartdcs/cdsdt/protocol-verifier-http-server/testFiles/events/FileRequest_HappyPath.json"' \ +--form 'upload=@"/Users/tomgibbs/repo/smartdcs/cdsdt/protocol-verifier-http-server/testFiles/events/FileRequest_LoopConstraintViolation.json"' +``` + +#### /download/verifierlog + +This POST request path allows the download of a single log from the verifier domain file (zipped or unzipped) given the files name as a JSON request with mime type application/json with JSON of the form +``` +{ + "fileName":"" +} +``` + +Example curl +```bash +curl --location --request POST \ +-d '{"fileName":"Verifier.log"}' \ +-H 'Content-Type: application/json' \ +'127.0.0.1:9000/download/verifierlog' +``` + +#### /download/aerlog + +This POST request path allows the download of a single log from the AER domain file (zipped or unzipped) given the files name as a JSON request with mime type application/json with JSON of the form +``` +{ + "fileName":"" +} +``` + +Example curl +```bash +curl --location --request POST \ +-d '{"fileName":"Reception.log"}' \ +-H 'Content-Type: application/json' \ +'127.0.0.1:9000/download/aerlog' +``` + +#### /download/verifier-log-file-names + +This GET request path responds with a JSON response that contains a list of log files that are in the verifier domain that start with the string "Verifier". The JSON is of the form +``` +{ + "fileName": [ + "Verifier.log", + "Verifier.log.gz" + ] +} +``` + +Example curl +```bash +curl --location --request GET \ +'127.0.0.1:9000/download/verifier-log-file-names' +``` + +#### /download/verifier-log-file-names + +This GET request path responds with a JSON response that contains a list of log files that are in the AER domain that start with the string "Reception". The JSON is of the form +``` +{ + "fileName": [ + "Reception.log", + "Reception.log.gz" + ] +} +``` + +Example curl +```bash +curl --location --request GET \ +'127.0.0.1:9000/download/aer-log-file-names' +``` + +#### /ioTracking/aer-incoming + +GET request path that provides a JSON response with the number of files in the AER incoming folder and a timestamp in UNIX format (i.e. the number of nanoseconds since 1970) the reading was taken at. The JSON response has the following form +``` +{ + "num_files": 1, + "t": 123456 +} +``` + +Example curl +```bash +curl --location --request GET \ +'127.0.0.1:9000/ioTracking/aer-incoming' +``` + +#### /ioTracking/verifier-processed + +GET request path that provides a JSON response with the number of files in the verifier processed folder and a timestamp in UNIX format (i.e. the number of nanoseconds since 1970) the reading was taken at. The JSON response has the following form +``` +{ + "num_files": 1, + "t": 123456 +} +``` + +Example curl +```bash +curl --location --request GET \ +'127.0.0.1:9000/ioTracking/verifier-processed' +``` + +#### /io/cleanup-test + +GET request path that cleans up the folders of the protocol verifier. WARNING should only be used once a test is completely finished. + + +Example curl +```bash +curl --location --request GET \ +'127.0.0.1:9000/io/cleanup-test' +``` \ No newline at end of file diff --git a/data/aeo_svdc_config/.gitkeep b/data/aeo_svdc_config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/aeo_svdc_config/job_definitions/.gitkeep b/data/aeo_svdc_config/job_definitions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/aerconfig/.gitkeep b/data/aerconfig/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/events/.gitkeep b/data/events/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/invariant_store/.gitkeep b/data/invariant_store/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/job_id_store/.gitkeep b/data/job_id_store/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/logs/reception/.gitkeep b/data/logs/reception/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/logs/verifier/.gitkeep b/data/logs/verifier/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/logs/verifier/Verifier.log b/data/logs/verifier/Verifier.log new file mode 100644 index 0000000..25a9179 --- /dev/null +++ b/data/logs/verifier/Verifier.log @@ -0,0 +1 @@ +some logs \ No newline at end of file diff --git a/data/logs/verifier/Verifier0101.zip b/data/logs/verifier/Verifier0101.zip new file mode 100644 index 0000000..2cbf591 --- /dev/null +++ b/data/logs/verifier/Verifier0101.zip @@ -0,0 +1 @@ +zipped file \ No newline at end of file diff --git a/data/verifier_incoming/.gitkeep b/data/verifier_incoming/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/verifier_processed/.gitkeep b/data/verifier_processed/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/doc/notes/README.adoc b/doc/notes/README.adoc new file mode 100644 index 0000000..c93220d --- /dev/null +++ b/doc/notes/README.adoc @@ -0,0 +1,6 @@ += Technical Notes + +Copy a template file from doc/process/templates when creating a new +engineering note. There are templates here for Analysis Note, Design +Note and Implementation Note among others. + diff --git a/doc/process/templates/README.adoc b/doc/process/templates/README.adoc new file mode 100644 index 0000000..ae43fa4 --- /dev/null +++ b/doc/process/templates/README.adoc @@ -0,0 +1,6 @@ += Document Templates + +Copy a template file from here when creating a new engineering note. +There are templates here for Analysis Note, Design Note and Implementation +Note among others. + diff --git a/doc/process/templates/ant_template.adoc b/doc/process/templates/ant_template.adoc new file mode 100644 index 0000000..2a5a9de --- /dev/null +++ b/doc/process/templates/ant_template.adoc @@ -0,0 +1,99 @@ += Title goes here + +xtUML Project Analysis Note + +NOTE: Each section has a description that states the purpose of that section. +Delete these section descriptions before checking in your note. Delete this +note as well. + +== 1 Abstract + +In this section, give a summary of the design that this note aims to +describe. + +TIP: This document is marked down with http://asciidoc.org/[AsciiDoc]. +You can get tips from the +https://asciidoctor.org/docs/asciidoc-syntax-quick-reference/[AsciiDoc Quick Reference]. + +== 2 Introduction and Background + +In this section, outline the important points relating to this issue/bug that +the reader would need to know in order to understand the rest of this +document. Here is an example reference to a document reference <>. +Here is an example with named text <>. +Here we refer to the image below, <>. + +.caption text under image +image::localimage.png[image description alternate text,width=25%] + +== 3 Requirements + +This section describes the requirements that need to be satisfied. If there +is an SRS, this section may simply refer to it. Each requirement should be as +short and simple as possible and must be clearly defined. + + Here is some text that + is not formatted and line-wrapped. Do this by simply prepending + a blank to the beginning of each line. + +Here we reference a section heading below, like <<3.2.1 Example sub-item>>. + +=== 3.1 Item +==== 3.1.1 Example sub-item +Paragraphs can start here. + +* Example Bulleted List Element + - Example Sub list item with example named document reference <> + +=== 3.2 Item +==== 3.2.1 Example sub-item + +1. First Example Numbered List Element +2. Second + a. enumerated sub first + b. enumerated sub second +3. third + +== 4 Analysis + +This section is only required if there is no preceding analysis note. If present +it sets out a brief analysis of the problem to be resolved by this design note. + + +== 5 Work Required + +In this section, break out the consequential work (as a numbered list) needed +to meet the requirements specified in the Requirements section. + +=== 5.1 Item + +.Recommended Type Mapping +[options="header"] +|=== +| MASL type | xtUML type recommendation +| boolean | Use native `boolean` +| byte | Create UDT with core type `integer` +| character | Create UDT with core type `integer` +|=== + +== 6 Acceptance Test + +In this section, list the tests that need to be performed in order to +verify that all the requirements are satisfied. + + + +== 7 Document References + +In this section, list all the documents that the reader may need to refer to. +Give the full path to reference a file. + +. [[dr-1]] https://support.onefact.net/issues/NNNNN[NNNNN - headline] +. [[dr-2]] ... +. [[dr-3]] link:../8073_masl_parser/8277_serial_masl_spec.md[Serial MASL (SMASL) Specification] + +--- + +This work is licensed under the Creative Commons CC0 License + +--- diff --git a/doc/process/templates/dnt_template.adoc b/doc/process/templates/dnt_template.adoc new file mode 100644 index 0000000..bee27ab --- /dev/null +++ b/doc/process/templates/dnt_template.adoc @@ -0,0 +1,126 @@ += Title goes here + +xtUML Project Design Note + +NOTE: Each section has a description that states the purpose of that section. +Delete these section descriptions before checking in your note. Delete this +note as well. + +== 1 Abstract + +In this section, give a summary of the design that this note aims to +describe. + +TIP: This document is marked down with http://asciidoc.org/[AsciiDoc]. +You can get tips from the +https://asciidoctor.org/docs/asciidoc-syntax-quick-reference/[AsciiDoc Quick Reference]. + +== 2 Introduction and Background + +In this section, outline the important points relating to this issue/bug that +the reader would need to know in order to understand the rest of this +document. Here is an example reference to a document reference <>. +Here is an example with named text <>. +Here we refer to the image below, <>. + +.caption text under image +image::localimage.png[image description alternate text,width=25%] + +== 3 Requirements + +This section is only required if there is no preceding analysis note. +If present it describes the requirements that need to be satisfied. If there +is an SRS, this section may simply refer to it. Each requirement should be as +short and simple as possible and must be clearly defined. + + Here is some text that + is not formatted and line-wrapped. Do this by simply prepending + a blank to the beginning of each line. + +Here we reference a section heading below, like <<3.2.1 Example sub-item>>. + +=== 3.1 Item +==== 3.1.1 Example sub-item +Paragraphs can start here. + +* example bulleted list item + - Example Sub list item with example named document reference <> + +=== 3.2 Section +==== 3.2.1 Example sub-section + +1. First Example Numbered List Element +2. Second + a. enumerated sub first + b. enumerated sub second +3. third + +== 4 Analysis + +This section is only required if there is no preceding analysis note. If present +it sets out a brief analysis of the problem to be resolved by this design note. + + +== 5 Design + +In this section, describe in detail each step of the Work Required section of +the analysis, how the task will be accomplished, what technologies will +be used, algorithms, etc. + +=== 5.1 Item + +[source,java] +---- + // java code example + public void clearDatabase(IProgressMonitor pm) + { + // clear the corresponding graphics-root's database + OoaofgraphicsUtil.clearGraphicsDatabase(rootId, pm); + + Ooaofooa.getDefaultInstance().fireModelElementUnloaded(this); + } +---- + +==== 5.1.1. Example sub-section + +== 6 Design Comments + +If research carried out during this phase shows that a requirement stated in the +analysis note is infeasible or needs some modification, enumerate those changes +here. If there was no preceding analysis note, then this section documents any +deviations from the design as presented at the design review. + +.Recommended Type Mapping +[options="header"] +|=== +| MASL type | xtUML type recommendation +| boolean | Use native `boolean` +| byte | Create UDT with core type `integer` +| character | Create UDT with core type `integer` +|=== + + +== 7 User Documentation + +Describe the end user documentation that was added for this change. + +== 8 Unit Test + +Outline all the unit tests that need to pass and describe the method that you +will use to design and perform the tests. + + +== 9 Document References + +In this section, list all the documents that the reader may need to refer to. +Give the full path to reference a file. + +. [[dr-1]] https://support.onefact.net/issues/NNNNN[NNNNN - headline] +. [[dr-2]] ... +. [[dr-3]] link:../8073_masl_parser/8277_serial_masl_spec.md[Serial MASL (SMASL) Specification] + +--- + +This work is licensed under the Creative Commons CC0 License + +--- diff --git a/doc/process/templates/int_template.adoc b/doc/process/templates/int_template.adoc new file mode 100644 index 0000000..6ce4fbd --- /dev/null +++ b/doc/process/templates/int_template.adoc @@ -0,0 +1,136 @@ += Title goes here + +xtUML Project Implementation Note + +NOTE: Each section has a description that states the purpose of that section. +Delete these section descriptions before checking in your note. Delete this +note as well. + +== 1 Abstract + +In this section, give a summary of the design that this note aims to +describe. + +TIP: This document is marked down with http://asciidoc.org/[AsciiDoc]. +You can get tips from the +https://asciidoctor.org/docs/asciidoc-syntax-quick-reference/[AsciiDoc Quick Reference]. + +== 2 Introduction and Background + +In this section, outline the important points relating to this issue/bug that +the reader would need to know in order to understand the rest of this +document. Here is an example reference to a document reference <>. +Here is an example with named text <>. +Here we refer to the image below, <>. + +.caption text under image +image::localimage.png[image description alternate text,width=25%] + +== 3 Requirements + +This section is only required if there is no preceding design note. +If present it describes the requirements that need to be satisfied. If there +is an SRS, this section may refer to it. Each requirement should be as short +and simple as possible and must be clearly defined. + + Here is some text that + is not formatted and line-wrapped. Do this by simply prepending + a blank to the beginning of each line. + +Here we reference a section heading below, like <<3.2.1 Example sub-item>>. + +=== 3.1 Item +==== 3.1.1 Example sub-item +Paragraphs can start here. + +* example bulleted list item + - Example Sub list item with example named document reference <> + +=== 3.2 Section +==== 3.2.1 Example sub-section + +1. First Example Numbered List Element +2. Second + a. enumerated sub first + b. enumerated sub second +3. third + +== 4 Work Required + +Elaborate on each point of the Work Required section of the design note and +describe how you implemented each step. If there is no design note, this +section, breaks out the consequential work (as a numbered list) needed to +meet the requirements specified in the Requirements section. + +== 5 Implementation Comments + +If the design cannot be implemented as written or if it needs some +modification, enumerate the changes to the design in this section. +If there was no preceding design note, then this section documents +any deviations from the implementation as presented at the +pre-implementation engineering review. + +.Recommended Type Mapping +[options="header"] +|=== +| MASL type | xtUML type recommendation +| boolean | Use native `boolean` +| byte | Create UDT with core type `integer` +| character | Create UDT with core type `integer` +|=== + +=== 5.1 Item 1 + +[source,java] +---- + // java code example + public void clearDatabase(IProgressMonitor pm) + { + // clear the corresponding graphics-root's database + OoaofgraphicsUtil.clearGraphicsDatabase(rootId, pm); + + Ooaofooa.getDefaultInstance().fireModelElementUnloaded(this); + } +---- + +==== 5.1.1 Example sub-item + +== 6 Unit Test + +Outline all the unit tests that need to pass and describe the method that you +will use to design and perform the tests. + +. Here is another ordered list element allowing the renderer to do the work. + This may be better for unit test steps that do not get explicitly referenced. +. second one + .. enumerated sub first + .. enumerated sub second +. third + +== 7 User Documentation + +Describe the end user documentation that was added for this change. + +== 8 Code Changes + +- fork/repository: your_fork/repo_name +- branch: your_branch_name_here + +---- + Put the file list here +---- + +== 9 Document References + +In this section, list all the documents that the reader may need to refer to. +Give the full path to reference a file. + +. [[dr-1]] https://support.onefact.net/issues/NNNNN[NNNNN - headline] +. [[dr-2]] ... +. [[dr-3]] link:../8073_masl_parser/8277_serial_masl_spec.md[Serial MASL (SMASL) Specification] + +--- + +This work is licensed under the Creative Commons CC0 License + +--- diff --git a/doc/process/templates/localimage.png b/doc/process/templates/localimage.png new file mode 100644 index 0000000..6d1ce8f Binary files /dev/null and b/doc/process/templates/localimage.png differ diff --git a/doc/process/templates/rvm_template.adoc b/doc/process/templates/rvm_template.adoc new file mode 100644 index 0000000..784bbbe --- /dev/null +++ b/doc/process/templates/rvm_template.adoc @@ -0,0 +1,17 @@ += Title goes here + +xtUML Project Review Minutes + + Reviewed: + Present: + + -- Num Type Who Section Comment + _- 01 min xxx xxx xxx + _- 02 min xxx xxx xxx + _- 03 min xxx xxx xxx + _- 04 min xxx xxx xxx + _- 05 min xxx xxx xxx + + No major observations, a re-review is not required. + (or... Major observations were recorded, a re-review is required.) + diff --git a/downloads/downloads.go b/downloads/downloads.go new file mode 100644 index 0000000..344d973 --- /dev/null +++ b/downloads/downloads.go @@ -0,0 +1,157 @@ +package downloads + +import ( + "errors" + "net/http" + "os" + "strings" + + "github.com/gin-gonic/gin" +) + +const ( + RECEPTION string = "/logs/reception" + VERIFIER string = "/logs/verifier" +) + +var inputMap = map[string]string { + "RECEPTION": RECEPTION, + "VERIFIER": VERIFIER, +} + +func GetStringToStringMap( + key string, + mapper map[string]string, +) (string, error) { + value, ok := mapper[key] + if ok { + return value, nil + } + return "", errors.New("the input key " + key + " does not exist") +} + + +type locationFileNameRequest struct { + Location string `json:"location" binding:"required"` + FilePrefix string `json:"file_prefix" binding:"required"` +} + + +type fileRequest struct { + FileName string `json:"fileName" binding:"required"` + Location string `json:"location"` +} + + +func BindContextBodyJsonToFileRequest( + ctx *gin.Context, +) (fileRequest, error) { + // Add file to request + var fileRequestObj fileRequest + + // Bind the JSON Data to the fileRequest structure + err := ctx.BindJSON(&fileRequestObj) + return fileRequestObj, err +} + +// Handle bad requests +func HandleBadRequest(err error, ctx *gin.Context) { + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + return + } +} + + +// Add file to request +func PostFileRequest(dirPath string, ctx *gin.Context) { + var fileNameObj fileRequest + fileNameObj, err := BindContextBodyJsonToFileRequest( + ctx, + ) + HandleBadRequest(err, ctx) + // get file path + filePath := dirPath + "/" + fileNameObj.FileName + ctx.File(filePath) +} + + +func GetFileNames( + dirPath string, + fileSubString string, + ctx *gin.Context) { + // Check files in directory with given file prefix + // and send as json response + fileNamesObjs, err := os.ReadDir(dirPath) + if err != nil { + ctx.String(http.StatusInternalServerError, "Request error: %s", err.Error()) + return + } + fileNames := []string{} + for _, fileNameObj := range fileNamesObjs { + fileName := fileNameObj.Name() + // Filter out filenames that do not contain the given substring + if strings.HasPrefix(fileName, fileSubString) { + fileNames = append(fileNames, fileName) + } + } + //fileNameString := "[" + strings.Join(fileNames, ",") + "]" + jsonResponse := gin.H{"fileNames": fileNames} + ctx.JSON(200, jsonResponse) +} + +func HandleGetLogFileLocation( + dataPath string, + location string, +) (string, error) { + + folderLocation, err := GetStringToStringMap( + location, + inputMap, + ) + if err != nil { + return "", err + } + dirPath := dataPath + folderLocation + return dirPath, nil +} + +func HandleGetLogFileNames( + dataPath string, + ctx *gin.Context, +) { + var locationFileObj locationFileNameRequest + // Bind the JSON Data to the fileRequest structure + err := ctx.BindJSON(&locationFileObj) + HandleBadRequest(err, ctx) + dirPath, err := HandleGetLogFileLocation( + dataPath, + locationFileObj.Location, + ) + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + return + } + GetFileNames( + dirPath, + locationFileObj.FilePrefix, + ctx, + ) +} + +func HandleDownloadLogFile( + dataPath string, + ctx *gin.Context, +) { + fileRequestObj, err := BindContextBodyJsonToFileRequest( + ctx, + ) + HandleBadRequest(err, ctx) + dirPath, err := HandleGetLogFileLocation( + dataPath, + fileRequestObj.Location, + ) + HandleBadRequest(err, ctx) + filePath := dirPath + "/" + fileRequestObj.FileName + ctx.File(filePath) +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c329875 --- /dev/null +++ b/go.mod @@ -0,0 +1,38 @@ +module gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server + +go 1.20 + +require ( + github.com/gin-gonic/gin v1.9.0 + github.com/stretchr/testify v1.8.1 +) + +require ( + github.com/bytedance/sonic v1.8.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.11.2 // indirect + github.com/goccy/go-json v0.10.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.9 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.5.0 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.9.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..eec5b2f --- /dev/null +++ b/go.sum @@ -0,0 +1,102 @@ +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= +github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= +github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= +github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= +github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.9.0 h1:CtBMYmb33qYal6XpayZzNXlyK/3FpZV8bDq4CZo57b8= +golang.org/x/tools v0.9.0/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/ioTracking/ioTracking.go b/ioTracking/ioTracking.go new file mode 100644 index 0000000..66eda10 --- /dev/null +++ b/ioTracking/ioTracking.go @@ -0,0 +1,113 @@ +package ioTracking + +import ( + "os" + "time" + "net/http" + "path/filepath" + "strings" + + "github.com/gin-gonic/gin" +) + +func GetNumFiles( + dir *os.File, + ctx *gin.Context) { + // Check number of files in directory + // and send as json response with unix time + fileNamesObjs, err := dir.Readdirnames(0) + if err != nil { + ctx.String(http.StatusInternalServerError, "Request error: %s", err.Error()) + return + } + num_files := len(fileNamesObjs) + t := time.Now().Unix() + jsonResponse := gin.H{ + "num_files": num_files, + "t": t, + } + ctx.JSON(200, jsonResponse) +} + +func CleanFolders( + dirPaths []string, + suffixes []string, +) error { + // Clean given folders of files with given suffix + + for _, dirPath := range dirPaths { + for _, suffix := range suffixes { + dirPathSuffix := dirPath + "/*" + suffix + filePaths, err := filepath.Glob( + dirPathSuffix, + ) + if err != nil { + return err + + } + for _, filePath := range filePaths { + _, fileName := filepath.Split(filePath) + if !strings.HasPrefix(fileName, ".") { + err := os.RemoveAll(filePath) + if err != nil { + return err + } + } + } + } + } + return nil +} + +type RestartPVStruct struct { + WaitTime int64 `json:"wait_time"` +} + +func CleanWaitCleanWait( + dirPaths []string, + suffixes []string, + waitTime int64, +) error { + // Function clean folders wait for a timeout and the clean and wait again + err := CleanFolders( + dirPaths, + suffixes, + ) + if err != nil { + return err + } + time.Sleep(time.Duration(waitTime) * time.Second) + err = CleanFolders( + dirPaths, + suffixes, + ) + if err != nil { + return err + } + time.Sleep(time.Duration(waitTime) * time.Second) + return nil +} + +func RestartPV( + dirPaths []string, + suffixes []string, + ctx *gin.Context, +) { + // Function to restart the Protocol Verifier + var waitTimeObj RestartPVStruct + + // Bind the JSON Data to the fileRequest structure + err := ctx.BindJSON(&waitTimeObj) + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + } + err = CleanWaitCleanWait( + dirPaths, + suffixes, + waitTimeObj.WaitTime, + ) + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + } + ctx.String(http.StatusOK, "Folders Cleaned Successfully") +} \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..08c0a6b --- /dev/null +++ b/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "flag" + "path/filepath" + + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/routers" +) + + +func main() { + // Define runtime flags + dataPath := flag.String("path", "/data", "File storage directory") + // Parse flags + flag.Parse() + dataPathString := *dataPath + templates_full_path, err := filepath.Abs("./templates") + if err != nil { + panic(err) + } + router := routers.SetupRouter( + dataPathString, + templates_full_path, + ) + + router.Run(":9000") + +} \ No newline at end of file diff --git a/routers/routers.go b/routers/routers.go new file mode 100644 index 0000000..358f367 --- /dev/null +++ b/routers/routers.go @@ -0,0 +1,187 @@ +package routers + +import ( + "os" + "net/http" + + "github.com/gin-gonic/gin" + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/uploads" + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/downloads" + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/ioTracking" +) + +func SetupRouter( + dataPathString string, + templates_path string, +) *gin.Engine { + + + // Setup gin router + router := gin.Default() + templates_glob := templates_path + "/*" + router.LoadHTMLGlob(templates_glob) + router.MaxMultipartMemory = 8 << 20 // 8MiB + + // Health Check. + router.GET("/health", func(ctx *gin.Context) { + ctx.JSON(200, gin.H{"status": "success", "message": "Server is healthy"}) + }) + + // Upload AEReception configuration file. + router.POST("/upload/aer-config", func(ctx *gin.Context) { + path := dataPathString + "/aerconfig/" + uploads.SingleUpload(path, ctx) + }) + + // Upload AEO_SVDC configuration file. + router.POST("/upload/aeo-svdc-config", func(ctx *gin.Context) { + path := dataPathString + "/aeo_svdc_config/" + uploads.MultiUpload(path, ctx) + }) + + // Upload job definition file + router.POST("/upload/job-definitions", func(ctx *gin.Context) { + path := dataPathString + "/aeo_svdc_config/job_definitions/" + uploads.MultiUpload(path, ctx) + }) + + // Upload Events + router.POST("/upload/events", func(ctx *gin.Context) { + path := dataPathString + "/events/" + uploads.MultiUpload(path, ctx) + }) + + // Route to web page. + router.GET("/", func(ctx *gin.Context) { + ctx.HTML(http.StatusOK, "index.html", nil) + }) + + // Get verifier logs + router.POST( + "/download/verifierlog", + func(ctx *gin.Context) { + path := dataPathString + "/logs/verifier" + downloads.PostFileRequest( + path, + ctx, + ) + }, + ) + + // Get all log filenames + router.GET( + "/download/verifier-log-file-names", + func(ctx *gin.Context) { + path := dataPathString + "/logs/verifier" + downloads.GetFileNames( + path, + "Verifier", + ctx, + ) + }, + ) + + // Get aer logs + router.POST( + "/download/aerlog", + func(ctx *gin.Context) { + path := dataPathString + "/logs/reception" + downloads.PostFileRequest( + path, + ctx, + ) + }, + ) + + // Get aer log filenames + router.GET( + "/download/aer-log-file-names", + func(ctx *gin.Context) { + path := dataPathString + "/logs/reception" + downloads.GetFileNames( + path, + "Reception", + ctx, + ) + }, + ) + + // POST request getting log files from specific locations + router.POST( + "/download/log-file-names", + func(ctx *gin.Context) { + downloads.HandleGetLogFileNames( + dataPathString, + ctx, + ) + }, + ) + // Get log file + router.POST( + "/download/log-file", + func(ctx *gin.Context) { + downloads.HandleDownloadLogFile( + dataPathString, + ctx, + ) + }, + ) + + // Get number of files in aer-incoming + router.GET( + "/ioTracking/aer-incoming", + func(ctx *gin.Context) { + path := dataPathString + "/events/" + dir, err := os.Open(path) + if err != nil { + ctx.String(http.StatusInternalServerError, "Request error: %s", err.Error()) + } + ioTracking.GetNumFiles( + dir, + ctx, + ) + }, + ) + + // Get number of files in verifier-processed + router.GET( + "/ioTracking/verifier-processed", + func(ctx *gin.Context) { + path := dataPathString + "/verifier_processed" + dir, err := os.Open(path) + if err != nil { + ctx.String(http.StatusInternalServerError, "Request error: %s", err.Error()) + } + ioTracking.GetNumFiles( + dir, + ctx, + ) + }, + ) + + // Endpoint to clean up protocol verifier folders + router.POST( + "io/cleanup-test", + func (ctx *gin.Context) { + dirPaths := []string { + dataPathString + "/aeo_svdc_config/job_definitions", + dataPathString + "/events", + dataPathString + "/verifier_incoming", + dataPathString + "/verifier_processed", + dataPathString + "/invariant_store", + dataPathString + "/job_id_store", + dataPathString + "/logs/reception", + dataPathString + "/logs/verifier", + } + suffixes := []string { + "", + } + ioTracking.RestartPV( + dirPaths, + suffixes, + ctx, + ) + }, + ) + return router +} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..67ca812 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,166 @@ + + + + + + + + File upload in Golang + + +
+
+
+ +
+
+

AEReception Configuration

+ +
+
+
+
+

AEO_SVDC Configuration

+ +
+
+
+
+

Events

+ +
+
+ + + diff --git a/testFiles/aeo_svdc/config/FileRequest.json b/testFiles/aeo_svdc/config/FileRequest.json new file mode 100644 index 0000000..1f84ec5 --- /dev/null +++ b/testFiles/aeo_svdc/config/FileRequest.json @@ -0,0 +1,88 @@ +{ + "JobDefinitionName": "FileRequest", + "Events": [ + { + "EventName": "RequestFile", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "client", + "SequenceStart": true + }, + { + "EventName": "PacketiseFile", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "server", + "PreviousEvents": [ + { + "PreviousEventName": "RequestFile", + "PreviousOccurrenceId": 1 + } + ], + "DynamicControl" : { + "DynamicControlName" : "numPackets", + "DynamicControlType" : "LOOPCOUNT", + "UserEventType" : "AssemblePacket", + "UserOccurrenceId" : 1 + } + }, + { + "EventName": "AssemblePacket", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "server", + "PreviousEvents": [ + { + "PreviousEventName": "PacketiseFile", + "PreviousOccurrenceId": 1 + }, + { + "PreviousEventName": "TransferPacket", + "PreviousOccurrenceId": 1 + } + ] + }, + { + "EventName": "TransferPacket", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "server", + "PreviousEvents": [ + { + "PreviousEventName": "AssemblePacket", + "PreviousOccurrenceId": 1 + } + ] + }, + { + "EventName": "CorruptedPacketError", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "server", + "IsBreak": true, + "PreviousEvents": [ + { + "PreviousEventName": "AssemblePacket", + "PreviousOccurrenceId": 1 + } + ] + }, + { + "EventName": "TransferResultStatus", + "OccurrenceId": 1, + "SequenceName": "Sequence", + "Application": "server", + "SequenceEnd": true, + "PreviousEvents": [ + { + "PreviousEventName": "TransferPacket", + "PreviousOccurrenceId": 1 + }, + { + "PreviousEventName": "CorruptedPacketError", + "PreviousOccurrenceId": 1 + } + ] + } + ] +} diff --git a/testFiles/aeo_svdc/config/FileRequest_event_data.json b/testFiles/aeo_svdc/config/FileRequest_event_data.json new file mode 100644 index 0000000..6197d32 --- /dev/null +++ b/testFiles/aeo_svdc/config/FileRequest_event_data.json @@ -0,0 +1,12 @@ +[{ + "EventDataName": "numPackets", + "EventDataType": "LOOPCONSTRAINT", + "JobDefinitionName": "FileRequest", + "Events": [ + { + "EventType": "PacketiseFile", + "OccurrenceId": 1, + "EventDataItemName" : "numPackets" + } + ] +}] diff --git a/testFiles/aeo_svdc/config/config.json b/testFiles/aeo_svdc/config/config.json new file mode 100644 index 0000000..e951118 --- /dev/null +++ b/testFiles/aeo_svdc/config/config.json @@ -0,0 +1,21 @@ +{ + "SpecUpdateRate": "PT2M", + "IncomingDirectory": "./incoming", + "ProcessingDirectory": "./processing", + "ProcessedDirectory": "./processed", + "EventThrottleRate": "PT0S", + "ReceptionDeletionTime": "PT10M", + "ConcurrentReceptionLimit": 1, + "MaxOutOfSequenceEvents": 10, + "MaximumJobTime": "PT10M", + "JobCompletePeriod": "PT24H", + "Jobs": [ + { + "JobDefinitionName": "FileRequest", + "JobDeprecated": false, + "JobTypeExpiryDate": "2022-04-11T18:08:00Z", + "StaleAuditEventDuration": "P99W", + "BlockedAuditEventDuration": "PT5M" + } + ] +} diff --git a/testFiles/aereception/config.json b/testFiles/aereception/config.json new file mode 100644 index 0000000..9f57eff --- /dev/null +++ b/testFiles/aereception/config.json @@ -0,0 +1,14 @@ +{ + "SpecUpdateRate": "PT2M", + "IncomingDirectory": "./incoming", + "ProcessingDirectory": "./processing", + "ProcessedDirectory": "./processed", + "OutgoingDirectory": "./outgoing", + "EventThrottleRate": "PT0S", + "ReceptionDeletionTime": "PT10M", + "MaxEventsPerFile": 0, + "ConcurrentReceptionLimit": 1, + "SchemaValidate" : false, + "SchemaValidateFrequency" : 10 + +} diff --git a/testFiles/events/FileRequest_HappyPath.json b/testFiles/events/FileRequest_HappyPath.json new file mode 100644 index 0000000..ea2e262 --- /dev/null +++ b/testFiles/events/FileRequest_HappyPath.json @@ -0,0 +1,89 @@ +[ + {"__comment__": "Request a file, loop 2 times, complete"}, + "client", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "applicationName": "client", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:07:34.000000Z", + "eventType": "RequestFile" + }, + "server", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e148", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:30.000000Z", + "eventType": "PacketiseFile", + "previousEventIds": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "numPackets": { + "dataItemType": "LOOPCOUNT", + "value": 3 + } + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e147", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e148" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:09:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e147" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "76a7886b-d53e-48b0-9463-4aed1bb71044", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:10:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "3ee2df8e-706a-4400-b0cf-86fbb5756b86", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:11:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "76a7886b-d53e-48b0-9463-4aed1bb71044" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "66e7281f-8f2e-4493-bf1d-c0e62e95be6f", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:12:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "3ee2df8e-706a-4400-b0cf-86fbb5756b86" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "be372211-ed09-4a9e-8713-3d965b0dc408", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:13:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "66e7281f-8f2e-4493-bf1d-c0e62e95be6f" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "232921f9-38bb-4df6-a2d1-82e9c8bc485e", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:14:34.000000Z", + "eventType": "TransferResultStatus", + "previousEventIds": "be372211-ed09-4a9e-8713-3d965b0dc408" + } +] diff --git a/testFiles/events/FileRequest_LoopConstraintViolation.json b/testFiles/events/FileRequest_LoopConstraintViolation.json new file mode 100644 index 0000000..4707704 --- /dev/null +++ b/testFiles/events/FileRequest_LoopConstraintViolation.json @@ -0,0 +1,71 @@ +[ + {"__comment__": "Request a file, loop 2 times, complete"}, + "client", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "applicationName": "client", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:07:34.000000Z", + "eventType": "RequestFile" + }, + "server", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e148", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:30.000000Z", + "eventType": "PacketiseFile", + "previousEventIds": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "numPackets": { + "dataItemType": "LOOPCOUNT", + "value": 3 + } + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e147", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e148" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:09:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e147" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "76a7886b-d53e-48b0-9463-4aed1bb71044", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:10:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "3ee2df8e-706a-4400-b0cf-86fbb5756b86", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:11:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "76a7886b-d53e-48b0-9463-4aed1bb71044" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "232921f9-38bb-4df6-a2d1-82e9c8bc485e", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:14:34.000000Z", + "eventType": "TransferResultStatus", + "previousEventIds": "3ee2df8e-706a-4400-b0cf-86fbb5756b86" + } +] diff --git a/testFiles/events/FileRequest_ValidEarlyBreak.json b/testFiles/events/FileRequest_ValidEarlyBreak.json new file mode 100644 index 0000000..8cfec41 --- /dev/null +++ b/testFiles/events/FileRequest_ValidEarlyBreak.json @@ -0,0 +1,89 @@ +[ + {"__comment__": "Request a file, loop 2 times, complete"}, + "client", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "applicationName": "client", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:07:34.000000Z", + "eventType": "RequestFile" + }, + "server", + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e148", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:30.000000Z", + "eventType": "PacketiseFile", + "previousEventIds": "b4156cbb-77f7-4dac-a557-df1ea5b45aeb", + "numPackets": { + "dataItemType": "LOOPCOUNT", + "value": 100 + } + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "4a273552-4709-4c72-863d-1b0adf34e147", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:08:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e148" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:09:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "4a273552-4709-4c72-863d-1b0adf34e147" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "76a7886b-d53e-48b0-9463-4aed1bb71044", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:10:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "1ffcef26-4fb8-4854-ae7a-cc3b5be616d2" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "3ee2df8e-706a-4400-b0cf-86fbb5756b86", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:11:34.000000Z", + "eventType": "TransferPacket", + "previousEventIds": "76a7886b-d53e-48b0-9463-4aed1bb71044" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "66e7281f-8f2e-4493-bf1d-c0e62e95be6f", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:12:34.000000Z", + "eventType": "AssemblePacket", + "previousEventIds": "3ee2df8e-706a-4400-b0cf-86fbb5756b86" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "be372211-ed09-4a9e-8713-3d965b0dc408", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:13:34.000000Z", + "eventType": "CorruptedPacketError", + "previousEventIds": "66e7281f-8f2e-4493-bf1d-c0e62e95be6f" + }, + { + "jobId": "95a3b3a7-fada-41d5-b7e2-902079e810d5", + "eventId": "232921f9-38bb-4df6-a2d1-82e9c8bc485e", + "applicationName": "server", + "jobName": "FileRequest", + "timestamp": "2021-05-13T09:14:34.000000Z", + "eventType": "TransferResultStatus", + "previousEventIds": "be372211-ed09-4a9e-8713-3d965b0dc408" + } +] diff --git a/tests/downloads/downloads_test.go b/tests/downloads/downloads_test.go new file mode 100644 index 0000000..5fb5a42 --- /dev/null +++ b/tests/downloads/downloads_test.go @@ -0,0 +1,54 @@ +package downloads_test + +import ( + "testing" + "os" + + "github.com/stretchr/testify/assert" + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/downloads" +) + +func check(e error) { + if e != nil { + panic(e) + } +} + +// Tests `HandleGetLogFileLocation` +func TestHandleGetLogFileLocation(t *testing.T) { + + dname, err := os.MkdirTemp("", "test_dir") + check(err) + logsDirName := dname + "/logs" + err = os.Mkdir(logsDirName, 0755) + check(err) + receptionDirName := logsDirName + "/reception" + err = os.Mkdir(receptionDirName, 0755) + check(err) + verifierDirName := logsDirName + "/verifier" + err = os.Mkdir(verifierDirName, 0755) + check(err) + receptionLocation, err := downloads.HandleGetLogFileLocation( + dname, + "RECEPTION", + ) + check(err) + assert.Equal(t, receptionDirName, receptionLocation) + verifierLocation, err := downloads.HandleGetLogFileLocation( + dname, + "VERIFIER", + ) + check(err) + assert.Equal(t, verifierDirName, verifierLocation) +} + +// Tests `HandleGetLogFileLocation` when an incorrect key has been provided +func TestHandleGetLogFileLocationIncorrectKey(t *testing.T) { + receptionLocation, err := downloads.HandleGetLogFileLocation( + "", + "RECEPTIONS", + ) + assert.Equal(t, "",receptionLocation) + assert.NotEqual(t, nil, err) + assert.Equal(t, "the input key RECEPTIONS does not exist", err.Error()) +} \ No newline at end of file diff --git a/tests/ioTracking/ioTracking_test.go b/tests/ioTracking/ioTracking_test.go new file mode 100644 index 0000000..3a13ea4 --- /dev/null +++ b/tests/ioTracking/ioTracking_test.go @@ -0,0 +1,103 @@ +package iotracking_test + +import ( + "testing" + "os" + + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/ioTracking" +) + +func check(e error) { + if e != nil { + panic(e) + } +} + +func TestCleanFolders(t *testing.T) { + // Tests clean folders + dname, err := os.MkdirTemp("", "test_dir") + check(err) + f, err := os.CreateTemp(dname, "test_file") + check(err) + dir_paths := []string { + dname, + } + suffixes := []string { + "", + } + ioTracking.CleanFolders( + dir_paths, + suffixes, + ) + _, err = os.Stat(dname) + if os.IsExist(err) { + t.Errorf("test_dir is not in the given path") + } + test_file_path := dname + "/" + f.Name() + _, err = os.Stat(test_file_path) + if os.IsExist(err) { + t.Errorf("test_file was not deleted") + } +} + +func TestCleanFolders_full_sub_folder(t *testing.T) { + // Tests clean folders but with a sub folder with a file in it + dname, err := os.MkdirTemp("", "test_dir") + check(err) + sub_dname, err := os.MkdirTemp(dname, "sub_test_dir") + check(err) + f, err := os.CreateTemp(sub_dname, "test_file") + check(err) + dir_paths := []string { + dname, + } + suffixes := []string { + "", + } + ioTracking.CleanFolders( + dir_paths, + suffixes, + ) + _, err = os.Stat(dname) + if os.IsExist(err) { + t.Errorf("test_dir is not in the given path") + } + _, err = os.Stat(sub_dname) + if os.IsExist(err) { + t.Errorf("sub_test_dir is not in the given path") + } + test_file_path := sub_dname + "/" + f.Name() + _, err = os.Stat(test_file_path) + if os.IsExist(err) { + t.Errorf("test_file was not deleted") + } +} + +func TestCleanWaitCleanWait(t *testing.T) { + // Tests clean folders + dname, err := os.MkdirTemp("", "test_dir") + check(err) + f, err := os.CreateTemp(dname, "test_file") + check(err) + dir_paths := []string { + dname, + } + suffixes := []string { + "", + } + waitTime := int64(1) + ioTracking.CleanWaitCleanWait( + dir_paths, + suffixes, + waitTime, + ) + _, err = os.Stat(dname) + if os.IsExist(err) { + t.Errorf("test_dir is not in the given path") + } + test_file_path := dname + "/" + f.Name() + _, err = os.Stat(test_file_path) + if os.IsExist(err) { + t.Errorf("test_file was not deleted") + } +} \ No newline at end of file diff --git a/tests/routers/routers_test.go b/tests/routers/routers_test.go new file mode 100644 index 0000000..6439470 --- /dev/null +++ b/tests/routers/routers_test.go @@ -0,0 +1,184 @@ +package routers_test + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "encoding/json" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "gitlab.com/smartdcs1/cdsdt/protocol-verifier-http-server/routers" +) + +type jsonFileNames struct { + FileNames []string `json:"fileNames"` +} + +func check(e error) { + if e != nil { + panic(e) + } +} + + +func SetupServerTest() (*gin.Engine, []string) { + dname, err := os.MkdirTemp("", "test_dir") + check(err) + router := routers.SetupRouter( + dname, + "../../templates", + ) + dirNames := []string { + "aeo_svdc_config/job_definitions", + "events", + "verifier_incoming", + "verifier_processed", + "invariant_store", + "job_id_store", + "logs/reception", + "logs/verifier", + } + var filepaths []string + for _, dirName := range dirNames { + s := strings.Split(dirName, "/") + dirpath := dname + for _, dirName := range s { + dirpath = dirpath + "/" + dirName + err = os.Mkdir(dirpath, 0755) + if err != nil { + if err.Error() == "mkdir " + dirpath + ": file exists" { + continue + } + check(err) + } + } + f, err := os.Create(dirpath + "/file" ) + f.WriteString("test file") + check(err) + filepaths = append(filepaths, dirpath + "/" + f.Name()) + } + return router, filepaths +} + + +func TestRestartPV(t *testing.T) { + router, filepaths := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"wait_time":1}`) + req, _ := http.NewRequest( + "POST", + "/io/cleanup-test", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + assert.Equal(t, 200, w.Code) + assert.Equal(t, "Folders Cleaned Successfully", w.Body.String()) + var err error + for _, filepath := range filepaths { + _, err = os.Stat(filepath) + if os.IsExist(err) { + t.Errorf("test_file was not deleted") + } + } +} + +func TestDownloadLogFileNamesRECEPTION(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"location":"RECEPTION","file_prefix":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/log-file-names", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + assert.Equal(t, 200, w.Code) + var j jsonFileNames + err := json.NewDecoder(w.Body).Decode(&j) + check(err) + assert.Contains(t, j.FileNames, "file") +} + +func TestDownloadLogFileNamesVERIFIER(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"location":"VERIFIER","file_prefix":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/log-file-names", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + assert.Equal(t, 200, w.Code) + var j jsonFileNames + err := json.NewDecoder(w.Body).Decode(&j) + check(err) + assert.Contains(t, j.FileNames, "file") +} + +func TestDownloadLogFileNamesIncorrectLocation(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"location":"VERIFIERS","file_prefix":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/log-file-names", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + assert.Equal(t, 400, w.Code) + assert.Equal( + t, + "Request error: the input key VERIFIERS does not exist", + w.Body.String(), + ) +} + +func TestDownloadLogFile(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"location":"VERIFIER","fileName":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/log-file", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + println(w.Body.String()) + assert.Equal(t, 200, w.Code) + assert.Equal(t, "test file", w.Body.String()) +} + +func TestDownloadAerLog(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"fileName":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/aerlog", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + println(w.Body.String()) + assert.Equal(t, 200, w.Code) + assert.Equal(t, "test file", w.Body.String()) +} + +func TestDownloadVerifierLog(t *testing.T) { + router, _ := SetupServerTest() + w := httptest.NewRecorder() + var jsonStr = []byte(`{"fileName":"file"}`) + req, _ := http.NewRequest( + "POST", + "/download/verifierlog", + bytes.NewBuffer(jsonStr), + ) + router.ServeHTTP(w, req) + println(w.Body.String()) + assert.Equal(t, 200, w.Code) + assert.Equal(t, "test file", w.Body.String()) +} \ No newline at end of file diff --git a/uploads/upload.go b/uploads/upload.go new file mode 100644 index 0000000..3d35cf0 --- /dev/null +++ b/uploads/upload.go @@ -0,0 +1,45 @@ +package uploads + +import ( + "fmt" + "net/http" + "path/filepath" + + "github.com/gin-gonic/gin" +) + +func MultiUpload(filePath string, ctx *gin.Context) { + // Multipart form + form, err := ctx.MultipartForm() + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + return + } + files := form.File["upload"] + + for _, file := range files { + fileFull := filePath + file.Filename + if err := ctx.SaveUploadedFile(file, fileFull); err != nil { + ctx.String(http.StatusBadRequest, "Upload File Error: %s", err.Error()) + return + } + } + ctx.String(http.StatusOK, fmt.Sprintf("'Uploaded %d Successfully", len(files))) +} + +func SingleUpload(filePath string, ctx *gin.Context) { + + file, err := ctx.FormFile("upload") + if err != nil { + ctx.String(http.StatusBadRequest, "Request error: %s", err.Error()) + return + } + + fileful := filePath + filepath.Base(file.Filename) + if err := ctx.SaveUploadedFile(file, fileful); err != nil { + ctx.String(http.StatusBadRequest, "Upload file error: `%s`", err.Error()) + return + } + + ctx.String(http.StatusOK, "File %s uploaded successfully ", file.Filename) +}