diff --git a/design/2019-challenges.md b/design/2019-challenges.md deleted file mode 100644 index 6493e7393f..0000000000 --- a/design/2019-challenges.md +++ /dev/null @@ -1,31 +0,0 @@ - -## Notes on 2019 Solution - -## completed-checks function (DONE) -does way too much. -persist straight off the queue into table storage? - partitionKey: school id(uuid) rowKey: checkCode -extract marking to separate service. consider where it stores marks to avoid too many sql connections -consider redis for transient status lookups -should we have a bit flag on the check table, that we can set when check received. this would be a low cost solution to maintaining 'checks not received yet' for teachers. -expiry flag would help with distinction on whether some havent been received in time, or whether we can still wait. -how do we deal with multiple processing of a complete check? - -## expire-prepared-checks function - -redis & cosmos table API support TTL on rows. This would be a much cleaner implementation. -- still using table storage. test expiry time on 2mil prepared checks - -## check-expiry function - -potential query optimisation to have better way to flag the expiry of the restart. - -## census-import - -move to functions-app as long running, and upload file direct to BLOB storage which triggers function. - -## check-started function - -To be retired -implement version construct as per other functions -Q: restarts depend on a check-started being received - is this brittle? Yes, restarts no longer depend on this. -Q: how could we record check-started in a non status related way? separate db / microservice? diff --git a/design/2020.md b/design/2020.md deleted file mode 100644 index 21028331cd..0000000000 --- a/design/2020.md +++ /dev/null @@ -1,130 +0,0 @@ -# Design proposals for 2020 architecture - -## Current questions -- restart: could require an existing receivedCheck to be overwritten / invalidated - - insertOrUpdate rather than insertOrMerge as we want any prior validation/marking data to be removed -- restart: must invalidate existing allocated check -- restart: store on existing check, but indicate pupil is allowed restart (update pupil flags) -- checkConfig: move data and functionality to dedicated microservice/function? -- checkNotifier: mtc_admin columns to support processing errors -- preparedCheck: should this be in redis with TTL? - -### Pupil API -> function -consider converting to function. -input trigger would be http -2nd input binding would be table storage with filter to get message automatically -#### why? -- less moving parts, part of a function app -- no docker/server to maintain -#### why not? -- extra dev work -- aint broke, don't fix it -- serves on average at 130ms~ - -### preparedCheck -> Redis -- TTL -- faster than table storage -- transient data -- significantly lowers demand overall on storage account connections - -### pupil prefs as a service -http function exposing GET/POST/DELETE. -has its own dedicated data store, preferably redis or table storage. -pupil-spa would need credential challenge -#### dependencies -- admin app (get/set prefs) -- pupil spa (set prefs) -- pupil api (get prefs) -#### why? -- isolates data and functionality (true microservice) -- moves demand away from SQL -#### why not? -- extra dev work -- ain't broke, don't fix it -- harder to report on -- breaks existing functionality - -### pupil-login function -- existing function updates check table directly with logged in time -- existing submits pupil-status update message to queue -#### changes -- create table storage 'pupilLogin' - - partitionKey: pupilUUID - - rowKey: checkCode -- remove pupil-status queue -- do not update sql - -Should not influence pupil status, but should serve as a support element to acknowledge when pupil logged in. -illustrates the issue of combining the pupils current status with check status. - -## Decisions - -### check receiver -- message is received from the check-submitted queue - - persisted as-is in an immutable state. - - Compressed 'large' complete check is ~24KB -- record is inserted into receivedCheck table - - partition key: schoolUUID - - record primary key: checkCode - - checkData: compressed archive property value - - dateReceived: current UTC datetime - - dispatches message onto check-notification queue to indicate received - -### check notifier-batch -triggered by message on check-notification service bus queue. -responsible for updating mtc_admin.check with submission progress. -- notification types: - - check-received - - sql.mtc_admin.check.receivedByServerAt is updated to current UTC datetime - - check-complete - - sql.mtc_admin.check.complete set to true - - sql.mtc_admin.check.completedAt set to current UTC datetime - - processing-failure - - sql.mtc_admin.check.processingFailed set to true - - sql.mtc_admin.check.completedAt set to current UTC datetime - - if sql operation fails or check is not found this information is recorded in sqlUpdateError column of receivedCheck record? (decision required) - - the execution timeout of the function must take into consideration the potential for waits / retries on sql operations - -### check-validator function & queue - -- triggered by check-validation bus message -- properties are validated against v2 schema -- message is dispatched to check-marking queue via output binding -- failure submits message to check-notification queue - -Hydrates and validates an entry in the receivedCheck table. -uses complete-check schema to validate check has same top level properties -Records 'validatedAt' in UTC against the entry on completion. -Submits a check-marking message onto the queue after recording validation datetime. -Records a 'validationError' against the entry on failure. -submits 'check-notification' message to indidate validation failure (shares same 'processing failure' outcome with check-marking function) - -#### check-validation queue properties (currently service bus) -- Max size: 5GB -- TTL: 14 days -- Lock duration: 5 minutes -- Duplicate detection: enabled -- Duplicate detection window: 1 day -- Dead lettering on expiration: true -- Sessions/FIFO: false -- Partitioning: false - -### check marker function & queue - - - sb message is received from check-marking queue - - properties are validated against V1 schema (schoolid and checkCode) - - receivedCheck entry is retrieved - - check is marked - - score is recorded in mark column - - if marking fails, error details are stored in markError column and row is updated (shares same 'processing failure' outcome with check-validator function) -Further actions to be defined, but will include signalling that check is ready for PS report and MI. - -#### check-marking queue properties -- Max size: 5GB -- TTL: 14 days -- Lock duration: 5 minutes -- Duplicate detection: enabled -- Duplicate detection window: 1 day -- Dead lettering on expiration: true -- Sessions/FIFO: false -- Partitioning: false diff --git a/design/pin-gen/check-allocation.png b/design/pin-gen/check-allocation.png deleted file mode 100644 index 28c0f11f8e..0000000000 Binary files a/design/pin-gen/check-allocation.png and /dev/null differ diff --git a/design/pin-gen/notes.md b/design/pin-gen/notes.md deleted file mode 100644 index 913bc96ebd..0000000000 --- a/design/pin-gen/notes.md +++ /dev/null @@ -1,25 +0,0 @@ -# overnight pin generation - -![Check Allocation Process](./check-allocation.png "Check allocation Sequence") - -## function apps -- update tsconfig.json, add eslint, add jasmine -- how do we share ts files across multiple function libraries to avoid duplication - -## check allocation -- allocate pins and check form for entire pupil dataset (700k-1mil) every night. -- in a restart situation, we must allow a single update to reallocate the check form. -- no point in regenerating the pin unless business case is presented - -### table storage -- delete and recreate table unsuitable due to deletion returning early and still running. blocking name from being reused -- could be time consuming to purge. - - Prove this with batch delete (TODO) - - max 100 ops per batch 😱 - - batch must operate within one partition - -### redis -- use TTL on allocations, no delete required 🎖 -- takes ~25 minutes to generate 1.25 million check allocations. the poc procedure uses a pre-fetched static check -- must include rebuild option for complete recovery when data lost (just run again via portal?) - diff --git a/design/pin-gen/old-sequence.txt b/design/pin-gen/old-sequence.txt deleted file mode 100644 index 96f6e09224..0000000000 --- a/design/pin-gen/old-sequence.txt +++ /dev/null @@ -1,59 +0,0 @@ -pupil pin controller->checkWindowV2Service: getActiveCheckWindow() -checkWindowV2Service->checkWindowDataService: sqlFindActiveCheckWindow() -checkWindowDataService->sqlService: query() -sqlService->SQLDB: SELECT() -pupil pin controller->businessAvailabilityService: determinePinGenerationEligibility() -businessAvailabilityService->businessAvailabilityService: isPinGenerationAllowed() -pupil pin controller->schoolDataService: sqlFindOneById() -schoolDataService->sqlService: query() -sqlService->SQLDB: SELECT() -pupil pin controller->pinGenerationService: generateSchoolPassword() -pinGenerationService->pinValidator: isActivePin() -pinGenerationService->pinGenerationService: generateCryptoRandomNumber() -pupil pin controller->schoolDataService: sqlUpdate() -schoolDataService->sqlService: modify() -sqlService->SQLDB: SELECT() -pupil pin controller->checkStartService: prepareCheck2() -checkStartService->pinGenerationV2Service: getPupilsEligibleForPinGenerationById() -pinGenerationV2Service->pinGenerationDataService: sqlFindPupilsEligibleForPinGenerationById() -pinGenerationDataService->sqlService: query() -sqlService->SQLDB: SELECT() -checkStartService->setValidationService: validate() -checkStartService->checkWindowDataService: sqlFindOneCurrent() -checkWindowDataService->checkWindowDataService: sqlFindCurrent() -checkWindowDataService->sqlService: query() -sqlService->SQLDB: SELECT() -checkStartService->checkFormService: getAllFormsForCheckWindowByType() -checkFormService->checkFormDataService: sqlFetchSortedActiveFormsByName() -checkFormDataService->sqlService: query() -sqlService->SQLDB: SELECT() -checkStartService->checkDataService: sqlFindAllFormsUsedByPupils() -checkDataService->sqlService: query() -sqlService->SQLDB: SELECT() -checkStartService->checkStartService: initialisePupilCheck() -checkStartService->checkFormService: allocateCheckForm() -checkStartService->pinGenerationService: getPinExpiryTime() -checkStartService->pinGenerationDataService: sqlFindChecksForPupilsByIds() -pinGenerationDataService->sqlService: query() -sqlService->SQLDB: SELECT() -checkStartService->pinGenerationV2Service: checkAndUpdateRestarts() -pinGenerationV2Service->pinGenerationDataService: sqlFindChecksForPupilsById() -pinGenerationDataService->sqlService: query() -sqlService->SQLDB: SELECT() -pinGenerationV2Service->pinGenerationDataService: updatePupilRestartsWithCheckInformation() -pinGenerationDataService->sqlService: modify() -sqlService->SQLDB: UPDATE() -checkStartService->azureQueueService: addMessageAsync(pupilStatus Queue) -checkStartService->checkStartService: prepareCheckQueueMessages() -checkStartService->checkFormAllocationDataService: sqlFindByIdsHydrated() -checkFormAllocationDataService->sqlService: query() -checkStartService->sasTokenService: generateSasToken() -sasTokenService->azureQueueService: generateSharedAccessSignature() -checkStartService->sasTokenService: generateSasToken() -sasTokenService->azureQueueService: generateSharedAccessSignature() -checkStartService->sasTokenService: generateSasToken() -sasTokenService->azureQueueService: generateSharedAccessSignature() -checkStartService->sasTokenService: generateSasToken() -sasTokenService->azureQueueService: generateSharedAccessSignature() -checkStartService->azureQueueService: addMessageAsync(prepareCheck Queue) - diff --git a/design/sql drafts/pupilregisterv2.sql b/design/sql drafts/pupilregisterv2.sql deleted file mode 100644 index dd668fd192..0000000000 --- a/design/sql drafts/pupilregisterv2.sql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE VIEW [mtc_admin].[vewPupilRegisterV2_Preview] AS - SELECT - p.id as pupilId, - p.foreName, - p.middleNames, - p.lastName, - p.urlSlug, - p.dateOfBirth, - p.school_id, - g.id as groupId, - ISNULL(g.name, '-') as groupName, - ps.code as pupilStatusCode, - s.check_id as lastCheckId, - cs.code as lastCheckStatusCode, - s.restart_id as pupilRestartId, - pr.check_id as pupilRestartCheckId - - FROM - [mtc_admin].[pupil] p - INNER JOIN [mtc_admin].[pupilStatus] ps - ON p.pupilStatus_id = ps.id - LEFT JOIN [mtc_admin].[group] g - ON p.group_id = g.id - LEFT JOIN [mtc_admin].[pupilState] s - ON s.pupil_id = p.id - INNER JOIN [mtc_admin].[check] c - ON s.check_id = c.id - INNER JOIN [mtc_admin].[checkStatus] cs - ON c.checkStatus_id = cs.id - INNER JOIN [mtc_admin].[pupilRestart] pr - ON s.restart_id = pr.id diff --git a/design/swaggerAPIs/check-completed.yaml b/design/swaggerAPIs/check-completed.yaml deleted file mode 100644 index 8e19b64b9d..0000000000 --- a/design/swaggerAPIs/check-completed.yaml +++ /dev/null @@ -1,124 +0,0 @@ -swagger: '2.0' -info: - description: Receives completed check payloads - version: 1.1.1 - title: MTC Check API - license: - name: GPL - url: 'https://opensource.org/licenses/GPL-3.0' -tags: - - name: check - description: endpoints that pupil app depends upon -paths: - /api/completed-check: - post: - summary: persists completed pupil check - description: > - By passing in the appropriate options, you can retrieve a set of - questions - consumes: - - application/json - produces: - - application/json - parameters: - - in: body - name: checkPayload - description: The completed check data - schema: - $ref: '#/definitions/CompletedCheck' - responses: - '201': - description: created - '400': - description: Bad Request - schema: - $ref: '#/definitions/Error' - '401': - description: Unauthorised - schema: - $ref: '#/definitions/Error' -definitions: - Error: - type: object - required: - - text - properties: - text: - type: string - example: error - CompletedCheck: - type: object - required: - - answers - - inputs - - audit - - config - - access_token - properties: - answers: - type: array - items: - $ref: '#/definitions/Answer' - inputs: - type: array - items: - $ref: '#/definitions/Inputs' - audit: - type: array - items: - $ref: '#/definitions/Audit' - access_token: - type: string - example: >- - eyJhbGR5cCI6IkpXVCJ9.eyJpc3MiOiJNVEMgQWRtaW4iLCJzdWIiOiI1OTk2ZTg1MDUzODMz.5qnuq3XWsXDC2PAhti4 - config: - $ref: '#/definitions/Config' - Answer: - type: object - properties: - factor1: - type: integer - example: 3 - description: the left hand part of the question - factor2: - type: integer - example: 2 - description: the right hand part of the question - answer: - type: integer - minimum: 1 - example: 1 - description: the answer to the question - Inputs: - type: object - properties: - inputType: - type: string - example: mouse_click - data: - type: string - example: '7' - clientTimestamp: - type: string - example: '2017-08-22 12:53:02.139Z' - Audit: - type: object - properties: - auditType: - type: string - example: Check_Started - clientTimestamp: - type: string - example: '2017-08-22 12:53:02.139Z' - data: - type: object - description: any data structure relevant to the audit entry - Config: - type: object - properties: - loadingTime: - type: integer - questionTime: - type: integer -schemes: - - https diff --git a/design/swaggerAPIs/questions.yaml b/design/swaggerAPIs/questions.yaml deleted file mode 100755 index 9f3189f56b..0000000000 --- a/design/swaggerAPIs/questions.yaml +++ /dev/null @@ -1,108 +0,0 @@ ---- -swagger: "2.0" -info: - description: "MTC Admin API" - version: "1.0.0" - title: "MTC Admin API" - license: - name: "GPL" - url: "https://opensource.org/licenses/GPL-3.0" -host: "virtserver.swaggerhub.com" -basePath: "/GuyHarwood/mtc-admin/1.0.0" -tags: -- name: "check data" - description: "endpoints that pupil app depends upon" -schemes: -- "https" -paths: - /api/questions: - post: - summary: "returns question set for check" - description: "By passing in the appropriate options, you can retrieve a set\ - \ of questions\n" - operationId: "getQuestions" - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - in: "body" - name: "pupilDetails" - description: "Auth Credentials for Pupil taking the check" - required: false - schema: - $ref: "#/definitions/PupilDetails" - responses: - 200: - description: "question set returned" - schema: - $ref: "#/definitions/PupilCheck" - 400: - description: "invalid input" - 401: - description: "unauthorised" - 500: - description: "question set not found for pupil" - x-swagger-router-controller: "Default" -definitions: - PupilDetails: - type: "object" - required: - - "pupilId" - - "schoolId" - properties: - pupilId: - type: "string" - example: "abc12345" - schoolId: - type: "string" - example: "9999a" - PupilCheck: - type: "object" - properties: - questions: - type: "array" - items: - $ref: "#/definitions/Question" - pupil: - $ref: "#/definitions/PupilCheck_pupil" - school: - $ref: "#/definitions/PupilCheck_school" - Question: - type: "object" - properties: - rank: - type: "integer" - example: 1 - minimum: 1 - factor1: - type: "integer" - example: 3 - factor2: - type: "integer" - example: 2 - PupilCheck_pupil: - properties: - firstName: - type: "string" - example: "Morgan" - lastName: - type: "string" - example: "Freeman" - sessionId: - type: "string" - questionTime: - type: "number" - loadingTime: - type: "number" - access_token: - type: "string" - PupilCheck_school: - properties: - Id: - type: "integer" - example: 123456 - name: - type: "string" - example: "Hogwarts School of Witchcraft and Wizardry" - description: "should this be URN?" diff --git a/design/swaggerAPIs/readme.md b/design/swaggerAPIs/readme.md deleted file mode 100644 index a1aa084036..0000000000 --- a/design/swaggerAPIs/readme.md +++ /dev/null @@ -1,14 +0,0 @@ -## Working with Swagger to define API specs - -The cleanest way to work with the local swagger instance is to run version 2.x within a docker image. -Version 3 has known issues, which are yet to be resolved. - -1. Pull and run the docker image... - -`docker pull swaggerapi/swagger-editor` -`docker run -p 80:8080 swaggerapi/swagger-editor` - -2. Browse to http://localhost -3. Load swagger yaml file from `design/swaggerAPIs/` into editor via file menu -4. Make your edits -5. Download YAML via file menu, or copy directly from the editor and paste into the relevant project file diff --git a/docs/architecture/MTC Functional Architecture.drawio b/docs/architecture/MTC Functional Architecture.drawio index 5d68c241eb..525b0c760a 100644 --- a/docs/architecture/MTC Functional Architecture.drawio +++ b/docs/architecture/MTC Functional Architecture.drawio @@ -1,6 +1,6 @@ - + - + @@ -154,12 +154,6 @@ - - - - - - @@ -178,34 +172,44 @@ - + + + - + + + - + + + - + + + - + + + @@ -245,133 +249,136 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + diff --git a/func-consumption/check-marker/function.json b/func-consumption/check-marker/function.json deleted file mode 100644 index 46cc4adf6a..0000000000 --- a/func-consumption/check-marker/function.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "markCheckMessage", - "queueName": "check-marking", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "in", - "type": "table", - "name": "receivedCheckTable", - "tableName": "receivedCheck", - "connection": "AZURE_STORAGE_CONNECTION_STRING", - "filter": "(PartitionKey eq '{schoolUUID}') and (RowKey eq '{checkCode}')", - "take": "1" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkNotificationQueue", - "queueName": "check-notification", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "table", - "name": "checkResultTable", - "tableName": "checkResult", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-marker/index.js" -} diff --git a/func-consumption/check-notifier-batch/function.json b/func-consumption/check-notifier-batch/function.json deleted file mode 100644 index d0e72d03ff..0000000000 --- a/func-consumption/check-notifier-batch/function.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "bindings": [ - { - "name": "timer", - "type": "timerTrigger", - "direction": "in", - "schedule": "*/30 * * * * *" - } - ], - "scriptFile": "../dist/functions/check-notifier-batch/index.js" -} diff --git a/func-consumption/check-pin-expiry/function.json b/func-consumption/check-pin-expiry/function.json deleted file mode 100644 index e9cf200978..0000000000 --- a/func-consumption/check-pin-expiry/function.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "bindings": [ - { - "_ref": "sec, min, hour, day, month, dow", - "name": "checkPinExpiry", - "type": "timerTrigger", - "direction": "in", - "schedule": "0 10 18,4 * * *" - } - ], - "scriptFile": "../dist/functions/check-pin-expiry/index.js" -} diff --git a/func-consumption/check-receiver-sb/function.json b/func-consumption/check-receiver-sb/function.json deleted file mode 100644 index 6f9bb7bc25..0000000000 --- a/func-consumption/check-receiver-sb/function.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "submittedCheckQueue", - "queueName": "check-submission", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkValidationQueue", - "queueName": "check-validation", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkNotificationQueue", - "queueName": "check-notification", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-receiver-sb/index.js" -} diff --git a/func-consumption/check-receiver/function.json b/func-consumption/check-receiver/function.json deleted file mode 100644 index b4a7406aa4..0000000000 --- a/func-consumption/check-receiver/function.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "queueTrigger", - "name": "submittedCheck", - "queueName": "check-submitted", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkValidationQueue", - "queueName": "check-validation", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkNotificationQueue", - "queueName": "check-notification", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-receiver/index.js" -} diff --git a/func-consumption/check-started/function.json b/func-consumption/check-started/function.json deleted file mode 100644 index 06db4f8f39..0000000000 --- a/func-consumption/check-started/function.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "disabled": false, - "bindings": [ - { - "name": "checkStartedMessage", - "type": "queueTrigger", - "direction": "in", - "queueName": "check-started", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-started/index.js" -} diff --git a/func-consumption/check-sync/function.json b/func-consumption/check-sync/function.json deleted file mode 100644 index 795e25f4e9..0000000000 --- a/func-consumption/check-sync/function.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "bindings": [ - { - "name": "preparedCheckSyncMessage", - "type": "serviceBusTrigger", - "direction": "in", - "queueName": "check-sync", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-sync/index.js" -} diff --git a/func-consumption/check-validator/function.json b/func-consumption/check-validator/function.json deleted file mode 100644 index 4a4d4ea90a..0000000000 --- a/func-consumption/check-validator/function.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "validateCheckMessage", - "queueName": "check-validation", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "in", - "type": "table", - "name": "receivedCheckTable", - "tableName": "receivedCheck", - "connection": "AZURE_STORAGE_CONNECTION_STRING", - "filter": "(PartitionKey eq '{schoolUUID}') and (RowKey eq '{checkCode}')", - "take": "1" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkMarkingQueue", - "queueName": "check-marking", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkNotificationQueue", - "queueName": "check-notification", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/check-validator/index.js" -} diff --git a/func-consumption/host.json b/func-consumption/host.json index 0b8dd3694c..9d5a44e183 100644 --- a/func-consumption/host.json +++ b/func-consumption/host.json @@ -29,6 +29,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[2.*, 3.0.0)" + "version": "[4.0.0, 5.0.0)" } } diff --git a/func-consumption/package.json b/func-consumption/package.json index 03f66d27bb..25315077eb 100644 --- a/func-consumption/package.json +++ b/func-consumption/package.json @@ -3,6 +3,7 @@ "description": "runtime host for functions defined in tslib that are designed to run on a consumption plan in Azure", "version": "0.1.0", "license": "GPL-3.0", + "main": "dist/{functions-util/**/*.js,functions/**/*.js}", "engines": { "node": ">= 18" }, @@ -16,13 +17,13 @@ "clean": "rm -rf ./dist", "start:host": "func start", "start": "yarn start:host", + "dev": "yarn build && yarn start", "start:dev": "concurrently -r 'env $(cat disable-functions.env | grep Azure | xargs ) yarn start' 'gulp watch'" }, "resolutions": { "**/**/lodash": "^4.17.21" }, "devDependencies": { - "@azure/functions": "^1.0.3", "@faker-js/faker": "^8.4.1", "@types/bcryptjs": "^2.4.2", "@types/bluebird": "^3.5.27", @@ -43,8 +44,9 @@ "dependencies": { "@azure/core-asynciterator-polyfill": "^1.0.2", "@azure/data-tables": "^13.2.1", + "@azure/functions": "^4.5.0", "@azure/service-bus": "^7.9.4", - "@azure/storage-blob": "^12.23.0", + "@azure/storage-blob": "~12.18.0", "@azure/storage-queue": "^12.22.0", "adm-zip": "^0.5.12", "applicationinsights": "^2.9.5", @@ -74,7 +76,7 @@ "function-settings": { "FUNCTIONS_WORKER_RUNTIME": "node", "FUNCTIONS_EXTENSION_VERSION": "~4", - "WEBSITE_NODE_DEFAULT_VERSION": "~16", + "WEBSITE_NODE_DEFAULT_VERSION": "~18", "AzureWebJobs.check-notifier.Disabled": "true" } } \ No newline at end of file diff --git a/func-consumption/pupil-feedback/function.json b/func-consumption/pupil-feedback/function.json deleted file mode 100644 index c370e7711a..0000000000 --- a/func-consumption/pupil-feedback/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "name": "feedbackMessage", - "type": "queueTrigger", - "direction": "in", - "queueName": "pupil-feedback", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - }, - { - "type": "table", - "direction": "out", - "name": "feedbackTable", - "tableName": "pupilFeedback", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/pupil-feedback/index.js" -} diff --git a/func-consumption/pupil-login/function.json b/func-consumption/pupil-login/function.json deleted file mode 100644 index bc6d8387d4..0000000000 --- a/func-consumption/pupil-login/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "name": "pupilLoginMessage", - "type": "serviceBusTrigger", - "direction": "in", - "queueName": "pupil-login", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "table", - "name": "pupilEventTable", - "tableName": "pupilEvent", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/pupil-login/index.js" -} diff --git a/func-consumption/pupil-prefs/function.json b/func-consumption/pupil-prefs/function.json deleted file mode 100644 index aed0e14897..0000000000 --- a/func-consumption/pupil-prefs/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "name": "pupilPrefsMessage", - "type": "queueTrigger", - "direction": "in", - "queueName": "pupil-prefs", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "checkSyncQueue", - "queueName": "check-sync", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/pupil-prefs/index.js" -} diff --git a/func-consumption/school-pin-generator/function.json b/func-consumption/school-pin-generator/function.json deleted file mode 100644 index 0875588614..0000000000 --- a/func-consumption/school-pin-generator/function.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "bindings": [ - { - "name": "timer", - "type": "timerTrigger", - "direction": "in", - "schedule-description": "every hour, on the hour", - "schedule": "0 0 * * * *" - } - ], - "scriptFile": "../dist/functions/school-pin-generator/index.js" -} diff --git a/func-consumption/util-compress-b64/function.json b/func-consumption/util-compress-b64/function.json deleted file mode 100644 index 7db9e2fde3..0000000000 --- a/func-consumption/util-compress-b64/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-compress-b64/index.js" -} diff --git a/func-consumption/util-create-taken-check/function.json b/func-consumption/util-create-taken-check/function.json deleted file mode 100644 index 0a647d0399..0000000000 --- a/func-consumption/util-create-taken-check/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "name": "req", - "authLevel": "anonymous", - "type": "httpTrigger", - "direction": "in", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-create-taken-check/index.js" -} diff --git a/func-consumption/util-diag/function.json b/func-consumption/util-diag/function.json deleted file mode 100644 index a487dbe219..0000000000 --- a/func-consumption/util-diag/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "get" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-diag/index.js" -} diff --git a/func-consumption/util-received-check-viewer/function.json b/func-consumption/util-received-check-viewer/function.json deleted file mode 100644 index b66547c5f0..0000000000 --- a/func-consumption/util-received-check-viewer/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "get" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-received-check-viewer/index.js" -} diff --git a/func-consumption/util-replay-check/function.json b/func-consumption/util-replay-check/function.json deleted file mode 100644 index 939ab7c210..0000000000 --- a/func-consumption/util-replay-check/function.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "anonymous", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - }, - { - "type": "queue", - "name": "submittedCheckQueue", - "direction": "out", - "queueName": "check-submitted", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/util-replay-check/index.js" -} diff --git a/func-consumption/util-school-pin-http-service/function.json b/func-consumption/util-school-pin-http-service/function.json deleted file mode 100644 index 4627a07558..0000000000 --- a/func-consumption/util-school-pin-http-service/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-school-pin-http-service/index.js" -} diff --git a/func-consumption/util-school-pin-sampler/function.json b/func-consumption/util-school-pin-sampler/function.json deleted file mode 100644 index 67ded03371..0000000000 --- a/func-consumption/util-school-pin-sampler/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-school-pin-sampler/index.js" -} diff --git a/func-consumption/util-submit-check/function.json b/func-consumption/util-submit-check/function.json deleted file mode 100644 index 4bba092b67..0000000000 --- a/func-consumption/util-submit-check/function.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "bindings": [ - { - "name": "req", - "authLevel": "anonymous", - "type": "httpTrigger", - "direction": "in", - "methods": [ - "post" - ] - }, - { - "type": "http", - "direction": "out", - "name": "res" - }, - { - "name": "submittedCheckQueue", - "type": "queue", - "direction": "out", - "queueName": "check-submitted", - "connection": "AZURE_STORAGE_CONNECTION_STRING" - }, - { - "name": "checkSubmissionQueue", - "type": "serviceBus", - "direction": "out", - "queueName": "check-submission", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions/util-submit-check/index.js" -} diff --git a/func-consumption/util-test-support-api/function.json b/func-consumption/util-test-support-api/function.json deleted file mode 100644 index fd2803e6c0..0000000000 --- a/func-consumption/util-test-support-api/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "disabled": false, - "bindings": [ - { - "authLevel": "function", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ "put" ], - "route": "test-support/{entity:alpha}" - }, - { - "type": "http", - "direction": "out", - "name": "res" - } - ], - "scriptFile": "../dist/functions/util-test-support-api/index.js" -} diff --git a/func-consumption/yarn.lock b/func-consumption/yarn.lock index cd3fbf2dc9..2e68c357b9 100644 --- a/func-consumption/yarn.lock +++ b/func-consumption/yarn.lock @@ -100,6 +100,26 @@ "@azure/core-client" "^1.3.0" "@azure/core-rest-pipeline" "^1.3.0" +"@azure/core-http@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-3.0.4.tgz#024b2909bbc0f2fce08c74f97a21312c4f42e922" + integrity sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/core-util" "^1.1.1" + "@azure/logger" "^1.0.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" + process "^0.11.10" + tslib "^2.2.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.5.0" + "@azure/core-lro@^2.2.0": version "2.7.2" resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" @@ -184,7 +204,15 @@ tslib "^2.2.0" uuid "^8.3.0" -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.2": +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== + dependencies: + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" + +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== @@ -238,10 +266,14 @@ tslib "^2.2.0" uuid "^8.3.0" -"@azure/functions@^1.0.3": - version "1.2.3" - resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-1.2.3.tgz#65765837e7319eedffbf8a971cb2f78d4e043d54" - integrity sha512-dZITbYPNg6ay6ngcCOjRUh1wDhlFITS0zIkqplyH5KfKEAVPooaoaye5mUFnR+WP9WdGRjlNXyl/y2tgWKHcRg== +"@azure/functions@^4.5.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.0.tgz#7aea7b0b4f2ae86dcc4d1663909e846ea2cca066" + integrity sha512-WNCiOHMQEZpezxgThD3o2McKEjUEljtQBvdw4X4oE5714eTw76h33kIj0660ZJGEnxYSx4dx18oAbg5kLMs9iQ== + dependencies: + cookie "^0.6.0" + long "^4.0.0" + undici "^5.13.0" "@azure/identity@^3.4.1": version "3.4.1" @@ -349,21 +381,16 @@ rhea-promise "^3.0.0" tslib "^2.2.0" -"@azure/storage-blob@^12.23.0": - version "12.24.0" - resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" - integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== +"@azure/storage-blob@~12.18.0": + version "12.18.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e" + integrity sha512-BzBZJobMoDyjJsPRMLNHvqHycTGrT8R/dtcTx9qUFcqwSRfGVK9A/cZ7Nx38UQydT9usZGbaDCN75QRNjezSAA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-client" "^1.6.2" - "@azure/core-http-compat" "^2.0.0" + "@azure/core-http" "^3.0.0" "@azure/core-lro" "^2.2.0" "@azure/core-paging" "^1.1.1" - "@azure/core-rest-pipeline" "^1.10.1" - "@azure/core-tracing" "^1.1.2" - "@azure/core-util" "^1.6.1" - "@azure/core-xml" "^1.3.2" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" events "^3.0.0" tslib "^2.2.0" @@ -409,6 +436,11 @@ resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-8.4.1.tgz#5d5e8aee8fce48f5e189bf730ebd1f758f491451" integrity sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg== +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@ioredis/commands@^1.1.1": version "1.2.0" resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" @@ -424,6 +456,11 @@ resolved "https://registry.yarnpkg.com/@microsoft/applicationinsights-web-snippet/-/applicationinsights-web-snippet-1.0.1.tgz#6bb788b2902e48bf5d460c38c6bb7fedd686ddd7" integrity sha512-2IHAOaLauc8qaAitvWS+U931T+ze+7MNWrDHY47IENP5y2UA0vqJDu67kWZDdpCN1fFC77sfgfB+HV7SrKshnQ== +"@opentelemetry/api@^1.0.1": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1": version "1.5.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.5.0.tgz#4ff2709035a9896ec1aa8f5353ba2277737cae5d" @@ -529,6 +566,14 @@ "@types/tedious" "*" tarn "^3.0.1" +"@types/node-fetch@^2.5.0": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "20.12.12" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" @@ -565,6 +610,13 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8" integrity sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== + dependencies: + "@types/node" "*" + "@types/uuid@^9.0.1": version "9.0.8" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba" @@ -1341,7 +1393,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.6.0: +cookie@0.6.0, cookie@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== @@ -3165,6 +3217,11 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + long@^5.2.0: version "5.2.3" resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" @@ -3379,6 +3436,13 @@ node-abort-controller@^3.1.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + normalize-package-data@^2.3.2: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" @@ -4117,6 +4181,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + semver-greatest-satisfied-range@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-1.1.0.tgz#13e8c2658ab9691cb0cd71093240280d36f77a5b" @@ -4606,6 +4675,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" @@ -4631,6 +4705,11 @@ tslib@^2.2.0, tslib@^2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -4744,6 +4823,13 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + union-value@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" @@ -4899,6 +4985,19 @@ w-json@1.3.10, w-json@^1.3.10: resolved "https://registry.yarnpkg.com/w-json/-/w-json-1.3.10.tgz#ac448a19ca22376e2753a684b52369c7b1e83313" integrity sha512-XadVyw0xE+oZ5FGApXsdswv96rOhStzKqL53uSe5UaTadABGkWIg1+DTx8kiZ/VqTZTBneoL0l65RcPe4W3ecw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -5020,6 +5119,19 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" diff --git a/func-ps-report/host.json b/func-ps-report/host.json index cc4ef3734d..83105d0164 100644 --- a/func-ps-report/host.json +++ b/func-ps-report/host.json @@ -44,6 +44,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[2.*, 3.0.0)" + "version": "[4.0.0, 5.0.0)" } } diff --git a/func-ps-report/package.json b/func-ps-report/package.json index 11f22dacc6..69f251d02c 100644 --- a/func-ps-report/package.json +++ b/func-ps-report/package.json @@ -3,6 +3,7 @@ "description": "runtime host for functions defined in tslib that are part of the ps report", "version": "0.1.0", "license": "GPL-3.0", + "main": "dist/functions-ps-report/**/*.js", "engines": { "node": ">= 18" }, @@ -33,8 +34,9 @@ "dependencies": { "@azure/core-asynciterator-polyfill": "^1.0.2", "@azure/data-tables": "^13.2.1", + "@azure/functions": "^4.5.0", "@azure/service-bus": "^7.9.4", - "@azure/storage-blob": "^12.23.0", + "@azure/storage-blob": "~12.18.0", "@azure/storage-queue": "^12.22.0", "adm-zip": "^0.5.12", "applicationinsights": "^2.9.5", @@ -64,6 +66,6 @@ "function-settings": { "FUNCTIONS_WORKER_RUNTIME": "node", "FUNCTIONS_EXTENSION_VERSION": "~4", - "WEBSITE_NODE_DEFAULT_VERSION": "~16" + "WEBSITE_NODE_DEFAULT_VERSION": "~18" } } \ No newline at end of file diff --git a/func-ps-report/ps-report-1-list-schools/function.json b/func-ps-report/ps-report-1-list-schools/function.json deleted file mode 100644 index df22cf784b..0000000000 --- a/func-ps-report/ps-report-1-list-schools/function.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "jobInfo", - "queueName": "ps-report-exec", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "schoolMessages", - "queueName": "ps-report-schools", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "stagingStart", - "queueName": "ps-report-staging-start", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-ps-report/ps-report-1-list-schools/index.js" -} diff --git a/func-ps-report/ps-report-2-pupil-data/function.json b/func-ps-report/ps-report-2-pupil-data/function.json deleted file mode 100644 index 2492edeabc..0000000000 --- a/func-ps-report/ps-report-2-pupil-data/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "name": "psReport2PupilData", - "type": "serviceBusTrigger", - "direction": "in", - "queueName": "ps-report-schools", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "name": "psReportExportOutput", - "type": "serviceBus", - "direction": "out", - "queueName": "ps-report-export", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-ps-report/ps-report-2-pupil-data/index.js" -} diff --git a/func-ps-report/ps-report-3b-stage-csv-file/function.json b/func-ps-report/ps-report-3b-stage-csv-file/function.json deleted file mode 100644 index e2f5bb13d5..0000000000 --- a/func-ps-report/ps-report-3b-stage-csv-file/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "inputData", - "queueName": "ps-report-staging-start", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "outputData", - "queueName": "ps-report-staging-complete", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-ps-report/ps-report-3b-stage-csv-file/index.js" -} diff --git a/func-ps-report/ps-report-4-writer/function.json b/func-ps-report/ps-report-4-writer/function.json deleted file mode 100644 index 8bc63f5897..0000000000 --- a/func-ps-report/ps-report-4-writer/function.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "bindings": [ - { - "name": "inputData", - "type": "serviceBusTrigger", - "direction": "in", - "queueName": "ps-report-staging-complete", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-ps-report/ps-report-4-writer/index.js" -} diff --git a/func-ps-report/yarn.lock b/func-ps-report/yarn.lock index 51070d5b27..2e31dc10e6 100644 --- a/func-ps-report/yarn.lock +++ b/func-ps-report/yarn.lock @@ -100,6 +100,26 @@ "@azure/core-client" "^1.3.0" "@azure/core-rest-pipeline" "^1.3.0" +"@azure/core-http@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-3.0.4.tgz#024b2909bbc0f2fce08c74f97a21312c4f42e922" + integrity sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/core-util" "^1.1.1" + "@azure/logger" "^1.0.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" + process "^0.11.10" + tslib "^2.2.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.5.0" + "@azure/core-lro@^2.2.0": version "2.7.2" resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" @@ -184,7 +204,15 @@ tslib "^2.2.0" uuid "^8.3.0" -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.2": +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== + dependencies: + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" + +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== @@ -238,6 +266,15 @@ tslib "^2.2.0" uuid "^8.3.0" +"@azure/functions@^4.5.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.0.tgz#7aea7b0b4f2ae86dcc4d1663909e846ea2cca066" + integrity sha512-WNCiOHMQEZpezxgThD3o2McKEjUEljtQBvdw4X4oE5714eTw76h33kIj0660ZJGEnxYSx4dx18oAbg5kLMs9iQ== + dependencies: + cookie "^0.6.0" + long "^4.0.0" + undici "^5.13.0" + "@azure/identity@^3.4.1": version "3.4.2" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" @@ -339,21 +376,16 @@ rhea-promise "^3.0.0" tslib "^2.2.0" -"@azure/storage-blob@^12.23.0": - version "12.24.0" - resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" - integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== +"@azure/storage-blob@~12.18.0": + version "12.18.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e" + integrity sha512-BzBZJobMoDyjJsPRMLNHvqHycTGrT8R/dtcTx9qUFcqwSRfGVK9A/cZ7Nx38UQydT9usZGbaDCN75QRNjezSAA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-client" "^1.6.2" - "@azure/core-http-compat" "^2.0.0" + "@azure/core-http" "^3.0.0" "@azure/core-lro" "^2.2.0" "@azure/core-paging" "^1.1.1" - "@azure/core-rest-pipeline" "^1.10.1" - "@azure/core-tracing" "^1.1.2" - "@azure/core-util" "^1.6.1" - "@azure/core-xml" "^1.3.2" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" events "^3.0.0" tslib "^2.2.0" @@ -394,6 +426,11 @@ enabled "2.0.x" kuler "^2.0.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@ioredis/commands@^1.1.1": version "1.2.0" resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" @@ -409,6 +446,11 @@ resolved "https://registry.yarnpkg.com/@microsoft/applicationinsights-web-snippet/-/applicationinsights-web-snippet-1.0.1.tgz#6bb788b2902e48bf5d460c38c6bb7fedd686ddd7" integrity sha512-2IHAOaLauc8qaAitvWS+U931T+ze+7MNWrDHY47IENP5y2UA0vqJDu67kWZDdpCN1fFC77sfgfB+HV7SrKshnQ== +"@opentelemetry/api@^1.0.1": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1": version "1.5.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.5.0.tgz#4ff2709035a9896ec1aa8f5353ba2277737cae5d" @@ -488,6 +530,14 @@ dependencies: "@types/node" "*" +"@types/node-fetch@^2.5.0": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "20.12.12" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" @@ -513,6 +563,13 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8" integrity sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== + dependencies: + "@types/node" "*" + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -1272,7 +1329,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.6.0: +cookie@0.6.0, cookie@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== @@ -3123,6 +3180,11 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + long@^5.2.0: version "5.2.3" resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" @@ -3337,6 +3399,13 @@ node-abort-controller@^3.1.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + normalize-package-data@^2.3.2: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" @@ -4071,6 +4140,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + semver-greatest-satisfied-range@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-1.1.0.tgz#13e8c2658ab9691cb0cd71093240280d36f77a5b" @@ -4551,6 +4625,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" @@ -4571,6 +4650,11 @@ tslib@^2.2.0, tslib@^2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -4689,6 +4773,13 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + union-value@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" @@ -4844,6 +4935,19 @@ w-json@1.3.10, w-json@^1.3.10: resolved "https://registry.yarnpkg.com/w-json/-/w-json-1.3.10.tgz#ac448a19ca22376e2753a684b52369c7b1e83313" integrity sha512-XadVyw0xE+oZ5FGApXsdswv96rOhStzKqL53uSe5UaTadABGkWIg1+DTx8kiZ/VqTZTBneoL0l65RcPe4W3ecw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -4954,6 +5058,19 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" diff --git a/func-throttled/census-import/function.json b/func-throttled/census-import/function.json deleted file mode 100644 index 1762095f64..0000000000 --- a/func-throttled/census-import/function.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "bindings": [ - { - "name": "censusImport", - "type": "blobTrigger", - "direction": "in", - "path": "census" - } - ], - "tracing": { - "consoleLevel": "verbose" - }, - "scriptFile": "../dist/functions-throttled/census-import/index.js" -} diff --git a/func-throttled/host.json b/func-throttled/host.json index 418bb37109..6288f6cfa4 100644 --- a/func-throttled/host.json +++ b/func-throttled/host.json @@ -43,6 +43,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[2.*, 3.0.0)" + "version": "[4.0.0, 5.0.0)" } } diff --git a/func-throttled/package.json b/func-throttled/package.json index 30494fb39d..971b97bf0e 100644 --- a/func-throttled/package.json +++ b/func-throttled/package.json @@ -3,6 +3,7 @@ "description": "runtime host for functions defined in tslib that are designed to run on a throttled app plan in Azure", "version": "0.1.0", "license": "GPL-3.0", + "main": "dist/functions-throttled/**/*.js", "engines": { "node": ">= 18" }, @@ -33,8 +34,9 @@ "dependencies": { "@azure/core-asynciterator-polyfill": "^1.0.2", "@azure/data-tables": "^13.2.1", + "@azure/functions": "^4.5.0", "@azure/service-bus": "^7.9.4", - "@azure/storage-blob": "^12.23.0", + "@azure/storage-blob": "~12.18.0", "@azure/storage-queue": "^12.22.0", "adm-zip": "^0.5.12", "applicationinsights": "^2.9.5", @@ -64,6 +66,6 @@ "function-settings": { "FUNCTIONS_WORKER_RUNTIME": "node", "FUNCTIONS_EXTENSION_VERSION": "~4", - "WEBSITE_NODE_DEFAULT_VERSION": "~16" + "WEBSITE_NODE_DEFAULT_VERSION": "~18" } } \ No newline at end of file diff --git a/func-throttled/school-import/function.json b/func-throttled/school-import/function.json deleted file mode 100644 index f3ff7c5039..0000000000 --- a/func-throttled/school-import/function.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "bindings": [ - { - "name": "schoolImportInput", - "type": "blobTrigger", - "direction": "in", - "path": "school-import/{name}.csv", - "connection": "AzureWebJobsStorage" - }, - { - "name": "schoolImportStdout", - "type": "blob", - "path": "school-import/{DateTime}-{name}-output-log.txt", - "connection": "AzureWebJobsStorage", - "direction": "out" - }, - { - "name": "schoolImportStderr", - "type": "blob", - "path": "school-import/{DateTime}-{name}-error-log.txt", - "connection": "AzureWebJobsStorage", - "direction": "out" - } - ], - "scriptFile": "../dist/functions-throttled/school-import/index.js" -} diff --git a/func-throttled/sync-results-init/function.json b/func-throttled/sync-results-init/function.json deleted file mode 100644 index 2e776e2fa4..0000000000 --- a/func-throttled/sync-results-init/function.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "bindings": [ - { - "name": "syncResultsInit", - "type": "timerTrigger", - "direction": "in", - "timerDescription": "{second} {minute} {hour} {day} {month} {day-of-week}", - "x-prod-schedule": "0 0 17 * * *", - "x-prod-description": "Every Day at 5pm GMT, or 6pm BST", - "schedule": "0 0 17 * * *" - } - ], - "scriptFile": "../dist/functions-throttled/sync-results-init/index.js" -} diff --git a/func-throttled/sync-results-to-sql/function.json b/func-throttled/sync-results-to-sql/function.json deleted file mode 100644 index f232e36be4..0000000000 --- a/func-throttled/sync-results-to-sql/function.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "checkCompletionQueue", - "queueName": "check-completion", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-throttled/sync-results-to-sql/index.js" -} diff --git a/func-throttled/yarn.lock b/func-throttled/yarn.lock index 55922c9990..b419344bac 100644 --- a/func-throttled/yarn.lock +++ b/func-throttled/yarn.lock @@ -100,6 +100,26 @@ "@azure/core-client" "^1.3.0" "@azure/core-rest-pipeline" "^1.3.0" +"@azure/core-http@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-3.0.4.tgz#024b2909bbc0f2fce08c74f97a21312c4f42e922" + integrity sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/core-util" "^1.1.1" + "@azure/logger" "^1.0.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" + process "^0.11.10" + tslib "^2.2.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.5.0" + "@azure/core-lro@^2.2.0": version "2.7.2" resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" @@ -184,7 +204,15 @@ tslib "^2.2.0" uuid "^8.3.0" -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.2": +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== + dependencies: + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" + +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== @@ -238,6 +266,15 @@ tslib "^2.2.0" uuid "^8.3.0" +"@azure/functions@^4.5.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.0.tgz#7aea7b0b4f2ae86dcc4d1663909e846ea2cca066" + integrity sha512-WNCiOHMQEZpezxgThD3o2McKEjUEljtQBvdw4X4oE5714eTw76h33kIj0660ZJGEnxYSx4dx18oAbg5kLMs9iQ== + dependencies: + cookie "^0.6.0" + long "^4.0.0" + undici "^5.13.0" + "@azure/identity@^3.4.1": version "3.4.1" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.1.tgz#18ba48b7421c818ef8116e8eec3c03ec1a62649a" @@ -339,21 +376,16 @@ rhea-promise "^3.0.0" tslib "^2.2.0" -"@azure/storage-blob@^12.23.0": - version "12.24.0" - resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" - integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== +"@azure/storage-blob@~12.18.0": + version "12.18.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e" + integrity sha512-BzBZJobMoDyjJsPRMLNHvqHycTGrT8R/dtcTx9qUFcqwSRfGVK9A/cZ7Nx38UQydT9usZGbaDCN75QRNjezSAA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-client" "^1.6.2" - "@azure/core-http-compat" "^2.0.0" + "@azure/core-http" "^3.0.0" "@azure/core-lro" "^2.2.0" "@azure/core-paging" "^1.1.1" - "@azure/core-rest-pipeline" "^1.10.1" - "@azure/core-tracing" "^1.1.2" - "@azure/core-util" "^1.6.1" - "@azure/core-xml" "^1.3.2" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" events "^3.0.0" tslib "^2.2.0" @@ -394,6 +426,11 @@ enabled "2.0.x" kuler "^2.0.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@ioredis/commands@^1.1.1": version "1.2.0" resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" @@ -409,6 +446,11 @@ resolved "https://registry.yarnpkg.com/@microsoft/applicationinsights-web-snippet/-/applicationinsights-web-snippet-1.0.1.tgz#6bb788b2902e48bf5d460c38c6bb7fedd686ddd7" integrity sha512-2IHAOaLauc8qaAitvWS+U931T+ze+7MNWrDHY47IENP5y2UA0vqJDu67kWZDdpCN1fFC77sfgfB+HV7SrKshnQ== +"@opentelemetry/api@^1.0.1": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1": version "1.5.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.5.0.tgz#4ff2709035a9896ec1aa8f5353ba2277737cae5d" @@ -488,6 +530,14 @@ dependencies: "@types/node" "*" +"@types/node-fetch@^2.5.0": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "20.12.12" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" @@ -505,6 +555,13 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.2.tgz#38ecb64f01aa0d02b7c8f4222d7c38af6316fef8" integrity sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== + dependencies: + "@types/node" "*" + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -1271,7 +1328,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.6.0: +cookie@0.6.0, cookie@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== @@ -3095,6 +3152,11 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + long@^5.2.0: version "5.2.3" resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" @@ -3309,6 +3371,13 @@ node-abort-controller@^3.1.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + normalize-package-data@^2.3.2: version "2.5.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" @@ -4047,6 +4116,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + semver-greatest-satisfied-range@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-1.1.0.tgz#13e8c2658ab9691cb0cd71093240280d36f77a5b" @@ -4536,6 +4610,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" @@ -4556,6 +4635,11 @@ tslib@^2.2.0, tslib@^2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -4669,6 +4753,13 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + union-value@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" @@ -4824,6 +4915,19 @@ w-json@1.3.10, w-json@^1.3.10: resolved "https://registry.yarnpkg.com/w-json/-/w-json-1.3.10.tgz#ac448a19ca22376e2753a684b52369c7b1e83313" integrity sha512-XadVyw0xE+oZ5FGApXsdswv96rOhStzKqL53uSe5UaTadABGkWIg1+DTx8kiZ/VqTZTBneoL0l65RcPe4W3ecw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -4945,6 +5049,19 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" diff --git a/pupil-api/package.json b/pupil-api/package.json index c097dd9980..a8cbd69696 100644 --- a/pupil-api/package.json +++ b/pupil-api/package.json @@ -15,8 +15,9 @@ "dependencies": { "@azure/core-asynciterator-polyfill": "^1.0.2", "@azure/data-tables": "^13.2.1", + "@azure/functions": "^4.5.0", "@azure/service-bus": "^7.9.4", - "@azure/storage-blob": "^12.23.0", + "@azure/storage-blob": "~12.18.0", "@azure/storage-queue": "^12.22.0", "adm-zip": "^0.5.12", "applicationinsights": "^2.9.5", diff --git a/pupil-api/yarn.lock b/pupil-api/yarn.lock index 22a452a3d8..a366175f2a 100644 --- a/pupil-api/yarn.lock +++ b/pupil-api/yarn.lock @@ -113,6 +113,26 @@ "@azure/core-client" "^1.3.0" "@azure/core-rest-pipeline" "^1.3.0" +"@azure/core-http@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-3.0.4.tgz#024b2909bbc0f2fce08c74f97a21312c4f42e922" + integrity sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/core-util" "^1.1.1" + "@azure/logger" "^1.0.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" + process "^0.11.10" + tslib "^2.2.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.5.0" + "@azure/core-lro@^2.2.0": version "2.7.2" resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" @@ -196,6 +216,14 @@ https-proxy-agent "^5.0.0" tslib "^2.2.0" +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== + dependencies: + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" + "@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.1.tgz#352a38cbea438c4a83c86b314f48017d70ba9503" @@ -273,6 +301,15 @@ tslib "^2.2.0" uuid "^8.3.0" +"@azure/functions@^4.5.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.0.tgz#7aea7b0b4f2ae86dcc4d1663909e846ea2cca066" + integrity sha512-WNCiOHMQEZpezxgThD3o2McKEjUEljtQBvdw4X4oE5714eTw76h33kIj0660ZJGEnxYSx4dx18oAbg5kLMs9iQ== + dependencies: + cookie "^0.6.0" + long "^4.0.0" + undici "^5.13.0" + "@azure/identity@^3.4.1": version "3.4.2" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" @@ -374,21 +411,16 @@ rhea-promise "^3.0.0" tslib "^2.2.0" -"@azure/storage-blob@^12.23.0": - version "12.24.0" - resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" - integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== +"@azure/storage-blob@~12.18.0": + version "12.18.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e" + integrity sha512-BzBZJobMoDyjJsPRMLNHvqHycTGrT8R/dtcTx9qUFcqwSRfGVK9A/cZ7Nx38UQydT9usZGbaDCN75QRNjezSAA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-client" "^1.6.2" - "@azure/core-http-compat" "^2.0.0" + "@azure/core-http" "^3.0.0" "@azure/core-lro" "^2.2.0" "@azure/core-paging" "^1.1.1" - "@azure/core-rest-pipeline" "^1.10.1" - "@azure/core-tracing" "^1.1.2" - "@azure/core-util" "^1.6.1" - "@azure/core-xml" "^1.3.2" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" events "^3.0.0" tslib "^2.2.0" @@ -424,6 +456,11 @@ enabled "2.0.x" kuler "^2.0.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@ioredis/commands@^1.1.1": version "1.2.0" resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" @@ -439,6 +476,11 @@ resolved "https://registry.yarnpkg.com/@microsoft/applicationinsights-web-snippet/-/applicationinsights-web-snippet-1.0.1.tgz#6bb788b2902e48bf5d460c38c6bb7fedd686ddd7" integrity sha512-2IHAOaLauc8qaAitvWS+U931T+ze+7MNWrDHY47IENP5y2UA0vqJDu67kWZDdpCN1fFC77sfgfB+HV7SrKshnQ== +"@opentelemetry/api@^1.0.1": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1", "@opentelemetry/api@^1.7.0": version "1.8.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" @@ -563,6 +605,14 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690" integrity sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w== +"@types/node-fetch@^2.5.0": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "20.12.12" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.12.tgz#7cbecdf902085cec634fdb362172dfe12b8f2050" @@ -622,6 +672,13 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== + dependencies: + "@types/node" "*" + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -1339,7 +1396,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.6.0: +cookie@0.6.0, cookie@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== @@ -3221,6 +3278,11 @@ logform@^2.3.2, logform@^2.4.0: safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + long@^5.2.0: version "5.2.3" resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" @@ -3440,6 +3502,13 @@ node-abort-controller@^3.1.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-mocks-http@^1.7.0: version "1.14.1" resolved "https://registry.yarnpkg.com/node-mocks-http/-/node-mocks-http-1.14.1.tgz#6a387ce09229fe545dcc0154d16bc3480618e013" @@ -4155,6 +4224,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + semver-greatest-satisfied-range@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-1.1.0.tgz#13e8c2658ab9691cb0cd71093240280d36f77a5b" @@ -4635,6 +4709,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + triple-beam@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984" @@ -4645,6 +4724,11 @@ tslib@^2.2.0, tslib@^2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-is@^1.6.18, type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -4802,6 +4886,13 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + union-value@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" @@ -4957,6 +5048,19 @@ w-json@1.3.10, w-json@^1.3.10: resolved "https://registry.yarnpkg.com/w-json/-/w-json-1.3.10.tgz#ac448a19ca22376e2753a684b52369c7b1e83313" integrity sha512-XadVyw0xE+oZ5FGApXsdswv96rOhStzKqL53uSe5UaTadABGkWIg1+DTx8kiZ/VqTZTBneoL0l65RcPe4W3ecw== +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -5064,6 +5168,19 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + xtend@~4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" diff --git a/test/admin-hpa/features/add_multiple_pupil_validation_error.feature b/test/admin-hpa/features/add_multiple_pupil_validation_error.feature index 505426dbbd..8e60ebabca 100644 --- a/test/admin-hpa/features/add_multiple_pupil_validation_error.feature +++ b/test/admin-hpa/features/add_multiple_pupil_validation_error.feature @@ -110,11 +110,9 @@ Feature: Add Multiple Pupil validation Error When I attempt to upload a valid CSV file to add Multiple Pupil with a pupil aged 11 Then I should see an error with the upload stating the DOB is invalid - Scenario: 10 year old pupils can be added + Scenario: 10 year old pupils cannot be added When I attempt to upload a valid CSV file to add Multiple Pupil with a pupil aged 10 - Then I should be taken to the Pupil register page - And I should see a flash message for the multiple pupil upload - And I can see the new pupils added to the list + Then I should see an error with the upload stating the DOB is invalid Scenario: 9 year old pupils can be added When I attempt to upload a valid CSV file to add Multiple Pupil with a pupil aged 9 diff --git a/test/admin-hpa/features/support/functions_helper.rb b/test/admin-hpa/features/support/functions_helper.rb index 6fed56778b..d486aa6c9b 100644 --- a/test/admin-hpa/features/support/functions_helper.rb +++ b/test/admin-hpa/features/support/functions_helper.rb @@ -10,7 +10,7 @@ def self.sync_all end def self.generate_school_pin(school_id) - HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-school-pin-http-service", :body => {'school_id' => school_id}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-school-pin-generator", :body => {'school_id' => school_id}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end def self.complete_check_via_check_code(check_code_array) @@ -18,11 +18,11 @@ def self.complete_check_via_check_code(check_code_array) end def self.create_school(lea_code, estab_code, name, urn) - HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/test-support/school", :body => {'leaCode' => lea_code, 'estabCode' => estab_code.to_s, 'name' => name, 'urn' => urn.to_s}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-create-school", :body => {'leaCode' => lea_code, 'estabCode' => estab_code.to_s, 'name' => name, 'urn' => urn.to_s}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end def self.create_user(school_uuid,username) - HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/test-support/user", :body => {'schoolUuid' => school_uuid, 'identifier' => username, 'password' => 'password', 'role' => 'teacher'}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-create-user", :body => {'schoolUUID' => school_uuid, 'identifier' => username, 'password' => 'password', 'role' => 'teacher'}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end end diff --git a/test/admin-hpa/features/support/hooks.rb b/test/admin-hpa/features/support/hooks.rb index 13101127e5..38fc52c301 100644 --- a/test/admin-hpa/features/support/hooks.rb +++ b/test/admin-hpa/features/support/hooks.rb @@ -9,7 +9,6 @@ @school_uuid = @school['urlSlug'] @school_user = SqlDbHelper.get_school_teacher(@urn) @username = @school_user['identifier'] - FunctionsHelper.generate_school_pin(@school_id) p "Login for #{@school_name} created as - #{@username}" step 'I am logged in' diff --git a/test/admin-hpa/features/support/page_objects/add_multiple_pupil_page.rb b/test/admin-hpa/features/support/page_objects/add_multiple_pupil_page.rb index dfdeb15cfb..c5e32b86b2 100644 --- a/test/admin-hpa/features/support/page_objects/add_multiple_pupil_page.rb +++ b/test/admin-hpa/features/support/page_objects/add_multiple_pupil_page.rb @@ -64,7 +64,7 @@ def create_and_upload_multiple_pupils(number,file_name) number.to_i.times do upn = UpnGenerator.generate @upn_list << upn - csv << ["#{(0...8).map { (65 + rand(26)).chr }.join}","#{(0...8).map { (65 + rand(26)).chr }.join}", "","02/09/2013","f",upn] + csv << ["#{(0...8).map { (65 + rand(26)).chr }.join}","#{(0...8).map { (65 + rand(26)).chr }.join}", "","02/09/2015","f",upn] end end page.attach_file('file-upload', File.expand_path("#{File.dirname(__FILE__)}/../../../data/#{file_name}")) diff --git a/test/pupil-hpa/features/support/functions_helper.rb b/test/pupil-hpa/features/support/functions_helper.rb index 74bbc9b835..43d7ad114c 100644 --- a/test/pupil-hpa/features/support/functions_helper.rb +++ b/test/pupil-hpa/features/support/functions_helper.rb @@ -14,15 +14,15 @@ def self.sync_all end def self.create_school(lea_code, estab_code, name, urn) - HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/test-support/school", :body => {'leaCode' => lea_code, 'estabCode' => estab_code.to_s, 'name' => name, 'urn' => urn.to_s}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-create-school", :body => {'leaCode' => lea_code, 'estabCode' => estab_code.to_s, 'name' => name, 'urn' => urn.to_s}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end def self.create_user(school_uuid,username) - HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/test-support/user", :body => {'schoolUuid' => school_uuid, 'identifier' => username, 'password' => 'password', 'role' => 'teacher'}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.put(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-create-user", :body => {'schoolUuid' => school_uuid, 'identifier' => username, 'password' => 'password', 'role' => 'teacher'}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end def self.generate_school_pin(school_id) - HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-school-pin-http-service", :body => {'school_id' => school_id}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) + HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-school-pin-generator", :body => {'school_id' => school_id}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']}) end def self.complete_check_via_check_code(check_code_array) diff --git a/test/se_manager/.cache/selenium/se-metadata.json b/test/se_manager/.cache/selenium/se-metadata.json index 996a35411d..75bb65a14f 100644 --- a/test/se_manager/.cache/selenium/se-metadata.json +++ b/test/se_manager/.cache/selenium/se-metadata.json @@ -1,5 +1,12 @@ { - "browsers": [], + "browsers": [ + { + "browser_name": "chrome", + "major_browser_version": "127", + "browser_version": "127.0.6533.119", + "browser_ttl": 1725372632 + } + ], "drivers": [], "stats": [ { @@ -9,7 +16,7 @@ "arch": "x86_64", "lang": "ruby", "selenium_version": "4.23", - "stats_ttl": 1724162592 + "stats_ttl": 1725372623 } ] } \ No newline at end of file diff --git a/tslib/package.json b/tslib/package.json index 700be1e310..8b66ce9221 100644 --- a/tslib/package.json +++ b/tslib/package.json @@ -25,7 +25,6 @@ }, "description": "Root MTC Typescript project. Contains all shared components and azure function implementations", "devDependencies": { - "@azure/functions": "^3.5.0", "@faker-js/faker": "^8.4.1", "@types/adm-zip": "^0.5.0", "@types/async": "^3.2.7", @@ -62,8 +61,9 @@ "dependencies": { "@azure/core-asynciterator-polyfill": "^1.0.2", "@azure/data-tables": "^13.2.1", + "@azure/functions": "^4.5.0", "@azure/service-bus": "^7.9.4", - "@azure/storage-blob": "^12.23.0", + "@azure/storage-blob": "~12.18.0", "@azure/storage-queue": "^12.22.0", "adm-zip": "^0.5.12", "applicationinsights": "^2.9.5", diff --git a/tslib/src/azure/functions.ts b/tslib/src/azure/functions.ts deleted file mode 100644 index ceb4e47e19..0000000000 --- a/tslib/src/azure/functions.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface IFunctionTimer { - isPastDue: boolean -} diff --git a/tslib/src/common/ContextLike.ts b/tslib/src/common/ContextLike.ts index 84761d0853..8f4b80909c 100644 --- a/tslib/src/common/ContextLike.ts +++ b/tslib/src/common/ContextLike.ts @@ -1,16 +1,13 @@ -import { type ILogger } from './logger' - -/** - * @description same signature as Azure Function ContextBindings - */ -export type IContextBindingsLike = Record - /** * @description Azure Function Context Like object that supports members required * by the Ps Report Logger */ export interface IContextLike { - bindings: IContextBindingsLike - log: ILogger + log (...args: any[]): void + trace (...args: any[]): void + debug (...args: any[]): void + info (...args: any[]): void + warn (...args: any[]): void + error (...args: any[]): void invocationId: string } diff --git a/tslib/src/common/bigint.ts b/tslib/src/common/bigint.ts new file mode 100644 index 0000000000..1bdddfde78 --- /dev/null +++ b/tslib/src/common/bigint.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @typescript-eslint/semi +// @ts-ignore: Unreachable code error +// eslint-disable-next-line no-extend-native +BigInt.prototype.toJSON = function (): string { + return this.toString() +} diff --git a/tslib/src/common/logger.ts b/tslib/src/common/logger.ts index aae44cabd5..4b501a3110 100644 --- a/tslib/src/common/logger.ts +++ b/tslib/src/common/logger.ts @@ -17,7 +17,7 @@ export interface ILogger { /** * Writes to verbose level logging. */ - verbose (...args: any[]): void + trace (...args: any[]): void } export class ConsoleLogger implements ILogger { @@ -33,7 +33,7 @@ export class ConsoleLogger implements ILogger { console.info(...args) } - verbose (...args: any[]): void { + trace (...args: any[]): void { console.log(...args) } } @@ -42,5 +42,5 @@ export class MockLogger implements ILogger { error (): void {} warn (): void {} info (): void {} - verbose (): void {} + trace (): void {} } diff --git a/tslib/src/functions-ps-report/common/ps-report-logger.ts b/tslib/src/functions-ps-report/common/ps-report-logger.ts index 214aa687c0..d782402ad0 100644 --- a/tslib/src/functions-ps-report/common/ps-report-logger.ts +++ b/tslib/src/functions-ps-report/common/ps-report-logger.ts @@ -28,21 +28,21 @@ export class PsReportLogger implements ILogger { info (message: string): void { const formatted = this.log(message, 'info') - this.context.log.info(formatted) + this.context.info(formatted) } - verbose (message: string): void { + trace (message: string): void { const formatted = this.log(message, 'verbose') - this.context.log.verbose(formatted) + this.context.trace(formatted) } warn (message: string): void { const formatted = this.log(message, 'warning') - this.context.log.warn(formatted) + this.context.warn(formatted) } error (message: string): void { const formatted = this.log(message, 'error') - this.context.log.error(formatted) + this.context.error(formatted) } } diff --git a/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts b/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts index 1259967218..875936916d 100644 --- a/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts +++ b/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { type ISchoolMessageSpecification, ListSchoolsService } from './list-schools-service' import { PsReportLogger } from '../common/ps-report-logger' @@ -8,9 +8,27 @@ import { JobStatusCode } from '../../common/job-status-code' import moment from 'moment' import type { PsReportStagingStartMessage, PsReportListSchoolsIncomingMessage } from '../common/ps-report-service-bus-messages' -const serviceBusTrigger: AzureFunction = async function (context: Context, jobInfo: PsReportListSchoolsIncomingMessage): Promise { +const schoolMessagesQueue = output.serviceBusQueue({ + queueName: 'ps-report-schools', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING' +}) + +const stagingStartQueue = output.serviceBusQueue({ + queueName: 'ps-report-staging-start', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING' +}) + +app.serviceBusQueue('psReport1ListSchools', { + queueName: 'ps-report-exec', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + handler: psReport1ListSchools, + extraOutputs: [schoolMessagesQueue, stagingStartQueue] +}) + +export async function psReport1ListSchools (triggerInput: unknown, context: InvocationContext): Promise { const logger = new PsReportLogger(context, PsReportSource.SchoolGenerator) - logger.verbose(`requested at ${jobInfo.dateTimeRequested} by ${jobInfo.requestedBy}`) + const jobInfo = triggerInput as PsReportListSchoolsIncomingMessage + logger.trace(`requested at ${jobInfo.dateTimeRequested} by ${jobInfo.requestedBy}`) const start = performance.now() const meta = { processCount: 0, errorCount: 0 } const jobDataService = new JobDataService() @@ -18,7 +36,7 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn // We need to store a filename for all the data to be written to during the staging process. const now = moment() const filename = `ps-report-staging-${now.format('YYYY-MM-DD-HHmm')}.csv` - await jobDataService.setJobStarted(jobInfo.jobUuid, { meta: { filename } }, context.log) + await jobDataService.setJobStarted(jobInfo.jobUuid, { meta: { filename } }, context) const schoolListService = new ListSchoolsService(logger) const messageSpec: ISchoolMessageSpecification = { jobUuid: jobInfo.jobUuid, @@ -26,7 +44,7 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn urns: jobInfo.urns } const messages = await schoolListService.getSchoolMessages(messageSpec) - context.bindings.schoolMessages = messages + context.extraOutputs.set(schoolMessagesQueue, messages) meta.processCount = messages.length // Send a message to start ps-report-3b-staging (the csv assembly) @@ -36,7 +54,7 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn jobUuid: jobInfo.jobUuid, filename } - context.bindings.stagingStart = stagingStartMessage + context.extraOutputs.set(stagingStartQueue, stagingStartMessage) logger.info(`staging-start message sent: ${JSON.stringify(stagingStartMessage)}`) } catch (error) { let errorMessage = 'unknown error' @@ -52,5 +70,3 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn const durationInMilliseconds = end - start logger.info(`processed ${meta.processCount} records, run took ${durationInMilliseconds} ms`) } - -export default serviceBusTrigger diff --git a/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.ts b/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.ts index c7ab38e226..f60f12e61a 100644 --- a/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.ts +++ b/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.ts @@ -45,7 +45,7 @@ export class ListSchoolsService implements IListSchoolsService { } public async getSchoolMessages (specification: ISchoolMessageSpecification): Promise { - this.logger.verbose('ListSchoolsService called - retrieving all schools') + this.logger.trace('ListSchoolsService called - retrieving all schools') const schools = await this.getSchools(specification.urns) const schoolMessages: PsReportSchoolFanOutMessage[] = schools.map(school => { return { diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts index 95dd02f710..5592c51f8d 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { PsReportService } from './ps-report.service' import { PsReportLogger } from '../common/ps-report-logger' @@ -7,31 +7,42 @@ import type { PsReportSchoolFanOutMessage } from '../common/ps-report-service-bu import config from '../../config' import type { IPsychometricReportLine } from './transformer-models' +const psReportExportOutputQueue = output.serviceBusQueue({ + queueName: 'ps-report-export', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING' +}) + +app.serviceBusQueue('ps-report-2-pupil-data', { + queueName: 'ps-report-schools', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + handler: psReport2PupilData, + extraOutputs: [psReportExportOutputQueue] +}) + +export interface IOutputBinding { + psReportExportOutput: IPsychometricReportLine[] +} + /** * Incoming message is just the name and UUID of the school to process * The name is used for logging * The UUID is used to fetch all pupils for the school */ -export interface IOutputBinding { - psReportExportOutput: IPsychometricReportLine[] -} - -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, incomingMessage: PsReportSchoolFanOutMessage): Promise { +export async function psReport2PupilData (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const incomingMessage = triggerInput as PsReportSchoolFanOutMessage const logger = new PsReportLogger(context, PsReportSource.PupilGenerator) if (config.Logging.DebugVerbosity > 1) { - logger.verbose(`called for school ${incomingMessage.name}`) + logger.trace(`called for school ${incomingMessage.name}`) } - const outputBinding: IOutputBinding = { psReportExportOutput: [] } - context.bindings = outputBinding - const psReportService = new PsReportService(outputBinding, logger) - await psReportService.process(incomingMessage) + + const psReportService = new PsReportService(logger) + const output = await psReportService.process(incomingMessage) + context.extraOutputs.set(psReportExportOutputQueue, output.psReportExportOutput) const end = performance.now() const durationInMilliseconds = end - start if (config.Logging.DebugVerbosity > 1) { - logger.info(`processed ${outputBinding.psReportExportOutput.length} pupils, run took ${durationInMilliseconds} ms`) + logger.info(`processed ${output.psReportExportOutput.length} pupils, run took ${durationInMilliseconds} ms`) } } - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts index 27722382f6..a73a244474 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts @@ -29,7 +29,7 @@ describe.skip('PsReportService', () => { } outputBindings.psReportExportOutput = [] - sut = new PsReportService(outputBindings, logger, psReportDataService) + sut = new PsReportService(logger, psReportDataService) }) test('it is defined', () => { diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts index a63d2d9caf..d7161b3069 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts @@ -1,27 +1,28 @@ import { type IPsReportDataService, PsReportDataService } from './ps-report.data.service' import { type Pupil, type PupilResult, type School } from './pupil-data.models' import { type ILogger } from '../../common/logger' -import { type IOutputBinding } from '.' import type { PsReportSchoolFanOutMessage } from '../common/ps-report-service-bus-messages' import config from '../../config' import { ReportLine } from './report-line.class' +import { type IPsychometricReportLine } from './transformer-models' const logName = 'ps-report-2-pupil-data: PsReportService' +export interface IPsReportServiceOutput { + psReportExportOutput: IPsychometricReportLine[] +} export class PsReportService { private readonly dataService: IPsReportDataService - private readonly outputBinding: IOutputBinding private readonly logger: ILogger - constructor (outputBinding: IOutputBinding, logger: ILogger, dataService?: IPsReportDataService) { - this.outputBinding = outputBinding + constructor (logger: ILogger, dataService?: IPsReportDataService) { this.logger = logger this.dataService = dataService ?? new PsReportDataService(this.logger) } - async process (incomingMessage: PsReportSchoolFanOutMessage): Promise { + async process (incomingMessage: PsReportSchoolFanOutMessage): Promise { if (config.Logging.DebugVerbosity > 1) { - this.logger.verbose(`${logName}.process() called with ${JSON.stringify(incomingMessage)}`) + this.logger.trace(`${logName}.process() called with ${JSON.stringify(incomingMessage)}`) } let pupils: readonly Pupil[] @@ -38,7 +39,9 @@ export class PsReportService { this.logger.error(`ERROR - unable to fetch pupils for school ${incomingMessage.uuid}`) throw error } - + const output: IPsReportServiceOutput = { + psReportExportOutput: [] + } for (let i = 0; i < pupils.length; i++) { const pupil = pupils[i] if (school === undefined) { @@ -60,14 +63,14 @@ export class PsReportService { // Now we have the pupil data we can transform it into the report format const reportLine = new ReportLine(pupilResult.answers, pupilResult.check, pupilResult.checkConfig, pupilResult.checkForm, pupilResult.device, pupilResult.events, pupilResult.pupil, pupilResult.school) const outputData = reportLine.transform() - + output.psReportExportOutput.push(outputData) // Send the transformed pupil data onto the ps-report-export queue using the output bindings. - this.outputBinding.psReportExportOutput.push(outputData) } catch (error: any) { // Ignore the error on the particular pupil and carry on so it reports on the rest of the school this.logger.error(`${logName}: ERROR: Failed to retrieve pupil data for pupil ${pupil.slug} in school ${incomingMessage.uuid} Error was ${error.message}`) } } + return output } } diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts index a0b03b43fe..2c4b87220d 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts @@ -16,7 +16,7 @@ export class CsvTransformer { constructor (logger: ILogger, psReportLineData: IPsychometricReportLine[]) { this.logger = logger this.psReportLineData = psReportLineData - this.logger.verbose('CsvTransformer initialised') + this.logger.trace('CsvTransformer initialised') } private transformAnswer (a: IReportLineAnswer | undefined): any[] { diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts index 8ad9770abc..8aecb58bc1 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import * as sb from '@azure/service-bus' import config from '../../config' @@ -17,18 +17,29 @@ let emptyPollTime: undefined | number const getEpoch = (): number => { const dt = +new Date(); return Math.floor(dt / 1000) } let psReportStagingDataService: PsReportStagingDataService +const stagingCompleteQueueOutput = output.serviceBusQueue({ + queueName: 'ps-report-staging-complete', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING' +}) + +app.serviceBusQueue(functionName, { + queueName: 'ps-report-staging-start', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + handler: PsReportStageCsvFile, + extraOutputs: [stagingCompleteQueueOutput] +}) + /** * The function is running as a singleton, and the receiver is therefore exclusive * we do not expect another receive operation to be in progress. if the message * is abandoned 10 times (the current 'max delivery count') it will be * put on the dead letter queue automatically. - * */ -const PsReportStageCsvFile: AzureFunction = async function (context: Context, incomingMessage: PsReportStagingStartMessage): Promise { - logPrefix = functionName + ': ' + context.invocationId +export async function PsReportStageCsvFile (triggerInput: unknown, context: InvocationContext): Promise { + logPrefix = `${functionName}:${context.invocationId}` context.log(`${logPrefix}: starting`) const start = performance.now() - + const incomingMessage = triggerInput as PsReportStagingStartMessage if (config.ServiceBus.ConnectionString === undefined) { throw new Error(`${logPrefix}: ServiceBusConnection env var is missing`) } @@ -56,13 +67,13 @@ const PsReportStageCsvFile: AzureFunction = async function (context: Context, in if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${logPrefix}: unable to connect to service bus at this time: ${errorMessage}`) + context.error(`${logPrefix}: unable to connect to service bus at this time: ${errorMessage}`) throw error } // Create the data service to upload to a blob file try { - psReportStagingDataService = new PsReportStagingDataService(context.log, containerName, blobName) + psReportStagingDataService = new PsReportStagingDataService(context, containerName, blobName) // At this point the file has not yet been created, so it needs to be created. An existing file will be // overwritten (erasing old data). await psReportStagingDataService.createAppendBlock() @@ -71,7 +82,7 @@ const PsReportStageCsvFile: AzureFunction = async function (context: Context, in if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${logPrefix}: unable to connect to service bus at this time: ${errorMessage}`) + context.error(`${logPrefix}: unable to connect to service bus at this time: ${errorMessage}`) throw error } @@ -93,14 +104,13 @@ const PsReportStageCsvFile: AzureFunction = async function (context: Context, in if (timeSinceLastMessage >= config.PsReport.StagingFile.WaitTimeToTriggerStagingComplete) { context.log(`${logPrefix}: exiting (and sending output binding message) as no new messages in ${config.PsReport.StagingFile.WaitTimeToTriggerStagingComplete} seconds.`) done = true - // This message should be delivered once - duplicatePrevention is on the sb queue, but outputbindings do not allow a messageId to be set. // ToDo: move away from output bindinds and send this message using the sbClient instead. const completeMessage: PsReportStagingCompleteMessage = { filename: incomingMessage.filename, jobUuid: incomingMessage.jobUuid } - context.bindings.outputData = [completeMessage] + context.extraOutputs.set(stagingCompleteQueueOutput, [completeMessage]) await disconnect() return finish(start, context) } else { @@ -123,14 +133,14 @@ const PsReportStageCsvFile: AzureFunction = async function (context: Context, in finish(start, context) } -function finish (start: number, context: Context): void { +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${logPrefix}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } -async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: Context): Promise { +async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: InvocationContext): Promise { // the sql updates are committed, complete the messages. // if any completes fail, just abandon. // the sql updates are idempotent and as such replaying a message @@ -148,7 +158,7 @@ async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], r if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${logPrefix}: unable to abandon message:${errorMessage}`) + context.error(`${logPrefix}: unable to abandon message:${errorMessage}`) // do nothing. // the lock will expire and message reprocessed at a later time } @@ -156,7 +166,7 @@ async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], r } } -async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: Context): Promise { +async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: InvocationContext): Promise { for (let index = 0; index < messageBatch.length; index++) { const msg = messageBatch[index] try { @@ -166,15 +176,15 @@ async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], re if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${logPrefix}: unable to abandon message:${errorMessage}`) + context.error(`${logPrefix}: unable to abandon message:${errorMessage}`) } } } -async function process (context: Context, messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver): Promise { +async function process (context: InvocationContext, messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver): Promise { try { const psReportData: IPsychometricReportLine[] = messageBatch.map(m => { return revive(m.body as IPsychometricReportLine) }) - const csvTransformer = new CsvTransformer(context.log, psReportData) + const csvTransformer = new CsvTransformer(context, psReportData) const linesOfData = csvTransformer.transform() await psReportStagingDataService.appendDataToBlob(linesOfData) await completeMessages(messageBatch, receiver, context) @@ -191,5 +201,3 @@ async function process (context: Context, messageBatch: sb.ServiceBusReceivedMes function revive (message: IPsychometricReportLine): IPsychometricReportLine { return JSON.parse(JSON.stringify(message), jsonReviver) as IPsychometricReportLine } - -export default PsReportStageCsvFile diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/ps-report-staging.data.service.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/ps-report-staging.data.service.ts index 7087e947fe..1f2f2454bf 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/ps-report-staging.data.service.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/ps-report-staging.data.service.ts @@ -15,7 +15,7 @@ export class PsReportStagingDataService { } this.blobName = blobName this.containerName = containerName - this.logger.verbose(`${this.logName} constructor entered`) + this.logger.trace(`${this.logName} constructor entered`) this.blobService = BlobServiceClient.fromConnectionString(config.AzureStorage.ConnectionString) } diff --git a/tslib/src/functions-ps-report/ps-report-4-writer/index.ts b/tslib/src/functions-ps-report/ps-report-4-writer/index.ts index 0e4218a8a2..01204128ff 100644 --- a/tslib/src/functions-ps-report/ps-report-4-writer/index.ts +++ b/tslib/src/functions-ps-report/ps-report-4-writer/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { PsReportWriterService } from './ps-report-writer.service' import { type PsReportStagingCompleteMessage } from '../common/ps-report-service-bus-messages' @@ -6,23 +6,30 @@ import { JobDataService } from '../../services/data/job.data.service' import { JobStatusCode } from '../../common/job-status-code' let funcName = 'ps-report-4-writer' -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, incomingMessage: PsReportStagingCompleteMessage): Promise { +app.serviceBusQueue(funcName, { + queueName: 'ps-report-staging-complete', + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + handler: psReport4Writer +}) + +export async function psReport4Writer (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const incomingMessage = triggerInput as PsReportStagingCompleteMessage await bulkUpload(context, incomingMessage) const end = performance.now() const durationInMilliseconds = end - start context.log(`${funcName} complete:run took ${durationInMilliseconds} ms`) } -async function bulkUpload (context: Context, incomingMessage: PsReportStagingCompleteMessage): Promise { +async function bulkUpload (context: InvocationContext, incomingMessage: PsReportStagingCompleteMessage): Promise { let dbTable: string = '' - const service = new PsReportWriterService(context.log, context.invocationId) + const service = new PsReportWriterService(context, context.invocationId) const jobDataService = new JobDataService() - funcName = funcName + ': ' + context.invocationId + funcName = `${funcName}: ${context.invocationId}` try { - context.log.verbose(`${funcName}: creating new destination table in SQL Server`) + context.trace(`${funcName}: creating new destination table in SQL Server`) dbTable = await service.createDestinationTableAndViewIfNotExists(incomingMessage) - context.log.verbose(`${funcName}: new table created ${dbTable}`) + context.trace(`${funcName}: new table created ${dbTable}`) await service.prepareForUpload(incomingMessage.filename) context.log(`${funcName}: starting bulk upload from ${incomingMessage.filename} into table ${dbTable}`) @@ -33,13 +40,11 @@ async function bulkUpload (context: Context, incomingMessage: PsReportStagingCom JobStatusCode.CompletedSuccessfully, 'bulk upload complete') } catch (error: any) { if (error instanceof Error) { - context.log.warn(`${funcName}: bulkUpload() failed: ${error.message}`) - context.log.warn(`${funcName}: ${JSON.stringify(error)}`) + context.warn(`${funcName}: bulkUpload() failed: ${error.message}`) + context.warn(`${funcName}: ${JSON.stringify(error)}`) await jobDataService.setJobComplete(incomingMessage.jobUuid, JobStatusCode.Failed, JSON.stringify(error)) } // await service.cleanup(incomingMessage.filename) } } - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions-ps-report/ps-report-4-writer/ps-report-writer.service.ts b/tslib/src/functions-ps-report/ps-report-4-writer/ps-report-writer.service.ts index ab51c05193..9d1fdd13ef 100644 --- a/tslib/src/functions-ps-report/ps-report-4-writer/ps-report-writer.service.ts +++ b/tslib/src/functions-ps-report/ps-report-4-writer/ps-report-writer.service.ts @@ -38,7 +38,7 @@ export class PsReportWriterService { { name: 'name', value: tableName, type: mssql.NVarChar } ] const res = await this.sqlService.query(sql, params) - // this.logger.verbose(`${this.logPrefix()}: res: ${JSON.stringify(res)}`) + // this.logger.trace(`${this.logPrefix()}: res: ${JSON.stringify(res)}`) // empty: res => [] if (R.isEmpty(res)) { return false @@ -148,16 +148,16 @@ export class PsReportWriterService { PupilUPN ASC ) WITH ( PAD_INDEX = OFF,FILLFACTOR = 100,SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, STATISTICS_NORECOMPUTE = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON ) ON [PRIMARY] ` - this.logger.verbose(`${this.logPrefix()}: creating table ${newTableName}`) + this.logger.trace(`${this.logPrefix()}: creating table ${newTableName}`) await this.sqlService.modify(fullSql, []) - this.logger.verbose(`${this.logPrefix()}: Creating trigger`) + this.logger.trace(`${this.logPrefix()}: Creating trigger`) await this.sqlService.modify(triggerSql, []) - this.logger.verbose(`${this.logPrefix()}: Creating IX1`) + this.logger.trace(`${this.logPrefix()}: Creating IX1`) await this.sqlService.modify(ix1Sql, []) - this.logger.verbose(`${this.logPrefix()}: Creating IX2`) + this.logger.trace(`${this.logPrefix()}: Creating IX2`) await this.sqlService.modify(ix2Sql, []) this.logger.info(`${this.logPrefix()}: new table ${newTableName} created`) @@ -179,7 +179,7 @@ export class PsReportWriterService { public async prepareForUpload (blobFile: string): Promise { const blobService = new BlobService() const containerUrl = await blobService.getContainerUrl(containerName) - this.logger.verbose(`${this.logPrefix()}: container url is ${containerUrl}`) + this.logger.trace(`${this.logPrefix()}: container url is ${containerUrl}`) const sasToken = await blobService.getBlobReadWriteSasToken(containerName, blobFile) const sql = ` IF (SELECT COUNT(*) FROM sys.database_scoped_credentials WHERE name = 'PsReportBulkUploadCredential') = 0 @@ -237,7 +237,7 @@ export class PsReportWriterService { * error in this block means we are running in Azure. */ const result = await this.sqlService.query(sql2, []) - this.logger.verbose(`${this.logPrefix()}: OS is ${JSON.stringify(result)}`) + this.logger.trace(`${this.logPrefix()}: OS is ${JSON.stringify(result)}`) let sqlConfig: mssql.config if (result[0].engineEdition === 'Enterprise' || result[0].engineEdition === 'Azure SQL Edge') { diff --git a/tslib/src/functions-throttled/census-import/index.ts b/tslib/src/functions-throttled/census-import/index.ts index 2b627d0335..a139b95e8b 100644 --- a/tslib/src/functions-throttled/census-import/index.ts +++ b/tslib/src/functions-throttled/census-import/index.ts @@ -1,12 +1,18 @@ 'use strict' -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { CensusImportV1 } from './v1' import * as mssql from 'mssql' import config from '../../config' -const blobTrigger: AzureFunction = async function (context: Context, blob: any): Promise { +app.storageBlob('census-import', { + handler: censusImportFunction, + connection: 'AZURE_STORAGE_CONNECTION_STRING', + path: 'census' +}) + +export async function censusImportFunction (blobTriggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() let pool: mssql.ConnectionPool | undefined let meta @@ -27,15 +33,17 @@ const blobTrigger: AzureFunction = async function (context: Context, blob: any): } pool = new mssql.ConnectionPool(sqlConfig) await pool.connect() - const v1 = new CensusImportV1(pool, context.log) - meta = await v1.process(blob, context.bindingData.uri) + const v1 = new CensusImportV1(pool, context) + // TODO how to get this? + const blobUri = context.triggerMetadata?.uri as string ?? '' + meta = await v1.process(blobTriggerInput, blobUri) await pool.close() } catch (error) { if (pool?.connected === true) { await pool.close() } if (error instanceof Error) { - context.log.error(`census-import: ERROR: ${error.message}`) + context.error(`census-import: ERROR: ${error.message}`) } throw error } @@ -44,5 +52,3 @@ const blobTrigger: AzureFunction = async function (context: Context, blob: any): const timeStamp = new Date().toISOString() context.log(`census-import: ${timeStamp} processed ${meta.processCount} pupil records, run took ${durationInMilliseconds} ms`) } - -export default blobTrigger diff --git a/tslib/src/functions-throttled/census-import/v1.spec.ts b/tslib/src/functions-throttled/census-import/v1.spec.ts index 7ce01ebb61..88cc90099f 100644 --- a/tslib/src/functions-throttled/census-import/v1.spec.ts +++ b/tslib/src/functions-throttled/census-import/v1.spec.ts @@ -30,7 +30,7 @@ const BlobServiceMock = jest.fn(() => ({ const LoggerMock = jest.fn(() => ({ error: jest.fn(), info: jest.fn(), - verbose: jest.fn(), + trace: jest.fn(), warn: jest.fn() })) diff --git a/tslib/src/functions-throttled/school-import/data-access/school.data.service.ts b/tslib/src/functions-throttled/school-import/data-access/school.data.service.ts index febd9baa4f..05d12a8fc8 100644 --- a/tslib/src/functions-throttled/school-import/data-access/school.data.service.ts +++ b/tslib/src/functions-throttled/school-import/data-access/school.data.service.ts @@ -36,7 +36,7 @@ export class SchoolDataService implements ISchoolDataService { * @return {SchoolImportJobOutput} */ async bulkUpload (schoolData: ISchoolRecord[]): Promise { - this.logger.verbose('SchoolDataService.bulkUpload() called') + this.logger.trace('SchoolDataService.bulkUpload() called') const table = new mssql.Table('[mtc_admin].[school]') table.create = false diff --git a/tslib/src/functions-throttled/school-import/index.ts b/tslib/src/functions-throttled/school-import/index.ts index d869cb03c7..ab14cb1fe5 100644 --- a/tslib/src/functions-throttled/school-import/index.ts +++ b/tslib/src/functions-throttled/school-import/index.ts @@ -1,41 +1,56 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { SchoolImportService } from './school-import.service' import type * as mssql from 'mssql' import * as ConnectionPoolService from '../../sql/pool.service' import { SchoolImportJobOutput } from './SchoolImportJobOutput' -const name = 'school-import' +const functionName = 'school-import' -const blobTrigger: AzureFunction = async function schoolImportIndex (context: Context, blob: any) { +const blobStdOutOutput = output.storageBlob({ + path: 'school-import/{DateTime}-{name}-output-log.txt', + connection: 'AZURE_STORAGE_CONNECTION_STRING' +}) + +const blobStdErrOutput = output.storageBlob({ + path: 'school-import/{DateTime}-{name}-error-log.txt', + connection: 'AZURE_STORAGE_CONNECTION_STRING' +}) + +app.storageBlob(functionName, { + handler: schoolImportIndex, + connection: 'AZURE_STORAGE_CONNECTION_STRING', + extraOutputs: [blobStdOutOutput, blobStdErrOutput], + path: 'school-import/{name}.csv' +}) + +export async function schoolImportIndex (blobInputTrigger: unknown, context: InvocationContext): Promise { const start = performance.now() - context.log(`${name} started for blob \n Name: ${context.bindingData.name} \n Blob Size: ${blob.length} Bytes`) + const blob = Buffer.from(blobInputTrigger as string, 'base64') // TODO copilot suggestion, not necessarily correct + const blobName = context.triggerMetadata?.name?.toString() ?? '' + context.log(`${functionName} started for blob \n Name: ${blobName} \n Blob Size: ${blob.length} Bytes`) let pool: mssql.ConnectionPool const jobResult = new SchoolImportJobOutput() let svc: SchoolImportService - // Setup try { - pool = await ConnectionPoolService.getInstance(context.log) - svc = new SchoolImportService(pool, jobResult, context.log) + pool = await ConnectionPoolService.getInstance(context) + svc = new SchoolImportService(pool, jobResult, context) } catch (error: any) { - context.log.error(`${name}: FATAL ERROR: ${error.message}`) + context.error(`${functionName}: FATAL ERROR: ${error.message}`) return } try { - await svc.process(blob, context.bindingData.name) + await svc.process(blob, blobName) } catch (error: any) { - context.log.error(`${name}: ERROR: ${error.message}`, error) + context.error(`${functionName}: ERROR: ${error.message}`, error) } - - context.bindings.schoolImportStderr = jobResult.getErrorOutput() - context.bindings.schoolImportStdout = jobResult.getStandardOutput() + context.extraOutputs.set(blobStdOutOutput, jobResult.getStandardOutput()) + context.extraOutputs.set(blobStdErrOutput, jobResult.getErrorOutput()) const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() - context.log(`${name}: ${timeStamp} processed ${jobResult.linesProcessed} schools, run took ${durationInMilliseconds} ms`) + context.log(`${functionName}: ${timeStamp} processed ${jobResult.linesProcessed} schools, run took ${durationInMilliseconds} ms`) } - -export default blobTrigger diff --git a/tslib/src/functions-throttled/school-import/school-import.service.ts b/tslib/src/functions-throttled/school-import/school-import.service.ts index afcc52bfe7..4e84212b7e 100644 --- a/tslib/src/functions-throttled/school-import/school-import.service.ts +++ b/tslib/src/functions-throttled/school-import/school-import.service.ts @@ -44,12 +44,12 @@ export class SchoolImportService { } private async updateJobStatusToProcessing (jobSlug: string): Promise { - this.logger.verbose(`${name}: updateJobStatusToProcessing() called`) + this.logger.trace(`${name}: updateJobStatusToProcessing() called`) return this.jobDataService.setJobStarted(jobSlug) } private async updateJobStatusToCompleted (jobSlug: string): Promise { - this.logger.verbose(`${name}: updateJobStatusToCompleted() called`) + this.logger.trace(`${name}: updateJobStatusToCompleted() called`) if (this.jobResult.hasError()) { return this.jobDataService.setJobComplete(jobSlug, JobStatusCode.CompletedWithErrors, this.jobResult.getStandardOutput(), this.jobResult.getErrorOutput()) } else { @@ -58,12 +58,12 @@ export class SchoolImportService { } private async updateJobStatusToFailed (jobSlug: string, error: Error): Promise { - this.logger.verbose(`${name}: updateJobStatusToFailed() called`) + this.logger.trace(`${name}: updateJobStatusToFailed() called`) return this.jobDataService.setJobComplete(jobSlug, JobStatusCode.Failed, this.jobResult.getStandardOutput(), error.message) } async process (blob: any, blobNameWithoutExtension: string): Promise { - this.logger.verbose(`${name}: process() called`) + this.logger.trace(`${name}: process() called`) // The admin app creates a job on file upload by service manager // The blob name is set to the url slug of the created job. if (blobNameWithoutExtension === undefined || blobNameWithoutExtension === '') { @@ -89,12 +89,12 @@ export class SchoolImportService { const exitMessage = `school records excluded in filtering:${exclusionCount}. No records to persist, exiting.` this.jobResult.stdout.push(exitMessage) await this.updateJobStatusToCompleted(jobSlug) - this.logger.verbose(`${name}: no schools found to upload. Exiting.`) + this.logger.trace(`${name}: no schools found to upload. Exiting.`) return this.jobResult } - this.logger.verbose(`${name} school import starting`) + this.logger.trace(`${name} school import starting`) await this.schoolDataService.individualUpload(filteredSchools) - this.logger.verbose(`${name} school import complete`) + this.logger.trace(`${name} school import complete`) await this.updateJobStatusToCompleted(jobSlug) return this.jobResult } catch (error) { diff --git a/tslib/src/functions-throttled/sync-results-init/index.ts b/tslib/src/functions-throttled/sync-results-init/index.ts index 091c46c89a..b2d3b6c925 100644 --- a/tslib/src/functions-throttled/sync-results-init/index.ts +++ b/tslib/src/functions-throttled/sync-results-init/index.ts @@ -1,11 +1,15 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { type Timer, app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { type ISyncResultsInitServiceOptions, SyncResultsInitService } from './sync-results-init.service' -import { type IFunctionTimer } from '../../azure/functions' - +import './../../common/bigint' const functionName = 'sync-results-init' -const timerTrigger: AzureFunction = async function (context: Context, timer: IFunctionTimer): Promise { +app.timer(functionName, { + schedule: '0 0 17 * * *', + handler: syncResultsInit +}) + +export async function syncResultsInit (timer: Timer, context: InvocationContext): Promise { if (timer.isPastDue) { // This function could potentially deliver a lot of work to do to the functions, and none of it is urgent. No surprises! context.log(`${functionName}: timer is past due, exiting.`) @@ -13,9 +17,10 @@ const timerTrigger: AzureFunction = async function (context: Context, timer: IFu } const start = performance.now() try { - const syncResultsInitService = new SyncResultsInitService(context.log) + const syncResultsInitService = new SyncResultsInitService(context) // If called via http there could be a message passed in - const options: ISyncResultsInitServiceOptions = context.bindingData.syncResultsInit !== undefined ? context.bindingData.syncResultsInit : {} + // TODO this might not be the correct way to access the http inputs + const options: ISyncResultsInitServiceOptions = context.triggerMetadata !== undefined ? context.triggerMetadata : {} const meta = await syncResultsInitService.processBatch(options) const memoryUsage = process.memoryUsage() const heapUsed = memoryUsage.heapUsed / 1024 / 1024 @@ -28,9 +33,7 @@ const timerTrigger: AzureFunction = async function (context: Context, timer: IFu if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } } - -export default timerTrigger diff --git a/tslib/src/functions-throttled/sync-results-to-sql/index.ts b/tslib/src/functions-throttled/sync-results-to-sql/index.ts index 9f3085a4e3..ce3cf5234c 100644 --- a/tslib/src/functions-throttled/sync-results-to-sql/index.ts +++ b/tslib/src/functions-throttled/sync-results-to-sql/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import config from '../../config' @@ -7,6 +7,12 @@ import { SyncResultsServiceFactory } from './sync-results.service.factory' const functionName = 'sync-results-to-sql' +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-completion', + handler: syncResultsToSql +}) + /** * We are unable to determine the maximum number of delivery attempts for the azure service-bus queue * dynamically. This will be a possibility in @azure/service-bus version 7 which has a new API @@ -39,14 +45,15 @@ const maxDeliveryAttempts = config.ServiceBus.CheckCompletionQueueMaxDeliveryCou * if the message is abandoned 10 times (the current 'max delivery count') it will be * put on the dead letter queue automatically. */ -const serviceBusTrigger: AzureFunction = async function (context: Context, checkCompletionMessage: ICheckCompletionMessage): Promise { +export async function syncResultsToSql (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() - const syncResultsServiceFactory = new SyncResultsServiceFactory(context.log) + const syncResultsServiceFactory = new SyncResultsServiceFactory(context) + const checkCompletionMessage = triggerInput as ICheckCompletionMessage await processV2(checkCompletionMessage, context, syncResultsServiceFactory) finish(start, context) } -async function processV2 (message: ICheckCompletionMessage, context: Context, syncResultsServiceFactory: SyncResultsServiceFactory): Promise { +async function processV2 (message: ICheckCompletionMessage, context: InvocationContext, syncResultsServiceFactory: SyncResultsServiceFactory): Promise { const syncResultsService = syncResultsServiceFactory.create() try { await syncResultsService.process(message) @@ -56,31 +63,29 @@ async function processV2 (message: ICheckCompletionMessage, context: Context, sy if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: Error syncing results for check ${message.markedCheck.checkCode}. Error:${errorMessage}`) + context.error(`${functionName}: Error syncing results for check ${message.markedCheck.checkCode}. Error:${errorMessage}`) if (isLastDeliveryAttempt(context)) { handleLastDeliveryAttempt(context, message) } } } -function isLastDeliveryAttempt (context: Context): boolean { +function isLastDeliveryAttempt (context: InvocationContext): boolean { // We need to know if this is the last delivery attempt. Note that deliveryCount property will not // be updated to the maximum until we call abandon() or complete() to release the lock. - if (context.bindingData.deliveryCount === (maxDeliveryAttempts - 1)) { + if (context.triggerMetadata?.deliveryCount === (maxDeliveryAttempts - 1)) { return true } return false } -function handleLastDeliveryAttempt (context: Context, message: ICheckCompletionMessage): void { - context.log.error(`${functionName}: Last delivery attempt for ${message.markedCheck.checkCode} it has had ${context.bindingData.deliveryCount} deliveries already`) +function handleLastDeliveryAttempt (context: InvocationContext, message: ICheckCompletionMessage): void { + context.error(`${functionName}: Last delivery attempt for ${message.markedCheck.checkCode} it has had ${context.triggerMetadata?.deliveryCount} deliveries already`) } -function finish (start: number, context: Context): void { +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default serviceBusTrigger diff --git a/tslib/src/functions-throttled/sync-results-to-sql/sync-results.service.ts b/tslib/src/functions-throttled/sync-results-to-sql/sync-results.service.ts index c9e4caf220..2899eb63cc 100644 --- a/tslib/src/functions-throttled/sync-results-to-sql/sync-results.service.ts +++ b/tslib/src/functions-throttled/sync-results-to-sql/sync-results.service.ts @@ -56,13 +56,13 @@ export class SyncResultsService { return } const schoolId = await this.syncResultsDataService.getSchoolId(schoolUuid) - this.logger.verbose(`${name}: school id retrieved: [${schoolId}]`) + this.logger.trace(`${name}: school id retrieved: [${schoolId}]`) if (schoolId === undefined) { this.logger.error(`${name}: dropRedisSchoolResult(): schoolId not found for uuid ${schoolUuid}`) return } const key = redisKeyService.getSchoolResultsKey(schoolId) - this.logger.verbose(`${name}: redis key to drop: [${key}]`) + this.logger.trace(`${name}: redis key to drop: [${key}]`) await this.redisService.drop([key]) } } diff --git a/tslib/src/functions-util/compress-b64/index.ts b/tslib/src/functions-util/compress-b64/index.ts new file mode 100644 index 0000000000..53a6e70867 --- /dev/null +++ b/tslib/src/functions-util/compress-b64/index.ts @@ -0,0 +1,59 @@ +import { app, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import { CompressionService } from '../../common/compression-service' +import config from '../../config' + +const svc = new CompressionService() + +app.http('util-compress-b64', { + methods: ['POST'], + authLevel: 'function', + handler: utilCompressBase64 +}) + +export async function utilCompressBase64 (request: HttpRequest, context: InvocationContext): Promise { + if (!config.DevTestUtils.TestSupportApi) { + context.log('exiting as not enabled (default behaviour)') + return { + status: 409, + body: 'feature unavailable' + } + } + + let compressed: string = '' + let temp: string = '' + + if (request.headers.get('content-type') === 'application/json') { + temp = JSON.stringify(await request.json()) + } else { + temp = await request.text() + } + if (temp === undefined) { + return { + status: 400, + jsonBody: { + error: 'missing request body' + } + } + } + + try { + compressed = svc.compressToBase64(temp) + } catch (error) { + let msg = 'unknown error' + if (error instanceof Error) { + msg = error.message + } + return { + jsonBody: { + error: msg + }, + status: 500 + } + } + + return { + jsonBody: { + compressed + } + } +} diff --git a/tslib/src/functions/util-compress-b64/readme.md b/tslib/src/functions-util/compress-b64/readme.md similarity index 100% rename from tslib/src/functions/util-compress-b64/readme.md rename to tslib/src/functions-util/compress-b64/readme.md diff --git a/tslib/src/functions/util-test-support-api/README.md b/tslib/src/functions-util/create-school/README.md similarity index 100% rename from tslib/src/functions/util-test-support-api/README.md rename to tslib/src/functions-util/create-school/README.md diff --git a/tslib/src/functions-util/create-school/index.ts b/tslib/src/functions-util/create-school/index.ts new file mode 100644 index 0000000000..9da1623fad --- /dev/null +++ b/tslib/src/functions-util/create-school/index.ts @@ -0,0 +1,78 @@ +import { app, output, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import { performance } from 'perf_hooks' +import config from '../../config' +import { type INewSchoolModel, SchoolApi } from '../create-school/school-api' + +const functionName = 'util-create-school' + +const outputCheckSubmissionStorageQueue = output.storageQueue({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'check-submitted' +}) + +const outputCheckSubmissionServiceBusQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-submission' +}) + +app.http(functionName, { + methods: ['PUT'], + authLevel: 'function', // TODO this was anonymous in v3 - why? ask QA about usage context + handler: utilCreateSchool, + extraOutputs: [outputCheckSubmissionStorageQueue, outputCheckSubmissionServiceBusQueue] +}) + +function finish (start: number, context: InvocationContext): void { + const end = performance.now() + const durationInMilliseconds = end - start + const timeStamp = new Date().toISOString() + context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) +} + +export async function utilCreateSchool (req: HttpRequest, context: InvocationContext): Promise { + if (!config.DevTestUtils.TestSupportApi) { + context.log('exiting as not enabled (default behaviour)') + return { + status: 409, + body: 'feature unavailable' + } + } + + if (req.method !== 'PUT') { + return generateResponse(context, 'Failed', 400, 'Bad request') + } + + const start = performance.now() + const response = await createSchool(context, req) + finish(start, context) + return response +} + +async function createSchool (context: InvocationContext, req: HttpRequest): Promise { + if (req.body === null) { + return generateResponse(context, 'Failed', 400, 'Missing body') + } + + try { + const schoolApi = new SchoolApi(context) + const newSchoolInfo = await req.json() as INewSchoolModel + const entity = await schoolApi.create(newSchoolInfo) + return generateResponse(context, 'Success', 201, 'Created', entity) + } catch (error) { + let errorMessage = 'unknown error' + if (error instanceof Error) { + errorMessage = error.message + } + return generateResponse(context, 'Failed', 500, errorMessage) + } +} + +const generateResponse = function (context: InvocationContext, result: 'Success' | 'Failed', statusCode: number, message: string, entity?: object): HttpResponseInit { + return { + jsonBody: { result, message, entity }, + status: statusCode, + headers: { + 'Content-Type': 'application/json' + } + } +} diff --git a/tslib/src/functions/util-test-support-api/school-api.ts b/tslib/src/functions-util/create-school/school-api.ts similarity index 63% rename from tslib/src/functions/util-test-support-api/school-api.ts rename to tslib/src/functions-util/create-school/school-api.ts index e527149801..a4f8d74090 100644 --- a/tslib/src/functions/util-test-support-api/school-api.ts +++ b/tslib/src/functions-util/create-school/school-api.ts @@ -12,9 +12,24 @@ export class SchoolApi { this.sqlService = sqlService ?? new SqlService(this.logger) } - public async create (body: any): Promise { - this.logger.verbose('SchoolAPI: create() called') - const { leaCode, estabCode, name, urn } = body + public async create (newSchoolInfo: INewSchoolModel): Promise { + if (newSchoolInfo === undefined) { + throw new Error('school to be created is required') + } + if (newSchoolInfo.leaCode === undefined) { + throw new Error('leaCode is required') + } + if (newSchoolInfo.estabCode === undefined) { + throw new Error('estabCode is required') + } + if (newSchoolInfo.name === undefined) { + throw new Error('name is required') + } + if (newSchoolInfo.urn === undefined) { + throw new Error('urn is required') + } + this.logger.trace('SchoolAPI: create() called') + const { leaCode, estabCode, name, urn } = newSchoolInfo const sql = `INSERT INTO mtc_admin.school (leaCode, estabCode, name, urn, dfeNumber) VALUES (@leaCode, @estabCode, @name, @urn, @dfeNumber)` const dfeNumber = parseInt(leaCode.toString().concat(estabCode.toString()), 10) @@ -35,3 +50,10 @@ export class SchoolApi { return R.head(data) } } + +export interface INewSchoolModel { + leaCode: number + estabCode: number + name: string + urn: number +} diff --git a/tslib/src/functions-util/create-taken-check/index.ts b/tslib/src/functions-util/create-taken-check/index.ts new file mode 100644 index 0000000000..c51c0906cf --- /dev/null +++ b/tslib/src/functions-util/create-taken-check/index.ts @@ -0,0 +1,53 @@ +import { app, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import { FakeCompletedCheckMessageGeneratorService } from '../util-submit-check/fake-submitted-check-generator.service' +import config from '../../config' + +const checkGenerator = new FakeCompletedCheckMessageGeneratorService() + +app.http('util-create-taken-check', { + methods: ['POST'], + authLevel: 'function', // TODO this was anonymous in v3 - why? used in browser testing? + handler: utilCreateTakenCheck +}) + +interface CreateTakenCheckRequest { + checkCode: string +} + +export async function utilCreateTakenCheck (request: HttpRequest, context: InvocationContext): Promise { + if (!config.DevTestUtils.TestSupportApi) { + context.log('exiting as not enabled (default behaviour)') + return { + status: 409, + body: 'feature unavailable' + } + } + const rawJsonBody = await request.json() + let checkCode = '' + if (rawJsonBody instanceof Object && 'checkCode' in rawJsonBody) { + const req = (rawJsonBody as CreateTakenCheckRequest) + checkCode = req.checkCode + } else { + return { + status: 400, + body: 'checkCode is required' + } + } + + try { + const outputPayload = await checkGenerator.createV3Message(checkCode) + return { + status: 200, + jsonBody: outputPayload + } + } catch (error) { + let msg = 'unknown error' + if (error instanceof Error) { + msg = error.message + } + return { + status: 500, + body: `An error occured: ${msg}` + } + } +} diff --git a/tslib/src/functions/util-create-taken-check/readme.md b/tslib/src/functions-util/create-taken-check/readme.md similarity index 100% rename from tslib/src/functions/util-create-taken-check/readme.md rename to tslib/src/functions-util/create-taken-check/readme.md diff --git a/tslib/src/functions-util/create-user/README.md b/tslib/src/functions-util/create-user/README.md new file mode 100644 index 0000000000..b34d61465d --- /dev/null +++ b/tslib/src/functions-util/create-user/README.md @@ -0,0 +1,18 @@ +# Test Support API + +The test support API is designed to aid testing on the local development and Test environments. + +## Routes +### Create a School +PUT: `http://localhost:7071/api/test-support/school` + +JSON body: `{"leaCode":"999","estabCode":"1006","name":"A Town Primary School","urn":"89006"}` + +### Create a User + +Note: Only Teacher roles are supported. + +PUT: http://localhost:7071/api/test-support/user + +JSON body: `{"schoolUuid":"3EA09267-8C0A-412E-94FA-C833D095ED41","identifier":"new-user","password":"secretP3ssw0rd!","role":"teacher"}` + diff --git a/tslib/src/functions-util/create-user/index.ts b/tslib/src/functions-util/create-user/index.ts new file mode 100644 index 0000000000..fdd45a4e69 --- /dev/null +++ b/tslib/src/functions-util/create-user/index.ts @@ -0,0 +1,78 @@ +import { app, output, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import { performance } from 'perf_hooks' +import config from '../../config' +import { type ICreateUserModel, UserApi } from '../create-user/user-api' + +const functionName = 'util-create-user' + +const outputCheckSubmissionStorageQueue = output.storageQueue({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'check-submitted' +}) + +const outputCheckSubmissionServiceBusQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-submission' +}) + +app.http(functionName, { + methods: ['PUT'], + authLevel: 'function', // TODO this was anonymous in v3 - why? ask QA about usage context + handler: utilCreateUser, + extraOutputs: [outputCheckSubmissionStorageQueue, outputCheckSubmissionServiceBusQueue] +}) + +function finish (start: number, context: InvocationContext): void { + const end = performance.now() + const durationInMilliseconds = end - start + const timeStamp = new Date().toISOString() + context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) +} + +export async function utilCreateUser (req: HttpRequest, context: InvocationContext): Promise { + if (!config.DevTestUtils.TestSupportApi) { + context.log('exiting as not enabled (default behaviour)') + return { + status: 409, + body: 'feature unavailable' + } + } + + if (req.method !== 'PUT') { + return generateResponse(context, 'Failed', 400, 'Bad request') + } + + const start = performance.now() + const response = await createUser(context, req) + finish(start, context) + return response +} + +async function createUser (context: InvocationContext, req: HttpRequest): Promise { + if (req.body === undefined) { + return generateResponse(context, 'Failed', 400, 'Missing body') + } + + try { + const userApi = new UserApi(context) + const newUserInfo = await req.json() as ICreateUserModel + const entity = await userApi.create(newUserInfo) + return generateResponse(context, 'Success', 201, 'Created', entity) + } catch (error) { + let errorMessage = 'unknown error' + if (error instanceof Error) { + errorMessage = error.message + } + return generateResponse(context, 'Failed', 500, errorMessage) + } +} + +const generateResponse = function (context: InvocationContext, result: 'Success' | 'Failed', statusCode: number, message: string, entity?: object): HttpResponseInit { + return { + jsonBody: { result, message, entity }, + status: statusCode, + headers: { + 'Content-Type': 'application/json' + } + } +} diff --git a/tslib/src/functions/util-test-support-api/user-api.ts b/tslib/src/functions-util/create-user/user-api.ts similarity index 76% rename from tslib/src/functions/util-test-support-api/user-api.ts rename to tslib/src/functions-util/create-user/user-api.ts index 32ed475a71..645ade168e 100644 --- a/tslib/src/functions/util-test-support-api/user-api.ts +++ b/tslib/src/functions-util/create-user/user-api.ts @@ -5,6 +5,13 @@ import * as R from 'ramda' // @ts-ignore - bcryptjs not very ts friendly import * as bcrypt from 'bcryptjs' +export interface ICreateUserModel { + schoolUUID: string + identifier: string + password: string + role: string +} + export class UserApi { private readonly sqlService: ISqlService private readonly logger: ILogger @@ -14,9 +21,21 @@ export class UserApi { this.sqlService = sqlService ?? new SqlService(this.logger) } - public async create (body: any): Promise { - this.logger.verbose('UserAPI: create() called') - const { schoolUuid, identifier, password, role } = body + public async create (newUserInfo: ICreateUserModel): Promise { + if (newUserInfo === undefined) { + throw new Error('user to be created is required') + } + if (newUserInfo.schoolUUID === undefined) { + throw new Error('schoolUUID is required') + } + if (newUserInfo.identifier === undefined) { + throw new Error('identifier is required') + } + if (newUserInfo.role === undefined) { + throw new Error('role is required') + } + this.logger.trace('UserAPI: create() called') + const { schoolUUID, identifier, password, role } = newUserInfo if (role.toUpperCase() !== 'TEACHER') { throw new Error('Only the `teacher` role is supported') } @@ -37,7 +56,7 @@ export class UserApi { THROW 50001, @errorMessage, 1; END - -- Find the role ID + -- Find the role ID SET @roleID = (SELECT TOP 1 id FROM mtc_admin.role WHERE title = @roleTitle); @@ -52,7 +71,7 @@ export class UserApi { VALUES (@passwordHash, @schoolId, @roleId, @identifier) ` const params = [ - { name: 'urlSlug', value: schoolUuid, type: TYPES.UniqueIdentifier }, + { name: 'urlSlug', value: schoolUUID, type: TYPES.UniqueIdentifier }, { name: 'passwordHash', value: passwordHash, type: TYPES.NVarChar }, { name: 'roleTitle', value: role.toUpperCase(), type: TYPES.NVarChar }, { name: 'identifier', value: identifier, type: TYPES.NVarChar(64) } diff --git a/tslib/src/functions/util-diag/index.ts b/tslib/src/functions-util/diag/index.ts similarity index 68% rename from tslib/src/functions/util-diag/index.ts rename to tslib/src/functions-util/diag/index.ts index 7ab8e31fa7..43162efac1 100644 --- a/tslib/src/functions/util-diag/index.ts +++ b/tslib/src/functions-util/diag/index.ts @@ -1,16 +1,21 @@ -import { type Context } from '@azure/functions' +import { type HttpRequest, app, type HttpResponseInit, type InvocationContext } from '@azure/functions' import { readFile } from 'fs' import { promisify } from 'util' import { join } from 'path' const readFileAsync = promisify(readFile) let buildNumber: string = '' -export default async function (context: Context): Promise { - context.res = { +app.http('utilDiag', { + methods: ['GET'], + authLevel: 'function', + handler: utilDiag +}) + +export async function utilDiag (request: HttpRequest, context: InvocationContext): Promise { + return { status: 200, body: `func-consumption. Build:${await getBuildNumber()}. Node version: ${process.version}` } - context.done() } async function getBuildNumber (): Promise { diff --git a/tslib/src/functions/util-received-check-viewer/index.ts b/tslib/src/functions-util/received-check-viewer/index.ts similarity index 57% rename from tslib/src/functions/util-received-check-viewer/index.ts rename to tslib/src/functions-util/received-check-viewer/index.ts index 4fe08841b6..3a95f058a0 100644 --- a/tslib/src/functions/util-received-check-viewer/index.ts +++ b/tslib/src/functions-util/received-check-viewer/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' +import { app, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import config from '../../config' import { TableService } from '../../azure/table-service' @@ -7,40 +7,46 @@ import { type ReceivedCheckTableEntity } from '../../schemas/models' const functionName = 'util-received-check-reader' -function finish (start: number, context: Context): void { +app.http(functionName, { + methods: ['GET'], + authLevel: 'function', + handler: utilReceivedCheckViewer +}) + +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } -const checkRetriever: AzureFunction = async function (context: Context, req: HttpRequest): Promise { +export async function utilReceivedCheckViewer (req: HttpRequest, context: InvocationContext): Promise { if (!config.DevTestUtils.TestSupportApi) { context.log('exiting as not enabled (default behaviour)') - context.done() - return + return { + status: 409, + body: 'feature unavailable' + } } const start = performance.now() - if (req.query.checkCode === undefined || req.query.schoolUUID === undefined) { - context.res = { - statusCode: 400, + if (req.query.get('checkCode') === undefined || req.query.get('schoolUUID') === undefined) { + return { + status: 400, body: 'checkCode and schoolUUID properties are required' } } const tableService = new TableService() - const schoolUUID = req.query.schoolUUID ?? '' - const checkCode = req.query.checkCode ?? '' + const schoolUUID = req.query.get('schoolUUID') ?? '' + const checkCode = req.query.get('checkCode') ?? '' const receivedCheck = await tableService.getEntity('receivedCheck', schoolUUID, checkCode) const archive = receivedCheck.archive ?? '' const compressionService = new CompressionService() - const decompressed = compressionService.decompressFromUTF16(archive) - context.res = { - body: decompressed, + const decompressed = compressionService.decompressFromBase64(archive) + finish(start, context) + return { + jsonBody: JSON.parse(decompressed), headers: { 'Content-Type': 'application/json' } } - finish(start, context) } - -export default checkRetriever diff --git a/tslib/src/functions/util-school-pin-http-service/index.ts b/tslib/src/functions-util/school-pin-http-service/index.ts similarity index 53% rename from tslib/src/functions/util-school-pin-http-service/index.ts rename to tslib/src/functions-util/school-pin-http-service/index.ts index aab293a37d..8fee107d5f 100644 --- a/tslib/src/functions/util-school-pin-http-service/index.ts +++ b/tslib/src/functions-util/school-pin-http-service/index.ts @@ -1,44 +1,52 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' -import { SchoolPinReplenishmnentService } from '../school-pin-generator/school-pin-replenishment.service' +import { app, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import { SchoolPinReplenishmnentService } from '../../functions/school-pin-generator/school-pin-replenishment.service' import { performance } from 'perf_hooks' import config from '../../config' -const functionName = 'school-pin-http-service' +const functionName = 'util-school-pin-generator' -function finish (start: number, context: Context): void { +app.http(functionName, { + methods: ['POST'], + authLevel: 'function', + handler: schoolPinGenerator +}) + +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } -const schoolPinHttpService: AzureFunction = async function (context: Context, req: HttpRequest): Promise { +export async function schoolPinGenerator (req: HttpRequest, context: InvocationContext): Promise { if (!config.DevTestUtils.TestSupportApi) { context.log(`${functionName}:exiting as not enabled (default behaviour)`) - return + return { + status: 409, + body: 'feature unavailable' + } } - const schoolIdParam = req?.body?.school_id + const rawJson = await req.json() + const reqBody = rawJson as Record + const schoolIdParam = reqBody.school_id if (schoolIdParam === undefined) { - context.res = { + return { status: 400, body: 'school_id is required' } - return } const start = performance.now() const schoolPinReplenishmentService = new SchoolPinReplenishmnentService() context.log(`${functionName}: requesting pin for school:${schoolIdParam}`) - const newPin = await schoolPinReplenishmentService.process(context.log, schoolIdParam) + const newPin = await schoolPinReplenishmentService.process(context, schoolIdParam) context.log(`${functionName}: pin:${newPin} generated for school:${schoolIdParam}`) - context.res = { + finish(start, context) + return { status: 200, - body: { + jsonBody: { pin: newPin } } - finish(start, context) } - -export default schoolPinHttpService diff --git a/tslib/src/functions-util/school-pin-sampler/index.ts b/tslib/src/functions-util/school-pin-sampler/index.ts new file mode 100644 index 0000000000..bea7da77a6 --- /dev/null +++ b/tslib/src/functions-util/school-pin-sampler/index.ts @@ -0,0 +1,50 @@ +import { app, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' +import moment from 'moment' +import { SchoolPinSampler } from './school-pin-sampler' +import { performance } from 'perf_hooks' +import config from '../../config' + +const functionName = 'util-school-pin-sampler' + +app.http(functionName, { + methods: ['POST'], + authLevel: 'function', + handler: schoolPinSampler +}) + +function finish (start: number, context: InvocationContext): void { + const end = performance.now() + const durationInMilliseconds = end - start + const timeStamp = new Date().toISOString() + context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) +} + +export async function schoolPinSampler (req: HttpRequest, context: InvocationContext): Promise { + if (!config.DevTestUtils.TestSupportApi) { + context.log('exiting as not enabled (default behaviour)') + return { + status: 409, + body: 'feature unavailable' + } + } + const start = performance.now() + const rawJsonBody = await req.json() + const reqBody = rawJsonBody as Record + const utcNow = reqBody.utcnow !== undefined ? moment(reqBody.utcnow) : moment.utc() + const sampleSize = reqBody.samplesize as number ?? 20 + const randomise = reqBody.randomise as boolean ?? false + const allowedWordsParam = reqBody.allowedWords as string ?? 'foo,bar,baz,qux,quu,cor,gra,gar,wal,frd,pla,xyz' + const allowedWords = allowedWordsParam.split(',').map(w => w.trim()) + context.info(`creating sample of ${sampleSize} school pins generated at ${utcNow.toISOString()}`) + const sampler = new SchoolPinSampler() + const sample = sampler.generateSample(sampleSize, utcNow, new Set(allowedWords), randomise) + finish(start, context) + return { + jsonBody: sample, + headers: { + 'Content-Type': 'application/json' + } + } +} + +export default schoolPinSampler diff --git a/tslib/src/functions/util-school-pin-sampler/school-pin-sampler.spec.ts b/tslib/src/functions-util/school-pin-sampler/school-pin-sampler.spec.ts similarity index 97% rename from tslib/src/functions/util-school-pin-sampler/school-pin-sampler.spec.ts rename to tslib/src/functions-util/school-pin-sampler/school-pin-sampler.spec.ts index 172c333a20..16a9a94563 100644 --- a/tslib/src/functions/util-school-pin-sampler/school-pin-sampler.spec.ts +++ b/tslib/src/functions-util/school-pin-sampler/school-pin-sampler.spec.ts @@ -1,5 +1,5 @@ import { SchoolPinSampler } from './school-pin-sampler' -import { TimezoneUtil } from '../school-pin-generator/timezone-util' +import { TimezoneUtil } from '../../functions/school-pin-generator/timezone-util' import moment = require('moment') let sut: SchoolPinSampler diff --git a/tslib/src/functions/util-school-pin-sampler/school-pin-sampler.ts b/tslib/src/functions-util/school-pin-sampler/school-pin-sampler.ts similarity index 84% rename from tslib/src/functions/util-school-pin-sampler/school-pin-sampler.ts rename to tslib/src/functions-util/school-pin-sampler/school-pin-sampler.ts index 51f76356d3..65ecc4063c 100644 --- a/tslib/src/functions/util-school-pin-sampler/school-pin-sampler.ts +++ b/tslib/src/functions-util/school-pin-sampler/school-pin-sampler.ts @@ -1,8 +1,8 @@ -import * as tzutil from '../school-pin-generator/timezone-util' -import { SchoolPinGenerator } from '../school-pin-generator/school-pin-generator' -import { SchoolPinExpiryGenerator } from '../school-pin-generator/school-pin-expiry-generator' +import * as tzutil from '../../functions/school-pin-generator/timezone-util' +import { SchoolPinGenerator } from '../../functions/school-pin-generator/school-pin-generator' +import { SchoolPinExpiryGenerator } from '../../functions/school-pin-generator/school-pin-expiry-generator' import { type IDateTimeService } from '../../common/datetime.service' -import { RandomGenerator } from '../school-pin-generator/random-generator' +import { RandomGenerator } from '../../functions/school-pin-generator/random-generator' import moment = require('moment') export class SchoolPinSampler { diff --git a/tslib/src/functions/util-submit-check/fake-completed-check-generator.service.spec.ts b/tslib/src/functions-util/util-submit-check/fake-completed-check-generator.service.spec.ts similarity index 100% rename from tslib/src/functions/util-submit-check/fake-completed-check-generator.service.spec.ts rename to tslib/src/functions-util/util-submit-check/fake-completed-check-generator.service.spec.ts diff --git a/tslib/src/functions/util-submit-check/fake-completed-check-generator.service.ts b/tslib/src/functions-util/util-submit-check/fake-completed-check-generator.service.ts similarity index 99% rename from tslib/src/functions/util-submit-check/fake-completed-check-generator.service.ts rename to tslib/src/functions-util/util-submit-check/fake-completed-check-generator.service.ts index 6f9786209e..4d02ab2da7 100644 --- a/tslib/src/functions/util-submit-check/fake-completed-check-generator.service.ts +++ b/tslib/src/functions-util/util-submit-check/fake-completed-check-generator.service.ts @@ -3,7 +3,7 @@ import { type CheckQuestion, type CompleteCheckAnswer, type CompleteCheckAuditEn import { faker } from '@faker-js/faker' import moment from 'moment' import { type IUtilSubmitCheckConfig } from '.' -import { type Answer } from '../check-marker/models' +import { type Answer } from '../../functions/check-marker/models' export interface ICompletedCheckGeneratorService { create (preparedCheck: PreparedCheck, funcConfig?: IUtilSubmitCheckConfig): ValidCheck diff --git a/tslib/src/functions/util-submit-check/fake-submitted-check-generator.service.spec.ts b/tslib/src/functions-util/util-submit-check/fake-submitted-check-generator.service.spec.ts similarity index 100% rename from tslib/src/functions/util-submit-check/fake-submitted-check-generator.service.spec.ts rename to tslib/src/functions-util/util-submit-check/fake-submitted-check-generator.service.spec.ts diff --git a/tslib/src/functions/util-submit-check/fake-submitted-check-generator.service.ts b/tslib/src/functions-util/util-submit-check/fake-submitted-check-generator.service.ts similarity index 100% rename from tslib/src/functions/util-submit-check/fake-submitted-check-generator.service.ts rename to tslib/src/functions-util/util-submit-check/fake-submitted-check-generator.service.ts diff --git a/tslib/src/functions/util-submit-check/index.ts b/tslib/src/functions-util/util-submit-check/index.ts similarity index 60% rename from tslib/src/functions/util-submit-check/index.ts rename to tslib/src/functions-util/util-submit-check/index.ts index 9a905e0c1c..b38cbf3ebb 100644 --- a/tslib/src/functions/util-submit-check/index.ts +++ b/tslib/src/functions-util/util-submit-check/index.ts @@ -1,4 +1,4 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' +import { app, output, type HttpRequest, type HttpResponseInit, type InvocationContext } from '@azure/functions' import config from '../../config' import { FakeCompletedCheckMessageGeneratorService } from './fake-submitted-check-generator.service' import { SubmittedCheckVersion } from '../../schemas/SubmittedCheckVersion' @@ -6,6 +6,23 @@ import { SchoolChecksDataService } from './school-checks.data.service' const functionName = 'util-submit-check' +const outputSubmittedCheckStorageQueue = output.storageQueue({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'check-submitted' +}) + +const outputCheckSubmissionServiceBusQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-submission' +}) + +app.http(functionName, { + methods: ['POST'], + authLevel: 'function', + handler: utilSubmitCheck, + extraOutputs: [outputSubmittedCheckStorageQueue, outputCheckSubmissionServiceBusQueue] +}) + const liveSchoolChecksDataService = new SchoolChecksDataService() export interface IUtilSubmitCheckConfig { @@ -18,36 +35,40 @@ export interface IUtilSubmitCheckConfig { } } -const httpTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise { +export async function utilSubmitCheck (req: HttpRequest, context: InvocationContext): Promise { if (!config.DevTestUtils.TestSupportApi) { context.log(`${functionName} exiting as config.DevTestUtils.TestSupportApi is not enabled (default behaviour)`) - return + return { + status: 409, + body: 'feature unavailable' + } } + const rawJsonBody = await req.json() + const reqBody = rawJsonBody as Record const funcConfig: IUtilSubmitCheckConfig = { - schoolUuid: req.body?.schoolUuid, - checkCodes: req.body?.checkCodes, + schoolUuid: reqBody.schoolUuid as string, + checkCodes: reqBody.checkCodes as string[], answers: { - numberFromCorrectCheckForm: req.body?.answerNumberFromCorrectCheckForm, - numberFromIncorrectCheckForm: req.body?.answerNumberFromIncorrectCheckForm, - numberOfDuplicateAnswers: req.body?.answerNumberOfDuplicates + numberFromCorrectCheckForm: reqBody.answerNumberFromCorrectCheckForm as number, + numberFromIncorrectCheckForm: reqBody.answerNumberFromIncorrectCheckForm as number, + numberOfDuplicateAnswers: reqBody.answerNumberOfDuplicates as number } } - const messageVersion: string = req.body?.messageVersion ?? SubmittedCheckVersion.V3 + const messageVersion: string = reqBody.messageVersion as string ?? SubmittedCheckVersion.V3 if (messageVersion.toString() !== SubmittedCheckVersion.V2.toString() && messageVersion.toString() !== SubmittedCheckVersion.V3.toString()) { - context.res = { + return { status: 400, body: 'unknown messageVersion specified' } - return } context.log(`${functionName} config parsed as: ${JSON.stringify(funcConfig)})`) const fakeSubmittedCheckBuilder = new FakeCompletedCheckMessageGeneratorService() - fakeSubmittedCheckBuilder.setLogger(context.log) + fakeSubmittedCheckBuilder.setLogger(context) fakeSubmittedCheckBuilder.setConfig(funcConfig) if (funcConfig.schoolUuid !== undefined) { const liveCheckCodes = await liveSchoolChecksDataService.fetchBySchoolUuid(funcConfig.schoolUuid) @@ -59,18 +80,19 @@ const httpTrigger: AzureFunction = async function (context: Context, req: HttpRe } }) const messages = await Promise.all(promises) - context.bindings.submittedCheckQueue = messages - return + context.extraOutputs.set(outputSubmittedCheckStorageQueue, messages) + return { + status: 200 + } } if (funcConfig.checkCodes === undefined || !Array.isArray(funcConfig.checkCodes)) { - context.res = { + return { status: 400, body: 'checkCodes array is required' } - return } - if (req.query.bad !== undefined) { + if (req.query.get('bad') !== null) { throw new Error('invalid check functionality not yet implemented') } const messages = [] @@ -78,12 +100,13 @@ const httpTrigger: AzureFunction = async function (context: Context, req: HttpRe const checkCode = funcConfig.checkCodes[index] if (messageVersion === SubmittedCheckVersion.V2.toString()) { messages.push(await fakeSubmittedCheckBuilder.createV2Message(checkCode)) - context.bindings.submittedCheckQueue = messages + context.extraOutputs.set(outputSubmittedCheckStorageQueue, messages) } else { messages.push(await fakeSubmittedCheckBuilder.createV3Message(checkCode)) - context.bindings.checkSubmissionQueue = messages + context.extraOutputs.set(outputCheckSubmissionServiceBusQueue, messages) } } + return { + status: 200 + } } - -export default httpTrigger diff --git a/tslib/src/functions/util-submit-check/school-checks.data.service.ts b/tslib/src/functions-util/util-submit-check/school-checks.data.service.ts similarity index 100% rename from tslib/src/functions/util-submit-check/school-checks.data.service.ts rename to tslib/src/functions-util/util-submit-check/school-checks.data.service.ts diff --git a/tslib/src/functions/check-marker/check-marker.v1.spec.ts b/tslib/src/functions/check-marker/check-marker.v1.spec.ts index da35dd0963..f9856986aa 100644 --- a/tslib/src/functions/check-marker/check-marker.v1.spec.ts +++ b/tslib/src/functions/check-marker/check-marker.v1.spec.ts @@ -5,7 +5,6 @@ import * as R from 'ramda' import * as Subject from './check-marker.v1' import { type ICheckFormService } from '../../services/check-form.service' import { type ILogger } from '../../common/logger' -import { type ICheckMarkerFunctionBindings } from './models' import answersMock from './answers-mock.json' import { CheckNotificationType, type ICheckNotificationMessage } from '../../schemas/check-notification-message' import { type ReceivedCheckFunctionBindingEntity } from '../../schemas/models' @@ -31,7 +30,7 @@ const SqlServiceMock = jest.fn(() => ({ const LoggerMock = jest.fn(() => ({ error: jest.fn(), info: jest.fn(), - verbose: jest.fn(), + trace: jest.fn(), warn: jest.fn() })) @@ -54,12 +53,7 @@ describe('check-marker/v1', () => { test('error is thrown when receivedCheck reference is not found', async () => { try { - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [], - checkNotificationQueue: [], - checkResultTable: [] - } - await sut.mark(functionBindings, loggerMock) + await sut.mark([], loggerMock) fail('error should have been thrown due to empty receivedCheckData') } catch (error) { let errorMessage = 'unknown error' @@ -82,12 +76,6 @@ describe('check-marker/v1', () => { answers: '' } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -95,7 +83,7 @@ describe('check-marker/v1', () => { actualEntity = entity }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('answers property not populated') @@ -114,12 +102,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify({ foo: 1 }) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -127,7 +109,7 @@ describe('check-marker/v1', () => { actualEntity = entity }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('answers data is not an array') @@ -146,12 +128,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answersMock.answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -161,7 +137,7 @@ describe('check-marker/v1', () => { jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode') - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('associated checkForm could not be found by checkCode') @@ -180,12 +156,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answersMock.answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -197,7 +167,7 @@ describe('check-marker/v1', () => { return 'not JSON' }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('associated checkForm data is not valid JSON') @@ -216,12 +186,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answersMock.answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -234,7 +198,7 @@ describe('check-marker/v1', () => { throw new Error(expectedErrorMessage) }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe(`checkForm lookup failed:${expectedErrorMessage}`) @@ -253,12 +217,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answersMock.answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - let actualTableName: string | undefined let actualEntity: any jest.spyOn(tableServiceMock, 'mergeUpdateEntity').mockImplementation(async (table: string, entity: any): Promise => { @@ -270,7 +228,7 @@ describe('check-marker/v1', () => { return JSON.stringify([]) }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('check form data is either empty or not an array') @@ -280,7 +238,7 @@ describe('check-marker/v1', () => { return JSON.stringify({ not: 'array' }) }) - await sut.mark(functionBindings, loggerMock) + await sut.mark(validatedCheckEntity, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(2) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('check form data is either empty or not an array') @@ -317,12 +275,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -334,19 +286,15 @@ describe('check-marker/v1', () => { f2: 2 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const output = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult: any = output.checkResultTable[0] expect(checkResult.mark).toBe(2) expect(checkResult.maxMarks).toBe(2) expect(checkResult.processingError).toBeUndefined() expect(checkResult.markedAt).toBeDefined() expect(checkResult.markedAnswers[0].isCorrect).toBe(true) expect(checkResult.markedAnswers[1].isCorrect).toBe(true) - - persistMarkSpy.mockRestore() }) test('marking updates entity with mark, maxMarks and timestamp: one answer wrong', async () => { @@ -379,12 +327,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -396,19 +338,15 @@ describe('check-marker/v1', () => { f2: 2 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const output = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult: any = output.checkResultTable[0] expect(checkResult.mark).toBe(1) expect(checkResult.maxMarks).toBe(2) expect(checkResult.processingError).toBeUndefined() expect(checkResult.markedAt).toBeInstanceOf(Date) expect(checkResult.markedAnswers[0].isCorrect).toBe(true) expect(checkResult.markedAnswers[1].isCorrect).toBe(false) - - persistMarkSpy.mockRestore() }) test('marking updates check result entity with mark, maxMarks and timestamp: both answers wrong', async () => { @@ -441,12 +379,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -458,21 +390,15 @@ describe('check-marker/v1', () => { f2: 2 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const output = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult = output.checkResultTable[0] expect(checkResult.mark).toBe(0) expect(checkResult.maxMarks).toBe(2) expect(checkResult.markedAt).toBeInstanceOf(Date) expect(checkResult.markedAnswers[0].isCorrect).toBe(false) expect(checkResult.markedAnswers[1].isCorrect).toBe(false) - - const receivedCheckEntity = functionBindings.receivedCheckTable[0] - expect(receivedCheckEntity.processingError).toBeUndefined() - - persistMarkSpy.mockRestore() + expect(validatedCheckEntity.processingError).toBeUndefined() // TODO this might fail now as we deal with outputs differently }) test('check notification is dispatched when marking successful', async () => { @@ -497,12 +423,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -511,9 +431,9 @@ describe('check-marker/v1', () => { }]) }) - await sut.mark(functionBindings, loggerMock) - expect(functionBindings.checkNotificationQueue).toHaveLength(1) - const notificationQueueMessage: ICheckNotificationMessage = R.head(functionBindings.checkNotificationQueue) + const markOutput = await sut.mark(validatedCheckEntity, loggerMock) + expect(markOutput.checkNotificationQueue).toHaveLength(1) + const notificationQueueMessage: ICheckNotificationMessage = R.head(markOutput.checkNotificationQueue) expect(notificationQueueMessage.checkCode).toBeDefined() expect(notificationQueueMessage.notificationType).toBe(CheckNotificationType.checkComplete) }) @@ -531,15 +451,9 @@ describe('check-marker/v1', () => { answers: JSON.stringify({ foo: 1 }) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - - await sut.mark(functionBindings, loggerMock) - expect(functionBindings.checkNotificationQueue).toHaveLength(1) - const notificationQueueMessage: ICheckNotificationMessage = R.head(functionBindings.checkNotificationQueue) + const markOutput = await sut.mark(validatedCheckEntity, loggerMock) + expect(markOutput.checkNotificationQueue).toHaveLength(1) + const notificationQueueMessage: ICheckNotificationMessage = R.head(markOutput.checkNotificationQueue) expect(notificationQueueMessage.checkCode).toBe(checkCode) expect(notificationQueueMessage.notificationType).toBe(CheckNotificationType.checkInvalid) }) @@ -582,12 +496,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -599,11 +507,9 @@ describe('check-marker/v1', () => { f2: 2 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const output = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult = output.checkResultTable[0] expect(checkResult.markedAnswers[0]).toStrictEqual({ factor1: 2, @@ -616,7 +522,6 @@ describe('check-marker/v1', () => { }) expect(checkResult.mark).toBe(2) - persistMarkSpy.mockRestore() }) test('marking uses the first provided answer if there are duplicates, with the same timestamp', async () => { @@ -672,12 +577,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -689,11 +588,9 @@ describe('check-marker/v1', () => { f2: 2 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const output = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult = output.checkResultTable[0] expect(checkResult.markedAnswers[0]).toStrictEqual({ factor1: 2, @@ -711,7 +608,6 @@ describe('check-marker/v1', () => { }) expect(checkResult.mark).toBe(1) - persistMarkSpy.mockRestore() }) test('marking is correct even if an answer is missing from the input', async () => { @@ -736,12 +632,6 @@ describe('check-marker/v1', () => { answers: JSON.stringify(answers) } - const functionBindings: ICheckMarkerFunctionBindings = { - receivedCheckTable: [validatedCheckEntity], - checkNotificationQueue: [], - checkResultTable: [] - } - jest.spyOn(sqlServiceMock, 'getCheckFormDataByCheckCode').mockImplementation(async () => { return JSON.stringify([ { @@ -753,11 +643,9 @@ describe('check-marker/v1', () => { f2: 5 }]) }) - const persistMarkSpy = jest.spyOn(sut, 'persistMark') - await sut.mark(functionBindings, loggerMock) - expect(persistMarkSpy).toHaveBeenCalledTimes(1) - const checkResult: any = persistMarkSpy.mock.calls[0][0] + const markOutput = await sut.mark(validatedCheckEntity, loggerMock) + const checkResult = markOutput.checkResultTable[0] expect(checkResult.markedAnswers[0]).toStrictEqual({ factor1: 2, @@ -781,6 +669,5 @@ describe('check-marker/v1', () => { expect(checkResult.mark).toBe(1) expect(checkResult.maxMarks).toBe(2) - persistMarkSpy.mockRestore() }) }) diff --git a/tslib/src/functions/check-marker/check-marker.v1.ts b/tslib/src/functions/check-marker/check-marker.v1.ts index ceb620e8a8..be6277e225 100644 --- a/tslib/src/functions/check-marker/check-marker.v1.ts +++ b/tslib/src/functions/check-marker/check-marker.v1.ts @@ -4,7 +4,7 @@ import { type ReceivedCheckFunctionBindingEntity } from '../../schemas/models' import moment from 'moment' import { type ICheckFormService, CheckFormService } from '../../services/check-form.service' import { type ILogger } from '../../common/logger' -import { type ICheckMarkerFunctionBindings, type MarkingData, type CheckResult, type MarkedAnswer } from './models' +import type { MarkingData, CheckResult, MarkedAnswer, ICheckMarkerOutputs, IMarkingEntity } from './models' import { type ICheckNotificationMessage, CheckNotificationType } from '../../schemas/check-notification-message' import { type ITableService, TableService } from '../../azure/table-service' import { ReceivedCheckBindingEntityTransformer } from '../../services/receivedCheckBindingEntityTransformer' @@ -31,45 +31,50 @@ export class CheckMarkerV1 { /** * This is the main entry-point called from index.ts - * @param functionBindings + * @param receivedCheckEntry * @param logger */ - async mark (functionBindings: ICheckMarkerFunctionBindings, logger: ILogger): Promise { - logger.verbose('mark() called') - const validatedCheck = this.findValidatedCheck(functionBindings.receivedCheckTable) + async mark (receivedCheckEntry: unknown, logger: ILogger): Promise { + const validatedCheck = this.findValidatedCheck(receivedCheckEntry) const markingData = await this.validateData(validatedCheck, logger) - functionBindings.checkResultTable = [] - functionBindings.checkNotificationQueue = [] + const outputs: ICheckMarkerOutputs = { + checkNotificationQueue: [], + checkResultTable: [] + } if (markingData === undefined) { - this.notifyProcessingFailure(validatedCheck, functionBindings) - return + const failure = this.createProcessingFailureMessage(validatedCheck) + outputs.checkNotificationQueue.push(failure) + return outputs } let checkResult: CheckResult try { checkResult = this.markCheck(markingData, validatedCheck.RowKey) - logger.verbose(`mark(): results ${JSON.stringify(checkResult)}`) - this.persistMark(checkResult, functionBindings, validatedCheck.PartitionKey) + logger.trace(`mark(): results ${JSON.stringify(checkResult)}`) + const markingEntity = this.createMarkingEntity(checkResult, validatedCheck.PartitionKey) + outputs.checkResultTable.push(markingEntity) } catch (error) { - this.notifyProcessingFailure(validatedCheck, functionBindings) - return + const failure = this.createProcessingFailureMessage(validatedCheck) + outputs.checkNotificationQueue.push(failure) + return outputs } const notification: ICheckNotificationMessage = { checkCode: validatedCheck.RowKey, notificationType: CheckNotificationType.checkComplete, version: 1 } - logger.verbose(`mark() setting notification msg to ${JSON.stringify(notification)}`) - functionBindings.checkNotificationQueue.push(notification) + logger.trace(`mark() setting notification msg to ${JSON.stringify(notification)}`) + outputs.checkNotificationQueue.push(notification) + return outputs } - private notifyProcessingFailure (validatedCheck: ReceivedCheckFunctionBindingEntity, functionBindings: ICheckMarkerFunctionBindings): void { + private createProcessingFailureMessage (validatedCheck: ReceivedCheckFunctionBindingEntity): ICheckNotificationMessage { const notification: ICheckNotificationMessage = { checkCode: validatedCheck.RowKey, notificationType: CheckNotificationType.checkInvalid, version: 1 } - functionBindings.checkNotificationQueue.push(notification) + return notification } private async validateData (validatedCheck: ReceivedCheckFunctionBindingEntity, logger: ILogger): Promise { @@ -96,7 +101,6 @@ export class CheckMarkerV1 { // Sort the answers by clientTimeStamp, so that we get a sequential timeline of events const sortedAnswers = this.answerSort(parsedAnswersJson) - const checkCode = validatedCheck.RowKey let rawCheckForm @@ -197,21 +201,22 @@ export class CheckMarkerV1 { return results } - private persistMark (checkResult: CheckResult, functionBindings: ICheckMarkerFunctionBindings, schoolUUID: string): void { - if (functionBindings.checkResultTable === undefined) { - functionBindings.checkResultTable = [] - } + private createMarkingEntity (checkResult: CheckResult, schoolUUID: string): IMarkingEntity { const markingEntity: any = R.omit(['checkCode'], checkResult) markingEntity.PartitionKey = schoolUUID markingEntity.RowKey = checkResult.checkCode - functionBindings.checkResultTable.push(markingEntity) + return markingEntity } - private findValidatedCheck (receivedCheckRef: any[]): ReceivedCheckFunctionBindingEntity { + private findValidatedCheck (receivedCheckRef: unknown): ReceivedCheckFunctionBindingEntity { if (RA.isEmptyArray(receivedCheckRef)) { throw new Error('received check reference is empty') } - return receivedCheckRef[0] + if (!RA.isArray(receivedCheckRef)) { + return receivedCheckRef as ReceivedCheckFunctionBindingEntity + } + const checkArray = receivedCheckRef as ReceivedCheckFunctionBindingEntity[] + return checkArray[0] } private async updateReceivedCheckWithMarkingError (receivedCheck: ReceivedCheckFunctionBindingEntity, markingError: string): Promise { diff --git a/tslib/src/functions/check-marker/index.ts b/tslib/src/functions/check-marker/index.ts index 9b30307936..6721978018 100644 --- a/tslib/src/functions/check-marker/index.ts +++ b/tslib/src/functions/check-marker/index.ts @@ -1,27 +1,56 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, input, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import * as V1 from './check-marker.v1' -import { type ICheckMarkerFunctionBindings } from './models' -import { type MarkCheckMessageV1 } from '../../schemas/models' +import type { ReceivedCheckFunctionBindingEntity, MarkCheckMessageV1 } from '../../schemas/models' const functionName = 'check-marker' const marker = new V1.CheckMarkerV1() -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, markCheckMessage: MarkCheckMessageV1): Promise { +const checkNotificationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-notification' +}) + +const checkResultOutputTable = output.table({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + tableName: 'checkResult' +}) + +const inputReceivedCheckTable = input.table({ + filter: "(PartitionKey eq '{schoolUUID}') and (RowKey eq '{checkCode}')", + connection: 'AZURE_STORAGE_CONNECTION_STRING', + tableName: 'receivedCheck', + take: 1 +}) + +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-marking', + handler: checkMarker, + extraOutputs: [checkNotificationOutputQueue, checkResultOutputTable], + extraInputs: [inputReceivedCheckTable] +}) + +export async function checkMarker (message: unknown, context: InvocationContext): Promise { const start = performance.now() + const markCheckMessage = message as MarkCheckMessageV1 const version = markCheckMessage.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${markCheckMessage.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${markCheckMessage.checkCode}`) try { if (version !== 1) { throw new Error(`Message schema version ${version} unsupported`) } - await marker.mark(context.bindings as ICheckMarkerFunctionBindings, context.log) + const tableInput = context.extraInputs.get(inputReceivedCheckTable) + const receivedCheckInput = tableInput as ReceivedCheckFunctionBindingEntity + const output = await marker.mark(receivedCheckInput, context) + context.extraOutputs.set(checkNotificationOutputQueue, output.checkNotificationQueue) + context.extraOutputs.set(checkResultOutputTable, output.checkResultTable) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -30,5 +59,3 @@ const serviceBusQueueTrigger: AzureFunction = async function (context: Context, const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions/check-marker/models.ts b/tslib/src/functions/check-marker/models.ts index 16fcea222d..882d4e9b15 100644 --- a/tslib/src/functions/check-marker/models.ts +++ b/tslib/src/functions/check-marker/models.ts @@ -6,6 +6,18 @@ export interface ICheckMarkerFunctionBindings { checkResultTable: any[] } +export interface ICheckMarkerOutputs { + checkNotificationQueue: any[] + checkResultTable: any[] +} + +export interface IMarkingEntity { + PartitionKey: string + RowKey: string + answers: Answer[] + formQuestions: CheckFormQuestion[] +} + export interface IMonotonicTimeDto { legacyDate?: string // ISO format string milliseconds?: number diff --git a/tslib/src/functions/check-notifier-batch/batch-check-notifier.data.service.ts b/tslib/src/functions/check-notifier-batch/batch-check-notifier.data.service.ts index 9e42d19999..f1251f0946 100644 --- a/tslib/src/functions/check-notifier-batch/batch-check-notifier.data.service.ts +++ b/tslib/src/functions/check-notifier-batch/batch-check-notifier.data.service.ts @@ -5,11 +5,8 @@ import { ConsoleLogger, type ILogger } from '../../common/logger' export interface IBatchCheckNotifierDataService { createCheckCompleteRequest (checkCode: string): Promise - createProcessingFailedRequest (checkCode: string): ITransactionRequest - createCheckReceivedRequest (checkCode: string): ITransactionRequest - executeRequestsInTransaction (requests: ITransactionRequest[]): Promise } @@ -24,7 +21,7 @@ export class BatchCheckNotifierDataService implements IBatchCheckNotifierDataSer } async createCheckCompleteRequest (checkCode: string): Promise { - this.logService?.verbose(`${this.serviceName}.checkCompleteRequest(): starting for checkCode [${checkCode}]`) + this.logService?.trace(`${this.serviceName}.checkCompleteRequest(): starting for checkCode [${checkCode}]`) const checkCodeParam: ISqlParameter = { type: mssql.UniqueIdentifier, name: 'checkCode', @@ -51,39 +48,39 @@ export class BatchCheckNotifierDataService implements IBatchCheckNotifierDataSer // Pupil Verification: we need to find the check id of the currentCheckId for the pupil, so that IF this checkCode that is now complete, // and it is the same as the currentCheckId THEN we can consider setting the pupil to checkComplete as long as the other checks succeed. const checkCodeSql: string = ` - SELECT + SELECT id FROM - mtc_admin.[check] + mtc_admin.[check] WHERE checkCode = @checkCode ` const checkData = await this.sqlService.query(checkCodeSql, [checkCodeParam]) - this.logService.verbose(`checkData: ${JSON.stringify(checkData)}`) + this.logService.trace(`checkData: ${JSON.stringify(checkData)}`) let checkIdToComplete: undefined | number if (isNonEmptyArray(checkData)) { checkIdToComplete = checkData[0]?.id - this.logService.verbose(`checkIdToComplete: ${checkIdToComplete}`) + this.logService.trace(`checkIdToComplete: ${checkIdToComplete}`) } const pupilSql: string = ` - SELECT - p.attendanceId, + SELECT + p.attendanceId, p.restartAvailable, p.currentCheckId - FROM + FROM mtc_admin.pupil p JOIN mtc_admin.[check] c on (c.pupil_id = p.id) - WHERE + WHERE c.checkCode = @checkCode` let pupil: undefined | { attendanceId: number, restartAvailable: boolean, currentCheckId: number } try { const pupilData = await this.sqlService.query(pupilSql, [checkCodeParam]) - this.logService.verbose(`pupilData: ${JSON.stringify(pupilData)}`) + this.logService.trace(`pupilData: ${JSON.stringify(pupilData)}`) if (isNonEmptyArray(pupilData)) { pupil = pupilData[0] } - this.logService.verbose(`pupil: ${JSON.stringify(pupil)}`) + this.logService.trace(`pupil: ${JSON.stringify(pupil)}`) } catch (error: any) { this.logService.warn(`ERROR: ${error.message}`) } diff --git a/tslib/src/functions/check-notifier-batch/index.ts b/tslib/src/functions/check-notifier-batch/index.ts index 8b98d8e982..acdc6a50a8 100644 --- a/tslib/src/functions/check-notifier-batch/index.ts +++ b/tslib/src/functions/check-notifier-batch/index.ts @@ -1,20 +1,24 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { type Timer, app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import * as sb from '@azure/service-bus' import config from '../../config' import { type ICheckNotificationMessage } from '../../schemas/check-notification-message' import { BatchCheckNotifier } from './batch-check-notifier.service' import * as RA from 'ramda-adjunct' -import { type IFunctionTimer } from '../../azure/functions' const functionName = 'check-notifier-batch' + +app.timer('checkNotifierBatch', { + schedule: '*/30 * * * * *', // execute every 30 seconds + handler: batchCheckNotifier +}) /* * The function is running as a singleton, and the receiver is therefore exclusive we do not expect another receive operation to be in progress. if the message is abandoned 10 times (the current 'max delivery count') it will be put on the dead letter queue automatically. */ -const batchCheckNotifier: AzureFunction = async function (context: Context, timer: IFunctionTimer): Promise { +export async function batchCheckNotifier (timer: Timer, context: InvocationContext): Promise { if (timer.isPastDue) { context.log(`${functionName}: timer is past due, exiting.`) return @@ -46,7 +50,7 @@ const batchCheckNotifier: AzureFunction = async function (context: Context, time if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: unable to connect to service bus at this time:${errorMessage}`) + context.error(`${functionName}: unable to connect to service bus at this time:${errorMessage}`) throw error } @@ -68,9 +72,9 @@ const batchCheckNotifier: AzureFunction = async function (context: Context, time finish(start, context) } -async function process (notifications: ICheckNotificationMessage[], context: Context, messages: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver): Promise { +async function process (notifications: ICheckNotificationMessage[], context: InvocationContext, messages: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver): Promise { try { - const batchNotifier = new BatchCheckNotifier(undefined, context.log) + const batchNotifier = new BatchCheckNotifier(undefined, context) await batchNotifier.notify(notifications) await completeMessages(messages, receiver, context) } catch (error) { @@ -79,7 +83,7 @@ async function process (notifications: ICheckNotificationMessage[], context: Con } } -async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: Context): Promise { +async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: InvocationContext): Promise { // the sql updates are committed, complete the messages. // if any completes fail, just abandon. // the sql updates are idempotent and as such replaying a message @@ -97,7 +101,7 @@ async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], r if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: unable to abandon message:${errorMessage}`) + context.error(`${functionName}: unable to abandon message:${errorMessage}`) // do nothing. // the lock will expire and message reprocessed at a later time } @@ -105,7 +109,7 @@ async function completeMessages (messageBatch: sb.ServiceBusReceivedMessage[], r } } -async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: Context): Promise { +async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], receiver: sb.ServiceBusReceiver, context: InvocationContext): Promise { for (let index = 0; index < messageBatch.length; index++) { const msg = messageBatch[index] try { @@ -115,12 +119,12 @@ async function abandonMessages (messageBatch: sb.ServiceBusReceivedMessage[], re if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: unable to abandon message:${errorMessage}`) + context.error(`${functionName}: unable to abandon message:${errorMessage}`) } } } -function finish (start: number, context: Context): void { +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() diff --git a/tslib/src/functions/check-pin-expiry/index.ts b/tslib/src/functions/check-pin-expiry/index.ts index 922403ab78..0a6760fa6a 100644 --- a/tslib/src/functions/check-pin-expiry/index.ts +++ b/tslib/src/functions/check-pin-expiry/index.ts @@ -1,10 +1,15 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type Timer, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { SqlService } from '../../sql/sql.service' import { CheckPinExpiryService } from './check-pin-expiry.service' const functionName = 'check-pin-expiry' -const timerTrigger: AzureFunction = async function (context: Context): Promise { +app.timer(functionName, { + schedule: '0 10 18,4 * * *', // sec, min, hour, day, month, dow + handler: checkPinExpiry +}) + +export async function checkPinExpiry (timer: Timer, context: InvocationContext): Promise { const start = performance.now() const checkPinExpiryService = new CheckPinExpiryService(new SqlService()) await checkPinExpiryService.process() @@ -13,5 +18,3 @@ const timerTrigger: AzureFunction = async function (context: Context): Promise { + async process (context: InvocationContext, receivedCheck: SubmittedCheckMessage): Promise { const receivedCheckEntity: ReceivedCheckTableEntity = { partitionKey: receivedCheck.schoolUUID.toLowerCase(), rowKey: receivedCheck.checkCode.toLowerCase(), @@ -27,20 +27,23 @@ export class CheckReceiverServiceBus { } await tableService.createEntity('receivedCheck', receivedCheckEntity) - // as per #48506 - check-receiver will now handle this event instead of check-notifier-batch const request = this.checkNotifierDataService.createCheckReceivedRequest(receivedCheck.checkCode.toLowerCase()) + const output: ICheckReceiverOutputs = { + checkNotificationQueue: [], + checkValidationQueue: [] + } try { await this.checkNotifierDataService.executeRequestsInTransaction([request]) } catch (error) { - context.log.error(`check-receiver: failed to write check received notification to database for check ${receivedCheck.checkCode}\n Falling back to check notification queue message.`) + context.error(`check-receiver: failed to write check received notification to database for check ${receivedCheck.checkCode}\n Falling back to check notification queue message.`) const receivedMessage: ICheckNotificationMessage = { version: 1, checkCode: receivedCheck.checkCode, notificationType: CheckNotificationType.checkReceived } - context.bindings.checkNotificationQueue = [receivedMessage] + output.checkNotificationQueue = [receivedMessage] } const message: ValidateCheckMessageV1 = { @@ -48,6 +51,12 @@ export class CheckReceiverServiceBus { checkCode: receivedCheck.checkCode.toLowerCase(), schoolUUID: receivedCheck.schoolUUID.toLowerCase() } - context.bindings.checkValidationQueue = [message] + output.checkValidationQueue = [message] + return output } } + +export interface ICheckReceiverOutputs { + checkValidationQueue: ValidateCheckMessageV1[] + checkNotificationQueue: ICheckNotificationMessage[] +} diff --git a/tslib/src/functions/check-receiver-sb/index.ts b/tslib/src/functions/check-receiver-sb/index.ts index 6f65a52f28..ffcc60e7db 100644 --- a/tslib/src/functions/check-receiver-sb/index.ts +++ b/tslib/src/functions/check-receiver-sb/index.ts @@ -1,29 +1,52 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { CheckReceiverServiceBus } from './check-receiver-sb' import { type SubmittedCheckMessage } from '../../schemas/models' import { SubmittedCheckVersion } from '../../schemas/SubmittedCheckVersion' const functionName = 'check-receiver-sb' +const checkValidationQueueName = 'check-validation' +const checkNotificationQueueName = 'check-notification' +const checkSubmissionQueueName = 'check-submission' -const queueTrigger: AzureFunction = async function (context: Context, submittedCheck: SubmittedCheckMessage): Promise { +const checkValidationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: checkValidationQueueName +}) + +const checkNotificationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: checkNotificationQueueName +}) + +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: checkSubmissionQueueName, + handler: checkReceiverSb, + extraOutputs: [checkValidationOutputQueue, checkNotificationOutputQueue] +}) + +export async function checkReceiverSb (triggerMessage: unknown, context: InvocationContext): Promise { const start = performance.now() + const submittedCheck = triggerMessage as SubmittedCheckMessage const version = submittedCheck.version const expectedVersion = SubmittedCheckVersion.V3 - context.log.info(`${functionName}: version:${version} message received for checkCode ${submittedCheck.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${submittedCheck.checkCode}`) const receiver = new CheckReceiverServiceBus() try { if (version.toString() !== expectedVersion.toString()) { // dead letter the message as we no longer support below v3 throw new Error(`Message schema version:${version} unsupported. Expected version:${expectedVersion}.`) } - await receiver.process(context, submittedCheck) + const output = await receiver.process(context, submittedCheck) + context.extraOutputs.set(checkValidationOutputQueue, output.checkValidationQueue) + context.extraOutputs.set(checkNotificationOutputQueue, output.checkNotificationQueue) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -32,5 +55,3 @@ const queueTrigger: AzureFunction = async function (context: Context, submittedC const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default queueTrigger diff --git a/tslib/src/functions/check-receiver/check-receiver.ts b/tslib/src/functions/check-receiver/check-receiver.ts index d97f047ea3..a4e3115601 100644 --- a/tslib/src/functions/check-receiver/check-receiver.ts +++ b/tslib/src/functions/check-receiver/check-receiver.ts @@ -1,4 +1,4 @@ -import { type Context } from '@azure/functions' +import { type InvocationContext } from '@azure/functions' import { TableService } from '../../azure/table-service' import Moment from 'moment' import { CheckNotificationType, type ICheckNotificationMessage } from '../../schemas/check-notification-message' @@ -16,7 +16,7 @@ export class CheckReceiver { this.checkNotifierDataService = batchCheckNotifierDataService ?? new BatchCheckNotifierDataService(this.logService) } - async process (context: Context, receivedCheck: SubmittedCheckMessage): Promise { + async process (context: InvocationContext, receivedCheck: SubmittedCheckMessage): Promise { const receivedCheckEntity: ReceivedCheckTableEntity = { partitionKey: receivedCheck.schoolUUID.toLowerCase(), rowKey: receivedCheck.checkCode.toLowerCase(), @@ -25,22 +25,25 @@ export class CheckReceiver { checkVersion: +receivedCheck.version, processingError: '' } + const output: ICheckReceiverOutputs = { + checkValidationQueue: [], + checkNotificationQueue: [] + } await tableService.createEntity('receivedCheck', receivedCheckEntity) - // as per #48506 - check-receiver will now handle this event instead of check-notifier-batch const request = this.checkNotifierDataService.createCheckReceivedRequest(receivedCheck.checkCode.toLowerCase()) try { await this.checkNotifierDataService.executeRequestsInTransaction([request]) } catch (error) { - context.log.error(`check-receiver: failed to write check received notification to database for check ${receivedCheck.checkCode}\n Falling back to check notification queue message.`) + context.error(`check-receiver: failed to write check received notification to database for check ${receivedCheck.checkCode}\n Falling back to check notification queue message.`) const receivedMessage: ICheckNotificationMessage = { version: 1, checkCode: receivedCheck.checkCode, notificationType: CheckNotificationType.checkReceived } - context.bindings.checkNotificationQueue = [receivedMessage] + output.checkNotificationQueue = [receivedMessage] } const message: ValidateCheckMessageV1 = { @@ -48,6 +51,12 @@ export class CheckReceiver { checkCode: receivedCheck.checkCode.toLowerCase(), schoolUUID: receivedCheck.schoolUUID.toLowerCase() } - context.bindings.checkValidationQueue = [message] + output.checkValidationQueue = [message] + return output } } + +export interface ICheckReceiverOutputs { + checkValidationQueue: ValidateCheckMessageV1[] + checkNotificationQueue: ICheckNotificationMessage[] +} diff --git a/tslib/src/functions/check-receiver/index.ts b/tslib/src/functions/check-receiver/index.ts index cec791308e..ea5467f6ab 100644 --- a/tslib/src/functions/check-receiver/index.ts +++ b/tslib/src/functions/check-receiver/index.ts @@ -1,28 +1,49 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { CheckReceiver } from './check-receiver' import { type SubmittedCheckMessage } from '../../schemas/models' import { SubmittedCheckVersion } from '../../schemas/SubmittedCheckVersion' - const functionName = 'check-receiver' +const checkValidationQueueName = 'check-validation' +const checkNotificationQueueName = 'check-notification' + +const checkValidationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: checkValidationQueueName +}) + +const checkNotificationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: checkNotificationQueueName +}) -const queueTrigger: AzureFunction = async function (context: Context, submittedCheck: SubmittedCheckMessage): Promise { +app.storageQueue(functionName, { + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'check-submitted', + handler: checkReceiver, + extraOutputs: [checkValidationOutputQueue, checkNotificationOutputQueue] +}) + +export async function checkReceiver (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const submittedCheck = triggerInput as SubmittedCheckMessage const version = submittedCheck.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${submittedCheck.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${submittedCheck.checkCode}`) const receiver = new CheckReceiver() try { if (version !== SubmittedCheckVersion.V2) { // dead letter the message as we no longer support below v3 throw new Error(`Message schema version:${version} unsupported`) } - await receiver.process(context, submittedCheck) + const outputs = await receiver.process(context, submittedCheck) + context.extraOutputs.set(checkValidationQueueName, outputs.checkValidationQueue) + context.extraOutputs.set(checkNotificationQueueName, outputs.checkNotificationQueue) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -31,5 +52,3 @@ const queueTrigger: AzureFunction = async function (context: Context, submittedC const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default queueTrigger diff --git a/tslib/src/functions/check-started/index.ts b/tslib/src/functions/check-started/index.ts index 4ee90c140b..fa57542862 100644 --- a/tslib/src/functions/check-started/index.ts +++ b/tslib/src/functions/check-started/index.ts @@ -1,14 +1,19 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { type ICheckStartedMessage, CheckStartedService } from './check-started.service' -import * as os from 'os' const functionName = 'check-started' -const queueTrigger: AzureFunction = async function (context: Context, checkStartedMessage: ICheckStartedMessage): Promise { +app.storageQueue(functionName, { + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'check-started', + handler: checkStartedFunction +}) + +export async function checkStartedFunction (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const checkStartedMessage = triggerInput as ICheckStartedMessage const version = checkStartedMessage.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${checkStartedMessage.checkCode}`) - context.log.info(`${functionName}: IP addresses:${getIp()}`) + context.info(`${functionName}: version:${version} message received for checkCode ${checkStartedMessage.checkCode}`) try { if (version !== 1) { @@ -22,7 +27,7 @@ const queueTrigger: AzureFunction = async function (context: Context, checkStart if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -31,24 +36,3 @@ const queueTrigger: AzureFunction = async function (context: Context, checkStart const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -function getIp (): string { - const addresses = new Array() - try { - const interfaces = os.networkInterfaces() - for (const key in interfaces) { - interfaces[key]?.forEach(iface => { - addresses.push(iface.address) - }) - } - } catch (error) { - let errorMessage = 'unknown error' - if (error instanceof Error) { - errorMessage = error.message - } - return `unable to obtain IP addresses: ${errorMessage}` - } - return addresses.join(', ') -} - -export default queueTrigger diff --git a/tslib/src/functions/check-sync/index.ts b/tslib/src/functions/check-sync/index.ts index ab577d0f2f..31ac5e5a8d 100644 --- a/tslib/src/functions/check-sync/index.ts +++ b/tslib/src/functions/check-sync/index.ts @@ -1,22 +1,29 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { PreparedCheckSyncService } from './prepared-check-sync.service' import { type IPreparedCheckSyncMessage } from './IPreparedCheckSyncMessage' const functionName = 'check-sync' +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-sync', + handler: checkSync +}) + /** * Synchronise access arrangements from SQL server to the prepared check(s) for a pupil. * @param context * @param preparedCheckSyncMessage */ -const queueTrigger: AzureFunction = async function (context: Context, preparedCheckSyncMessage: IPreparedCheckSyncMessage): Promise { +export async function checkSync (triggerMessage: unknown, context: InvocationContext): Promise { const start = performance.now() + const preparedCheckSyncMessage = triggerMessage as IPreparedCheckSyncMessage const version = preparedCheckSyncMessage.version - context.log.info(`${functionName}: version:${version} message received`) + context.info(`${functionName}: version:${version} message received`) if (version !== 1) { // dead letter the message const message = `Message schema version:${version} unsupported` - context.log.error(message) + context.error(message) throw new Error(message) } try { @@ -27,7 +34,7 @@ const queueTrigger: AzureFunction = async function (context: Context, preparedCh if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -36,5 +43,3 @@ const queueTrigger: AzureFunction = async function (context: Context, preparedCh const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default queueTrigger diff --git a/tslib/src/functions/check-validator/check-validator.spec.ts b/tslib/src/functions/check-validator/check-validator.spec.ts index 43170f68d8..c1f6074800 100644 --- a/tslib/src/functions/check-validator/check-validator.spec.ts +++ b/tslib/src/functions/check-validator/check-validator.spec.ts @@ -1,4 +1,4 @@ -import { CheckValidator, type ICheckValidatorFunctionBindings } from './check-validator' +import { CheckValidator } from './check-validator' import { type ReceivedCheckTableEntity, type MarkCheckMessageV1, @@ -52,7 +52,7 @@ describe('check-validator', () => { error: jest.fn(), warn: jest.fn(), info: jest.fn(), - verbose: jest.fn() + trace: jest.fn() } const checkFormQuestionsFromAnswers = mockV3SubmittedCheck.answers.map((answer: any) => { return { @@ -74,17 +74,12 @@ describe('check-validator', () => { test('error should be thrown when receivedCheck reference is an empty array', async () => { try { - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate([], message, loggerMock) fail('error should have been thrown due to empty receivedCheckData') } catch (error: any) { expect(error.message).toBe('check-validator: received check reference is empty') @@ -106,17 +101,12 @@ describe('check-validator', () => { checkReceivedAt: moment().toDate(), checkVersion: unsupportedCheckVersion } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 123 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe(`check-validator: unsupported check version:'${unsupportedCheckVersion}'`) @@ -142,12 +132,7 @@ describe('check-validator', () => { checkReceivedAt: moment().toDate(), checkVersion: SubmittedCheckVersion.V2 } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('check-validator: message is missing [archive] property') @@ -162,17 +147,12 @@ describe('check-validator', () => { checkReceivedAt: moment().toDate(), checkVersion: SubmittedCheckVersion.V2 } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(compressionServiceMock.decompressFromUTF16).toHaveBeenCalledWith('foo') }) @@ -184,17 +164,12 @@ describe('check-validator', () => { checkReceivedAt: moment().toDate(), checkVersion: SubmittedCheckVersion.V3 } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(compressionServiceMock.decompressFromBase64).toHaveBeenCalledWith('foo') }) @@ -217,17 +192,12 @@ describe('check-validator', () => { foo: 'bar' }) }) - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBeDefined() @@ -254,15 +224,10 @@ describe('check-validator', () => { foo: 'bar' }) }) - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } - await sut.validate(functionBindings, message, loggerMock) - expect(functionBindings.checkNotificationQueue).toBeDefined() - expect(functionBindings.checkNotificationQueue).toHaveLength(1) - const validationFailureMessage = functionBindings.checkNotificationQueue[0] + const output = await sut.validate(receivedCheckEntity, message, loggerMock) + expect(output.checkNotificationQueue).toBeDefined() + expect(output.checkNotificationQueue).toHaveLength(1) + const validationFailureMessage = output.checkNotificationQueue[0] expect(validationFailureMessage.checkCode).toStrictEqual(message.checkCode) expect(validationFailureMessage.notificationType).toBe(CheckNotificationType.checkInvalid) expect(validationFailureMessage.version).toBe(1) @@ -287,12 +252,7 @@ describe('check-validator', () => { checkReceivedAt: moment().toDate(), checkVersion: SubmittedCheckVersion.V3 } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBe('check-validator: message is missing [archive] property') @@ -316,17 +276,12 @@ describe('check-validator', () => { jest.spyOn(compressionServiceMock, 'decompressFromUTF16').mockImplementation(() => { return JSON.stringify(mockV3SubmittedCheck) }) - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBeUndefined() expect(actualEntity.isValid).toBe(true) @@ -343,19 +298,14 @@ describe('check-validator', () => { jest.spyOn(compressionServiceMock, 'decompressFromUTF16').mockImplementation(() => { return JSON.stringify(mockV3SubmittedCheck) }) - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) - expect(functionBindings.checkMarkingQueue).toHaveLength(1) - const checkMarkingMessage: MarkCheckMessageV1 = functionBindings.checkMarkingQueue[0] + const output = await sut.validate(receivedCheckEntity, message, loggerMock) + expect(output.checkMarkingQueue).toHaveLength(1) + const checkMarkingMessage: MarkCheckMessageV1 = output.checkMarkingQueue[0] expect(checkMarkingMessage.checkCode).toStrictEqual(message.checkCode) expect(checkMarkingMessage.schoolUUID).toStrictEqual(message.schoolUUID) expect(checkMarkingMessage.version).toBe(1) @@ -388,17 +338,12 @@ describe('check-validator', () => { maxMarks: undefined, processingError: undefined } - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntry], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message: ValidateCheckMessageV1 = { checkCode: mockV3SubmittedCheck.checkCode, schoolUUID: mockV3SubmittedCheck.schoolUUID, version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntry, message, loggerMock) expect(tableServiceMock.mergeUpdateEntity).toHaveBeenCalledTimes(1) expect(capturedIsValidFlag).toBe(true) expect(capturedAnswers).toStrictEqual(JSON.stringify(mockV3SubmittedCheck.answers)) @@ -421,17 +366,12 @@ describe('check-validator', () => { jest.spyOn(compressionServiceMock, 'decompressFromUTF16').mockImplementation(() => { return JSON.stringify(mockV3SubmittedCheck) }) - const functionBindings: ICheckValidatorFunctionBindings = { - receivedCheckTable: [receivedCheckEntity], - checkMarkingQueue: [], - checkNotificationQueue: [] - } const message = { schoolUUID: 'uuid', checkCode: 'code', version: 1 } - await sut.validate(functionBindings, message, loggerMock) + await sut.validate(receivedCheckEntity, message, loggerMock) expect(actualTableName).toBe('receivedCheck') expect(actualEntity.processingError).toBeUndefined() expect(actualEntity.answers).toStrictEqual(JSON.stringify(mockV3SubmittedCheck.answers)) diff --git a/tslib/src/functions/check-validator/check-validator.ts b/tslib/src/functions/check-validator/check-validator.ts index 6619ce1420..2cf1ced7ec 100644 --- a/tslib/src/functions/check-validator/check-validator.ts +++ b/tslib/src/functions/check-validator/check-validator.ts @@ -18,7 +18,6 @@ const functionName = 'check-validator' const tableStorageTableName = 'receivedCheck' export interface ICheckValidatorFunctionBindings { - receivedCheckTable: any[] checkMarkingQueue: any[] checkNotificationQueue: ICheckNotificationMessage[] } @@ -36,10 +35,14 @@ export class CheckValidator { this.receivedCheckTransformer = new ReceivedCheckBindingEntityTransformer() } - async validate (functionBindings: ICheckValidatorFunctionBindings, validateCheckMessage: ValidateCheckMessageV1, logger: ILogger): Promise { + async validate (receivedCheckTable: unknown, validateCheckMessage: ValidateCheckMessageV1, logger: ILogger): Promise { + const output: ICheckValidatorFunctionBindings = { + checkMarkingQueue: [], + checkNotificationQueue: [] + } // this should fail outside of the catch as we wont be able to update the entity // without a reference to it and should rightly go on the dead letter queue - const receivedCheck = this.findReceivedCheck(functionBindings.receivedCheckTable) + const receivedCheck = this.findReceivedCheck(receivedCheckTable) logger.info(`${functionName}: received check to validate. checkVersion:${receivedCheck.checkVersion}`) let checkData @@ -67,9 +70,9 @@ export class CheckValidator { notificationType: CheckNotificationType.checkInvalid, version: 1 } - functionBindings.checkNotificationQueue = [validationFailure] + output.checkNotificationQueue = [validationFailure] logger.error(error.message) - return + return output } await this.setReceivedCheckAsValid(receivedCheck, checkData) @@ -79,7 +82,8 @@ export class CheckValidator { checkCode: validateCheckMessage.checkCode, version: 1 } - functionBindings.checkMarkingQueue = [markingMessage] + output.checkMarkingQueue = [markingMessage] + return output } private async setReceivedCheckAsValid (receivedCheckEntity: ReceivedCheckFunctionBindingEntity, checkData: any): Promise { @@ -98,11 +102,18 @@ export class CheckValidator { await this.tableService.mergeUpdateEntity(tableStorageTableName, transformedEntity) } - private findReceivedCheck (receivedCheckRef: any[]): ReceivedCheckFunctionBindingEntity { + private findReceivedCheck (receivedCheckRef: unknown): ReceivedCheckFunctionBindingEntity { + if (receivedCheckRef === undefined) { + throw new Error(`${functionName}: received check reference is undefined`) + } if (RA.isEmptyArray(receivedCheckRef)) { throw new Error(`${functionName}: received check reference is empty`) } - return receivedCheckRef[0] + if (!RA.isArray(receivedCheckRef)) { + return receivedCheckRef as ReceivedCheckFunctionBindingEntity + } + const checkArray = receivedCheckRef as ReceivedCheckFunctionBindingEntity[] + return checkArray[0] } private async validateCheckStructure (submittedCheck: any): Promise { diff --git a/tslib/src/functions/check-validator/index.ts b/tslib/src/functions/check-validator/index.ts index 6265361d74..6088a3eb88 100644 --- a/tslib/src/functions/check-validator/index.ts +++ b/tslib/src/functions/check-validator/index.ts @@ -1,22 +1,52 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { type InvocationContext, app, input, output } from '@azure/functions' import { performance } from 'perf_hooks' -import { CheckValidator, type ICheckValidatorFunctionBindings } from './check-validator' -import { type ValidateCheckMessageV1 } from '../../schemas/models' +import { CheckValidator } from './check-validator' +import type { ReceivedCheckFunctionBindingEntity, ValidateCheckMessageV1 } from '../../schemas/models' const validator = new CheckValidator() const functionName = 'check-validator' -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, validateCheckMessage: ValidateCheckMessageV1): Promise { +const checkNotificationOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-notification' +}) + +const checkMarkingOutputQueue = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-marking' +}) + +const inputReceivedCheckTable = input.table({ + filter: "(PartitionKey eq '{schoolUUID}') and (RowKey eq '{checkCode}')", + connection: 'AZURE_STORAGE_CONNECTION_STRING', + tableName: 'receivedCheck', + take: 1 +}) + +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-validation', + handler: checkValidator, + extraOutputs: [checkNotificationOutputQueue, checkMarkingOutputQueue], + extraInputs: [inputReceivedCheckTable] +}) + +export async function checkValidator (triggerMessage: unknown, context: InvocationContext): Promise { const start = performance.now() + const validateCheckMessage = triggerMessage as ValidateCheckMessageV1 const version = validateCheckMessage.version - context.log.info(`${functionName}: version:${version} check validation message received for checkCode ${validateCheckMessage.checkCode}`) + context.info(`${functionName}: version:${version} check validation message received for checkCode ${validateCheckMessage.checkCode}`) try { if (version !== 1) { throw new Error(`Check validation message schema version ${version} unsupported`) } - await validator.validate(context.bindings as ICheckValidatorFunctionBindings, validateCheckMessage, context.log) + const tableInput = context.extraInputs.get(inputReceivedCheckTable) + const receivedCheckInput = tableInput as ReceivedCheckFunctionBindingEntity + const output = await validator.validate(receivedCheckInput, validateCheckMessage, context) + context.extraOutputs.set(checkNotificationOutputQueue, output.checkNotificationQueue) + context.extraOutputs.set(checkMarkingOutputQueue, output.checkMarkingQueue) } catch (error: any) { - context.log.error(`${functionName}: ERROR: ${error.message}`) + context.error(`${functionName}: ERROR: ${error.message}`) throw error } @@ -25,5 +55,3 @@ const serviceBusQueueTrigger: AzureFunction = async function (context: Context, const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions/pupil-feedback/feedback.service.spec.ts b/tslib/src/functions/pupil-feedback/feedback.service.spec.ts index bf6ed44eb5..cfc90d5e07 100644 --- a/tslib/src/functions/pupil-feedback/feedback.service.spec.ts +++ b/tslib/src/functions/pupil-feedback/feedback.service.spec.ts @@ -1,9 +1,8 @@ -import { PupilFeedbackService, type IPupilFeedbackFunctionBinding, type IPupilFeedbackMessage, type IPupilFeedbackTableEntity } from './feedback.service' +import { PupilFeedbackService, type IPupilFeedbackMessage, type IPupilFeedbackTableEntity } from './feedback.service' import { v4 as uuidv4 } from 'uuid' let sut: PupilFeedbackService let message: IPupilFeedbackMessage -let bindings: IPupilFeedbackFunctionBinding describe('pupil feedback service', () => { beforeEach(() => { @@ -15,9 +14,6 @@ describe('pupil feedback service', () => { inputType: 'inputType', satisfactionRating: 'rating' } - bindings = { - feedbackTable: [] - } }) test('subject should be defined', () => { @@ -27,7 +23,7 @@ describe('pupil feedback service', () => { test('unsupported message version throws error', () => { try { message.version = 1 - sut.process(bindings, message) + sut.process(message) fail('error should have been thrown') } catch (error) { let errorMessage = 'unknown error' @@ -39,13 +35,13 @@ describe('pupil feedback service', () => { }) test('feedback message should be added to feedback table binding', () => { - sut.process(bindings, message) - expect(bindings.feedbackTable).toHaveLength(1) + const output = sut.process(message) + expect(output.feedbackTable).toHaveLength(1) }) test('all expected message properties should be inserted into feedback table', () => { - sut.process(bindings, message) - const entity = bindings.feedbackTable[0] as IPupilFeedbackTableEntity + const output = sut.process(message) + const entity = output.feedbackTable[0] as IPupilFeedbackTableEntity expect(entity.PartitionKey).toStrictEqual(message.checkCode) expect(entity.RowKey).toBeDefined() expect(entity.RowKey).toHaveLength(uuidv4().length) diff --git a/tslib/src/functions/pupil-feedback/feedback.service.ts b/tslib/src/functions/pupil-feedback/feedback.service.ts index 31a13d71c9..8c9798be1b 100644 --- a/tslib/src/functions/pupil-feedback/feedback.service.ts +++ b/tslib/src/functions/pupil-feedback/feedback.service.ts @@ -22,11 +22,14 @@ export interface IPupilFeedbackFunctionBinding { } export class PupilFeedbackService { - process (binding: IPupilFeedbackFunctionBinding, message: IPupilFeedbackMessage): void { + process (message: IPupilFeedbackMessage): IPupilFeedbackFunctionBinding { if (message.version !== 2) { throw new Error(`version:${message.version} unsupported`) } - binding.feedbackTable = [] + const output: IPupilFeedbackFunctionBinding = { + feedbackTable: [] + } + const entity: IPupilFeedbackTableEntity = { PartitionKey: message.checkCode, RowKey: uuidv4(), @@ -35,6 +38,7 @@ export class PupilFeedbackService { inputType: message.inputType, satisfactionRating: message.satisfactionRating } - binding.feedbackTable.push(entity) + output.feedbackTable.push(entity) + return output } } diff --git a/tslib/src/functions/pupil-feedback/index.ts b/tslib/src/functions/pupil-feedback/index.ts index 23143001c6..848777a2dc 100644 --- a/tslib/src/functions/pupil-feedback/index.ts +++ b/tslib/src/functions/pupil-feedback/index.ts @@ -1,26 +1,40 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' -import { type IPupilFeedbackMessage, PupilFeedbackService, type IPupilFeedbackFunctionBinding } from './feedback.service' +import { type IPupilFeedbackMessage, PupilFeedbackService } from './feedback.service' const functionName = 'pupil-feedback' const service = new PupilFeedbackService() -const queueTrigger: AzureFunction = async function (context: Context, feedbackMessage: IPupilFeedbackMessage): Promise { +const outputTable = output.table({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + tableName: 'pupilFeedback' +}) + +app.storageQueue(functionName, { + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'pupil-feedback', + handler: pupilFeedback, + extraOutputs: [outputTable] +}) + +export async function pupilFeedback (triggerMessage: unknown, context: InvocationContext): Promise { const start = performance.now() + const feedbackMessage = triggerMessage as IPupilFeedbackMessage const version = feedbackMessage.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${feedbackMessage.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${feedbackMessage.checkCode}`) try { if (version !== 2) { // dead letter the message as we no longer support below v2 throw new Error(`Message schema version:${version} unsupported`) } - service.process(context.bindings as IPupilFeedbackFunctionBinding, feedbackMessage) + const output = service.process(feedbackMessage) + context.extraOutputs.set(outputTable, output.feedbackTable) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -29,5 +43,3 @@ const queueTrigger: AzureFunction = async function (context: Context, feedbackMe const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default queueTrigger diff --git a/tslib/src/functions/pupil-login/index.ts b/tslib/src/functions/pupil-login/index.ts index 40231a2cae..2c26b3ba77 100644 --- a/tslib/src/functions/pupil-login/index.ts +++ b/tslib/src/functions/pupil-login/index.ts @@ -1,21 +1,33 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' -import { PupilLoginService, type IPupilLoginMessage, type IPupilLoginFunctionBindings } from './pupil-login.service' +import { PupilLoginService, type IPupilLoginMessage } from './pupil-login.service' const functionName = 'pupil-login' const pupilLoginService = new PupilLoginService() -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, pupilLoginMessage: IPupilLoginMessage): Promise { +const outputTable = output.table({ + connection: 'AZURE_STORAGE_CONNECTION_STRING', + tableName: 'pupilEvent' +}) + +app.serviceBusQueue(functionName, { + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'pupil-login', + handler: pupilLogin, + extraOutputs: [outputTable] +}) + +export async function pupilLogin (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const pupilLoginMessage = triggerInput as IPupilLoginMessage const version = pupilLoginMessage.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${pupilLoginMessage.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${pupilLoginMessage.checkCode}`) - await pupilLoginService.process(pupilLoginMessage, context.bindings as IPupilLoginFunctionBindings) + const outputs = await pupilLoginService.process(pupilLoginMessage) + context.extraOutputs.set(outputTable, outputs.pupilEventTable) const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions/pupil-login/pupil-login.service.spec.ts b/tslib/src/functions/pupil-login/pupil-login.service.spec.ts index 8d3da3b337..e764abb594 100644 --- a/tslib/src/functions/pupil-login/pupil-login.service.spec.ts +++ b/tslib/src/functions/pupil-login/pupil-login.service.spec.ts @@ -1,11 +1,8 @@ import { type IPupilLoginDataService } from './pupil-login.data.service' -import { PupilLoginService, type IPupilLoginMessage, type IPupilLoginFunctionBindings, type IPupilEvent } from './pupil-login.service' +import { PupilLoginService, type IPupilLoginMessage, type IPupilEvent } from './pupil-login.service' let sut: PupilLoginService let dataServiceMock: IPupilLoginDataService -const bindings: IPupilLoginFunctionBindings = { - pupilEventTable: [] -} const DataServiceMock = jest.fn(() => ({ updateCheckWithLoginTimestamp: jest.fn() @@ -29,7 +26,7 @@ describe('pupil-login.service', () => { practice: true } try { - await sut.process(message, bindings) + await sut.process(message) fail('error should have been thrown') } catch (error) { let errorMessage = 'unknown error' @@ -47,7 +44,7 @@ describe('pupil-login.service', () => { loginAt: new Date(), practice: false } - await sut.process(message, bindings) + await sut.process(message) expect(dataServiceMock.updateCheckWithLoginTimestamp).toHaveBeenCalledWith(message.checkCode, message.loginAt) }) @@ -58,7 +55,7 @@ describe('pupil-login.service', () => { loginAt: new Date(), practice: true } - await sut.process(message, bindings) + await sut.process(message) expect(dataServiceMock.updateCheckWithLoginTimestamp).toHaveBeenCalledWith(message.checkCode, message.loginAt) }) @@ -69,9 +66,9 @@ describe('pupil-login.service', () => { loginAt: new Date(), practice: true } - await sut.process(message, bindings) - expect(bindings.pupilEventTable).toHaveLength(1) - const entry = bindings.pupilEventTable[0] as IPupilEvent + const output = await sut.process(message) + expect(output.pupilEventTable).toHaveLength(1) + const entry = output.pupilEventTable[0] as IPupilEvent expect(entry.PartitionKey).toStrictEqual(message.checkCode) expect(entry.RowKey).toBeDefined() expect(entry.eventType).toBe('pupil-login') diff --git a/tslib/src/functions/pupil-login/pupil-login.service.ts b/tslib/src/functions/pupil-login/pupil-login.service.ts index e39e337581..34fac5a34b 100644 --- a/tslib/src/functions/pupil-login/pupil-login.service.ts +++ b/tslib/src/functions/pupil-login/pupil-login.service.ts @@ -1,7 +1,7 @@ import { type IPupilLoginDataService, PupilLoginDataService } from './pupil-login.data.service' import { v4 as uuid } from 'uuid' import moment = require('moment') -import { type IModifyResult } from '../../sql/sql.service' + export interface IPupilLoginMessage { version: number checkCode: string @@ -9,7 +9,7 @@ export interface IPupilLoginMessage { practice: boolean } -export interface IPupilLoginFunctionBindings { +export interface IPupilLoginOutputs { pupilEventTable: any[] } @@ -31,12 +31,14 @@ export class PupilLoginService { this.dataService = dataService } - async process (message: IPupilLoginMessage, bindings: IPupilLoginFunctionBindings): Promise { + async process (message: IPupilLoginMessage): Promise { if (message.version !== 1) { throw new Error(`pupil-login message version:${message.version} unsupported`) } - bindings.pupilEventTable = [] - bindings.pupilEventTable.push({ + const output: IPupilLoginOutputs = { + pupilEventTable: [] + } + output.pupilEventTable.push({ PartitionKey: message.checkCode.toLowerCase(), RowKey: uuid(), eventType: 'pupil-login', @@ -44,6 +46,7 @@ export class PupilLoginService { processedAt: moment().toDate() }) const loginDate = new Date(message.loginAt) - return this.dataService.updateCheckWithLoginTimestamp(message.checkCode, loginDate) + await this.dataService.updateCheckWithLoginTimestamp(message.checkCode, loginDate) + return output } } diff --git a/tslib/src/functions/pupil-prefs/index.ts b/tslib/src/functions/pupil-prefs/index.ts index f446fc4ac8..df8524be39 100644 --- a/tslib/src/functions/pupil-prefs/index.ts +++ b/tslib/src/functions/pupil-prefs/index.ts @@ -1,27 +1,40 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, output, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' -import { PupilPrefsService } from './pupil-prefs.service' -import { type IPupilPrefsFunctionBindings } from './IPupilPrefsFunctionBindings' +import { type IPupilPreferenceUpdate, PupilPrefsService } from './pupil-prefs.service' const functionName = 'pupil-prefs' -const queueTrigger: AzureFunction = async function (context: Context, pupilPrefsMessage: any): Promise { +const checkSyncQueueOutput = output.serviceBusQueue({ + connection: 'AZURE_SERVICE_BUS_CONNECTION_STRING', + queueName: 'check-sync' +}) + +app.storageQueue(functionName, { + connection: 'AZURE_STORAGE_CONNECTION_STRING', + queueName: 'pupil-prefs', + handler: pupilPrefs, + extraOutputs: [checkSyncQueueOutput] +}) + +export async function pupilPrefs (triggerInput: unknown, context: InvocationContext): Promise { const start = performance.now() + const pupilPrefsMessage = triggerInput as IPupilPreferenceUpdate const version = pupilPrefsMessage.version - context.log.info(`${functionName}: version:${version} message received for checkCode ${pupilPrefsMessage.checkCode}`) + context.info(`${functionName}: version:${version} message received for checkCode ${pupilPrefsMessage.checkCode}`) try { if (version !== 1) { // dead letter the message as we only support v1 throw new Error(`Message schema version:${version} unsupported`) } - const prefsService = new PupilPrefsService(undefined, context.log) - await prefsService.update(pupilPrefsMessage, context.bindings as IPupilPrefsFunctionBindings) + const prefsService = new PupilPrefsService(undefined, context) + const output = await prefsService.update(pupilPrefsMessage) + context.extraOutputs.set(checkSyncQueueOutput, output.checkSyncQueue) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { errorMessage = error.message } - context.log.error(`${functionName}: ERROR: ${errorMessage}`) + context.error(`${functionName}: ERROR: ${errorMessage}`) throw error } @@ -30,5 +43,3 @@ const queueTrigger: AzureFunction = async function (context: Context, pupilPrefs const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } - -export default queueTrigger diff --git a/tslib/src/functions/pupil-prefs/pupil-prefs.service.spec.ts b/tslib/src/functions/pupil-prefs/pupil-prefs.service.spec.ts index 8abaa5b6b4..6a735cd432 100644 --- a/tslib/src/functions/pupil-prefs/pupil-prefs.service.spec.ts +++ b/tslib/src/functions/pupil-prefs/pupil-prefs.service.spec.ts @@ -4,7 +4,6 @@ import { type IPupilPreferenceDataUpdate, type IPupilPreferenceUpdate } from './pupil-prefs.service' -import { type IPupilPrefsFunctionBindings } from './IPupilPrefsFunctionBindings' const PupilPrefsDataServiceMock = jest.fn(() => ({ updatePupilPreferences: jest.fn(), @@ -14,9 +13,6 @@ const PupilPrefsDataServiceMock = jest.fn(() => ({ let sut: PupilPrefsService let dataServiceMock: IPupilPrefsDataService -const functionBindings: IPupilPrefsFunctionBindings = { - checkSyncQueue: [] -} describe('pupil-prefs.service', () => { beforeEach(() => { @@ -39,9 +35,10 @@ describe('pupil-prefs.service', () => { preferences: { colourContrastCode: 'FTS', fontSizeCode: 'CCT' - } + }, + version: 123 } - await sut.update(update, functionBindings) + await sut.update(update) expect(dataServiceMock.updatePupilPreferences).toHaveBeenCalledTimes(1) expect(dataUpdates).toHaveLength(2) }) @@ -56,9 +53,10 @@ describe('pupil-prefs.service', () => { checkCode: 'check-code', preferences: { colourContrastCode: 'FTS' - } + }, + version: 123 } - await sut.update(update, functionBindings) + await sut.update(update) expect(dataServiceMock.updatePupilPreferences).toHaveBeenCalledTimes(1) expect(dataUpdates).toHaveLength(1) expect(dataUpdates[0].prefTable).toBe('[colourContrastLookUp]') @@ -75,9 +73,10 @@ describe('pupil-prefs.service', () => { checkCode: 'check-code', preferences: { fontSizeCode: 'CCT' - } + }, + version: 123 } - await sut.update(update, functionBindings) + await sut.update(update) expect(dataServiceMock.updatePupilPreferences).toHaveBeenCalledTimes(1) expect(dataUpdates).toHaveLength(1) expect(dataUpdates[0].prefTable).toBe('[fontSizeLookUp]') diff --git a/tslib/src/functions/pupil-prefs/pupil-prefs.service.ts b/tslib/src/functions/pupil-prefs/pupil-prefs.service.ts index fb51124467..e74f609d69 100644 --- a/tslib/src/functions/pupil-prefs/pupil-prefs.service.ts +++ b/tslib/src/functions/pupil-prefs/pupil-prefs.service.ts @@ -18,8 +18,11 @@ export class PupilPrefsService { this.dataService = pupilPrefsDataService } - async update (preferenceUpdate: IPupilPreferenceUpdate, functionBindings: IPupilPrefsFunctionBindings): Promise { + async update (preferenceUpdate: IPupilPreferenceUpdate): Promise { const dataUpdates = new Array() + const output: IPupilPrefsFunctionBindings = { + checkSyncQueue: [] + } if (preferenceUpdate.preferences.colourContrastCode !== undefined) { const colourContrastDataUpdate: IPupilPreferenceDataUpdate = { @@ -44,11 +47,11 @@ export class PupilPrefsService { } await this.dataService.updatePupilPreferences(dataUpdates) const pupilUUID = await this.dataService.getPupilUUIDByCheckCode(preferenceUpdate.checkCode) - functionBindings.checkSyncQueue = [] - functionBindings.checkSyncQueue.push({ + output.checkSyncQueue.push({ pupilUUID, version: 1 }) + return output } } @@ -58,6 +61,7 @@ export interface IPupilPreferenceUpdate { fontSizeCode?: string colourContrastCode?: string } + version: number } export interface IPupilPreferenceDataUpdate { diff --git a/tslib/src/functions/queue-replay-service-bus/index.ts b/tslib/src/functions/queue-replay-service-bus/index.ts deleted file mode 100644 index aaca7323d0..0000000000 --- a/tslib/src/functions/queue-replay-service-bus/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { type AzureFunction, type Context } from '@azure/functions' -import { performance } from 'perf_hooks' -import { type IQueueMessageReplayRequest } from './message-replay.service' - -const functionName = 'queue-replay-service-bus' - -function finish (start: number, context: Context): void { - const end = performance.now() - const durationInMilliseconds = end - start - const timeStamp = new Date().toISOString() - context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) -} - -const queueReplayServiceBus: AzureFunction = async function (context: Context, busMessage: IQueueMessageReplayRequest): Promise { - const start = performance.now() - context.log('nothing to do, yet...') - finish(start, context) -} - -export default queueReplayServiceBus diff --git a/tslib/src/functions/queue-replay-service-bus/message-replay.service.ts b/tslib/src/functions/queue-replay-service-bus/message-replay.service.ts deleted file mode 100644 index b7e4a0d0d0..0000000000 --- a/tslib/src/functions/queue-replay-service-bus/message-replay.service.ts +++ /dev/null @@ -1,20 +0,0 @@ -export interface IMessageReplayService { - replay (request: IQueueMessageReplayRequest): Promise -} - -export class ServiceBusMessageReplayService implements IMessageReplayService { - async replay (request: IQueueMessageReplayRequest): Promise { - throw new Error('not implemented') - } -} - -export class StorageQueueMessageReplayService implements IMessageReplayService { - async replay (request: IQueueMessageReplayRequest): Promise { - throw new Error('not implemented') - } -} - -export interface IQueueMessageReplayRequest { - queueName: string - messageCount?: number -} diff --git a/tslib/src/functions/school-pin-generator/index.ts b/tslib/src/functions/school-pin-generator/index.ts index 3a4b77af9a..10458eebf1 100644 --- a/tslib/src/functions/school-pin-generator/index.ts +++ b/tslib/src/functions/school-pin-generator/index.ts @@ -1,22 +1,25 @@ -import { type AzureFunction, type Context } from '@azure/functions' +import { app, type Timer, type InvocationContext } from '@azure/functions' import { performance } from 'perf_hooks' import { SchoolPinReplenishmnentService } from './school-pin-replenishment.service' const functionName = 'school-pin-generator' -function finish (start: number, context: Context): void { +app.timer(functionName, { + schedule: '0 0 * * * *', // every hour, on the hour + handler: schoolPinGenerator +}) + +function finish (start: number, context: InvocationContext): void { const end = performance.now() const durationInMilliseconds = end - start const timeStamp = new Date().toISOString() context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) } -const schoolPinGenerator: AzureFunction = async function (context: Context): Promise { +export async function schoolPinGenerator (timer: Timer, context: InvocationContext): Promise { const start = performance.now() context.log(`${functionName} starting`) const replenishmentService = new SchoolPinReplenishmnentService() - await replenishmentService.process(context.log) + await replenishmentService.process(context) finish(start, context) } - -export default schoolPinGenerator diff --git a/tslib/src/functions/school-pin-generator/school-pin-replenishment.service.ts b/tslib/src/functions/school-pin-generator/school-pin-replenishment.service.ts index f193618cb8..9f0615ce13 100644 --- a/tslib/src/functions/school-pin-generator/school-pin-replenishment.service.ts +++ b/tslib/src/functions/school-pin-generator/school-pin-replenishment.service.ts @@ -46,7 +46,7 @@ export class SchoolPinReplenishmnentService { let schoolsToProcess: School[] let returnGeneratedPin = false let pinToReturn = '' - logger.verbose(`${this.logName}: process() called for schoolId [${schoolId}]`) + logger.trace(`${this.logName}: process() called for schoolId [${schoolId}]`) if (schoolId === undefined) { logger.info(`${this.logName}: generating pins for all schools`) @@ -54,7 +54,7 @@ export class SchoolPinReplenishmnentService { } else { returnGeneratedPin = true const school = await this.dataService.getSchoolById(schoolId) - logger.verbose(`${this.logName} school found in db ${JSON.stringify(school)}`) + logger.trace(`${this.logName} school found in db ${JSON.stringify(school)}`) schoolsToProcess = [] if (school !== undefined) { schoolsToProcess.push(school) diff --git a/tslib/src/functions/util-compress-b64/index.ts b/tslib/src/functions/util-compress-b64/index.ts deleted file mode 100644 index 169e354ad6..0000000000 --- a/tslib/src/functions/util-compress-b64/index.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { type Context } from '@azure/functions' -import { CompressionService } from '../../common/compression-service' - -const svc = new CompressionService() - -export default async function (context: Context): Promise { - let input = context?.req?.body - if (input === undefined) { - context.res = { - status: 400, - body: 'input is required' - } - return - } - let compressed: string = '' - try { - if (context.req?.headers['content-type'] === 'application/json') { - input = JSON.stringify(input) - } - compressed = svc.compressToBase64(input) - } catch (error) { - let msg = 'unknown error' - if (error instanceof Error) { - msg = error.message - } - context.res = { - status: 500, - body: `An error occured: ${msg}` - } - return - } - - context.res = { - status: 200, - body: { - compressed - } - } -} diff --git a/tslib/src/functions/util-create-taken-check/index.ts b/tslib/src/functions/util-create-taken-check/index.ts deleted file mode 100644 index 5065404127..0000000000 --- a/tslib/src/functions/util-create-taken-check/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { type Context } from '@azure/functions' -import { FakeCompletedCheckMessageGeneratorService } from '../util-submit-check/fake-submitted-check-generator.service' - -const checkGenerator = new FakeCompletedCheckMessageGeneratorService() - -export default async function (context: Context): Promise { - const requestBody = context?.req?.body - if (requestBody === undefined || requestBody.checkCode === undefined) { - context.res = { - status: 400, - body: 'checkCode is required' - } - return - } - - try { - const outputPayload = await checkGenerator.createV3Message(requestBody.checkCode) - context.res = { - status: 200, - body: outputPayload - } - } catch (error) { - let msg = 'unknown error' - if (error instanceof Error) { - msg = error.message - } - context.res = { - status: 500, - body: `An error occured: ${msg}` - } - } -} diff --git a/tslib/src/functions/util-replay-check/index.ts b/tslib/src/functions/util-replay-check/index.ts deleted file mode 100644 index f6a3133862..0000000000 --- a/tslib/src/functions/util-replay-check/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' -import config from '../../config' -import { ReceivedCheckPayloadService } from './received-check-payload.service' -const functionName = 'util-replay-check' - -const receivedCheckPayloadService = new ReceivedCheckPayloadService() - -const httpTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise { - if (!config.DevTestUtils.TestSupportApi) { - context.log(`${functionName} exiting as config.DevTestUtils.TestSupportApi is not enabled (default behaviour)`) - context.done() - return - } - - const schoolUuid = req.body?.schoolUuid - if (schoolUuid !== undefined) { - const messages = await receivedCheckPayloadService.fetchBySchool(schoolUuid) - context.log(`found ${messages.length} checks to replay for school.uuid:${schoolUuid}`) - context.bindings.submittedCheckQueue = messages - context.done() - return - } - - const checkCodes: string[] = req.body?.checkCodes - if (checkCodes === undefined || !Array.isArray(checkCodes)) { - context.res = { - status: 400, - body: 'checkCodes array is required' - } - return - } - // console.dir(checkCodes) - const messages = await receivedCheckPayloadService.fetch(checkCodes) - // console.dir(messages) - context.bindings.submittedCheckQueue = messages -} - -export default httpTrigger diff --git a/tslib/src/functions/util-replay-check/readme.md b/tslib/src/functions/util-replay-check/readme.md deleted file mode 100644 index 5a8199fa45..0000000000 --- a/tslib/src/functions/util-replay-check/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Check Replay Utility - -Please see the documentation [here](../../../../docs/support-utils/util-check-replay.md) diff --git a/tslib/src/functions/util-replay-check/received-check-payload.data.service.ts b/tslib/src/functions/util-replay-check/received-check-payload.data.service.ts deleted file mode 100644 index 26b3e00013..0000000000 --- a/tslib/src/functions/util-replay-check/received-check-payload.data.service.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { TYPES } from 'mssql' -import { isArray } from 'ramda-adjunct' -import { type ISqlParameter, type ISqlService, SqlService } from '../../sql/sql.service' - -export interface IReceivedCheckPayloadDataService { - fetchCompressedArchives (checkCodes: string[]): Promise - fetchArchivesForSchool (schoolUuid: string): Promise -} - -export interface IArchiveEntry { - checkCode: string - archive: string -} - -export class ReceivedCheckPayloadDataService implements IReceivedCheckPayloadDataService { - private readonly sqlService: ISqlService - - constructor () { - this.sqlService = new SqlService() - } - - async fetchCompressedArchives (checkCodes: string[]): Promise { - const params: ISqlParameter[] = [] - const paramIds: string[] = [] - for (let index = 0; index < checkCodes.length; index++) { - const checkCode = checkCodes[index] - params.push({ - name: `checkCode${index}`, - type: TYPES.UniqueIdentifier, - value: checkCode - }) - paramIds.push(`@checkCode${index}`) - } - const sql = `SELECT archive FROM mtc_admin.receivedCheck WHERE RowKey IN (${paramIds.join(',')})` - const result = await this.sqlService.query(sql, params) - if (!isArray(result)) return [] - if (result.length === 0) return [] - return result.map(r => r.archive) - } - - async fetchArchivesForSchool (schoolUuid: string): Promise { - const sql = ` - SELECT chk.checkCode, r.archive FROM mtc_admin.[check] chk - INNER JOIN mtc_admin.pupil p ON chk.pupil_id = p.id - INNER JOIN mtc_admin.school s ON p.school_id = s.id - INNER JOIN mtc_admin.receivedCheck r ON r.RowKey = chk.checkCode - WHERE s.urlSlug = @schoolUuid - AND chk.complete = 0 AND chk.processingFailed = 0 - AND chk.isLiveCheck = 1 AND chk.received = 0 - ` - const param: ISqlParameter = { - name: 'schoolUuid', - type: TYPES.UniqueIdentifier, - value: schoolUuid - } - return this.sqlService.query(sql, [param]) - } -} diff --git a/tslib/src/functions/util-replay-check/received-check-payload.service.spec.ts b/tslib/src/functions/util-replay-check/received-check-payload.service.spec.ts deleted file mode 100644 index e514eb03eb..0000000000 --- a/tslib/src/functions/util-replay-check/received-check-payload.service.spec.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { CompressionService } from '../../common/compression-service' -import { type IReceivedCheckPayloadDataService } from './received-check-payload.data.service' -import { ReceivedCheckPayloadService } from './received-check-payload.service' -import mockCompleteCheck from '../../schemas/large-complete-check-V2.json' - -let sut: ReceivedCheckPayloadService -let compressionService: CompressionService -let dataServiceMock: IReceivedCheckPayloadDataService - -const DataServiceMock = jest.fn(() => ({ - fetchCompressedArchives: jest.fn(), - fetchArchivesForSchool: jest.fn() -})) - -describe('received-check-payload.service', () => { - beforeEach(() => { - compressionService = new CompressionService() - dataServiceMock = new DataServiceMock() - sut = new ReceivedCheckPayloadService(compressionService, dataServiceMock) - }) - - test('subject should be defined', () => { - expect(sut).toBeDefined() - }) - - describe('fetch', () => { - test('error should be thrown when checkCodes array is empty', async () => { - const checkCodes: string[] = [] - await expect(sut.fetch(checkCodes)).rejects.toThrow(/at least 1 checkCode is required/) - }) - - test('error should be thrown when at least one checkCode is not a valid UUID', async () => { - const checkCodes = ['82fe6ebd-4933-49b5-accd-4dc78b7303f7', 'foo', 'b1f3f5c0-78ed-417b-b6d9-61280f795eb2'] - await expect(sut.fetch(checkCodes)).rejects.toThrow(/checkCode 'foo' is not a valid UUID/) - }) - - test('if check not found empty array is returned', async () => { - jest.spyOn(dataServiceMock, 'fetchCompressedArchives').mockResolvedValue([]) - const checkCodes = ['721fdee4-26ef-4111-8bbf-b2c5a7d602e3'] - const response = await sut.fetch(checkCodes) - expect(response).toStrictEqual([]) - }) - - test('returns array containing json message with expected properties when found', async () => { - const mockArchive = compressionService.compressToUTF16(JSON.stringify(mockCompleteCheck)) - jest.spyOn(dataServiceMock, 'fetchCompressedArchives').mockResolvedValue([mockArchive]) - const checkCodes = [mockCompleteCheck.checkCode] - const schoolUUID: string = mockCompleteCheck.schoolUUID - const response = await sut.fetch(checkCodes) - if (response === undefined) fail('response not defined') - expect(response).toHaveLength(1) - expect(response[0].checkCode).toStrictEqual(checkCodes[0]) - expect(response[0].schoolUUID).toStrictEqual(schoolUUID) - expect(response[0].version).toBe(2) - expect(response[0].archive).toStrictEqual(mockArchive) - }) - }) - - describe('fetchBySchool', () => { - test('error should be thrown when schoolUuid is empty string', async () => { - await expect(sut.fetchBySchool('')).rejects.toThrow(/schoolUuid is required/) - }) - - test('error should be thrown when schoolUuid is not a valid UUID', async () => { - const schoolUuid: string = 'foo' - await expect(sut.fetchBySchool(schoolUuid)).rejects.toThrow(/schoolUuid is not a valid UUID/) - }) - - test('if no checks found empty array is returned', async () => { - jest.spyOn(dataServiceMock, 'fetchArchivesForSchool').mockResolvedValue([]) - const schoolUuid: string = '721fdee4-26ef-4111-8bbf-b2c5a7d602e3' - await expect(sut.fetchBySchool(schoolUuid)).resolves.toStrictEqual([]) - }) - - test('returns json array of submitted check messages when found', async () => { - const mockDataServiceResponse = [ - { - checkCode: '123', - archive: 'abc' - }, - { - checkCode: '456', - archive: 'def' - }, - { - checkCode: '789', - archive: 'ghi' - } - ] - jest.spyOn(dataServiceMock, 'fetchArchivesForSchool').mockResolvedValue(mockDataServiceResponse) - const schoolUUID: string = '1f5ac7e4-2d79-4ee2-aa22-362cedcb11af' - const messages = await sut.fetchBySchool(schoolUUID) - if (messages === undefined) fail('message not defined') - expect(messages).toHaveLength(3) - const message1 = messages[0] - expect(message1.archive).toBe('abc') - expect(message1.checkCode).toBe('123') - expect(message1.schoolUUID).toStrictEqual(schoolUUID) - expect(message1.version).toBe(2) - const message2 = messages[1] - expect(message2.archive).toBe('def') - expect(message2.checkCode).toBe('456') - expect(message2.schoolUUID).toStrictEqual(schoolUUID) - expect(message2.version).toBe(2) - const message3 = messages[2] - expect(message3.archive).toBe('ghi') - expect(message3.checkCode).toBe('789') - expect(message3.schoolUUID).toStrictEqual(schoolUUID) - expect(message3.version).toBe(2) - }) - }) -}) diff --git a/tslib/src/functions/util-replay-check/received-check-payload.service.ts b/tslib/src/functions/util-replay-check/received-check-payload.service.ts deleted file mode 100644 index 2f5af0ad0e..0000000000 --- a/tslib/src/functions/util-replay-check/received-check-payload.service.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { CompressionService, type ICompressionService } from '../../common/compression-service' -import { validate as validateUuid } from 'uuid' -import { type IReceivedCheckPayloadDataService, ReceivedCheckPayloadDataService } from './received-check-payload.data.service' -import { type SubmittedCheckMessage } from '../../schemas/models' - -export class ReceivedCheckPayloadService { - private readonly compressionService: ICompressionService - private readonly dataService: IReceivedCheckPayloadDataService - - constructor (compressionService?: ICompressionService, dataService?: IReceivedCheckPayloadDataService) { - this.compressionService = compressionService ?? new CompressionService() - this.dataService = dataService ?? new ReceivedCheckPayloadDataService() - } - - async fetch (checkCodes: string[]): Promise { - if (checkCodes.length === 0) { - throw new Error('at least 1 checkCode is required') - } - - for (let index = 0; index < checkCodes.length; index++) { - const checkCode = checkCodes[index] - if (!validateUuid(checkCode)) { - throw new Error(`checkCode '${checkCode}' is not a valid UUID`) - } - } - - const archives = await this.dataService.fetchCompressedArchives(checkCodes) - if (archives === undefined || archives.length === 0) return [] - const decompressedArchives = archives.map(a => this.compressionService.decompressFromUTF16(a)) - const payloads: SubmittedCheckMessage[] = [] - for (let index = 0; index < decompressedArchives.length; index++) { - const da = decompressedArchives[index] - const jsonData = JSON.parse(da) - payloads.push({ - version: 2, - checkCode: jsonData.checkCode, - schoolUUID: jsonData.schoolUUID, - archive: archives[index] - }) - } - return payloads - } - - async fetchBySchool (schoolUuid: string): Promise { - if (schoolUuid === '') { - throw new Error('schoolUuid is required') - } - if (!validateUuid(schoolUuid)) { - throw new Error('schoolUuid is not a valid UUID') - } - const response = await this.dataService.fetchArchivesForSchool(schoolUuid) - if (response.length === 0) return [] - const toReturn: SubmittedCheckMessage[] = response.map(item => { - return { - version: 2, - checkCode: item.checkCode, - schoolUUID: schoolUuid, - archive: item.archive - } - }) - return toReturn - } -} diff --git a/tslib/src/functions/util-school-pin-sampler/index.ts b/tslib/src/functions/util-school-pin-sampler/index.ts deleted file mode 100644 index eedde5aece..0000000000 --- a/tslib/src/functions/util-school-pin-sampler/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' -import moment from 'moment' -import { SchoolPinSampler } from './school-pin-sampler' -import { performance } from 'perf_hooks' -import config from '../../config' - -const functionName = 'util-school-pin-sampler' - -function finish (start: number, context: Context): void { - const end = performance.now() - const durationInMilliseconds = end - start - const timeStamp = new Date().toISOString() - context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) - // required as non-async - context.done() -} - -const schoolPinSampler: AzureFunction = function (context: Context, req: HttpRequest): void { - if (!config.DevTestUtils.TestSupportApi) { - context.log('exiting as not enabled (default behaviour)') - context.done() - return - } - const start = performance.now() - if (req.body === undefined) req.body = {} - const utcNow = req.body?.utcnow !== undefined ? moment(req.body.utcnow) : moment.utc() - const sampleSize = req.body.samplesize ?? 20 - const randomise = req.body.randomise ?? false - context.log.info(`creating sample of ${sampleSize} school pins generated at ${utcNow.toISOString()}`) - const sampler = new SchoolPinSampler() - const sample = sampler.generateSample(sampleSize, utcNow, randomise) - context.res = { - body: sample, - headers: { - 'Content-Type': 'application/json' - } - } - finish(start, context) -} - -export default schoolPinSampler diff --git a/tslib/src/functions/util-test-support-api/index.ts b/tslib/src/functions/util-test-support-api/index.ts deleted file mode 100644 index 2af807217d..0000000000 --- a/tslib/src/functions/util-test-support-api/index.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { type AzureFunction, type Context, type HttpRequest } from '@azure/functions' -import { performance } from 'perf_hooks' -import config from '../../config' -import { SchoolApi } from './school-api' -import { UserApi } from './user-api' - -const functionName = 'util-test-support-api-create-school' - -function finish (start: number, context: Context): void { - const end = performance.now() - const durationInMilliseconds = end - start - const timeStamp = new Date().toISOString() - context.log(`${functionName}: ${timeStamp} run complete: ${durationInMilliseconds} ms`) -} - -const httpTriggerFunc: AzureFunction = async function (context: Context, req: HttpRequest): Promise { - if (!config.DevTestUtils.TestSupportApi) { - context.log('exiting as not enabled (default behaviour)') - return - } - // Respond in 230 seconds or the load balancer will time-out - // https://docs.microsoft.com/en-us/azure/azure-functions/functions-bindings-http-webhook-trigger?tabs=javascript#limits - const start = performance.now() - const entity = context.bindingData.entity - - switch (entity) { - case 'school': - if (req.method === 'PUT') { - await createSchool(context, req) - } - break - - case 'user': - if (req.method === 'PUT') { - await createUser(context, req) - } - break - - default: - generateResponse(context, 'Failed', 400, 'Bad request') - } - - finish(start, context) -} - -async function createSchool (context: Context, req: HttpRequest): Promise { - if (req.body === undefined || req.rawBody?.length === 0) { - generateResponse(context, 'Failed', 400, 'Missing body'); return - } - - try { - const schoolApi = new SchoolApi(context.log) - const entity = await schoolApi.create(req.body) - generateResponse(context, 'Success', 201, 'Created', entity) - } catch (error) { - let errorMessage = 'unknown error' - if (error instanceof Error) { - errorMessage = error.message - } - generateResponse(context, 'Failed', 500, errorMessage) - } -} - -async function createUser (context: Context, req: HttpRequest): Promise { - if (req.body === undefined || req.rawBody?.length === 0) { - generateResponse(context, 'Failed', 400, 'Missing body'); return - } - - try { - const userApi = new UserApi(context.log) - const entity = await userApi.create(req.body) - generateResponse(context, 'Success', 201, 'Created', entity) - } catch (error) { - let errorMessage = 'unknown error' - if (error instanceof Error) { - errorMessage = error.message - } - generateResponse(context, 'Failed', 500, errorMessage) - } -} - -const generateResponse = function (context: Context, result: 'Success' | 'Failed', statusCode: number, message: string, entity?: object): void { - const res = { - body: JSON.stringify({ result, message, entity }), - headers: { - Status: statusCode, - 'Content-Type': 'application/json' - } - } - context.res = res -} - -export default httpTriggerFunc diff --git a/tslib/src/sql/sql.service.ts b/tslib/src/sql/sql.service.ts index cd2243fb58..b99748a459 100644 --- a/tslib/src/sql/sql.service.ts +++ b/tslib/src/sql/sql.service.ts @@ -65,7 +65,7 @@ export class SqlService implements ISqlService { async query (sql: string, params?: ISqlParameter[]): Promise { if (config.Logging.DebugVerbosity > 1) { - this.logger.verbose(`sql.service.query(): ${sql}`) + this.logger.trace(`sql.service.query(): ${sql}`) } const query = async (): Promise => { const request = new Request(await connectionPool.getInstance()) @@ -86,7 +86,7 @@ export class SqlService implements ISqlService { */ async modify (sql: string, params: ISqlParameter[]): Promise { if (config.Logging.DebugVerbosity > 1) { - this.logger.verbose(`sql.service.modify(): ${sql}`) + this.logger.trace(`sql.service.modify(): ${sql}`) } const modify = async (): Promise> => { const request = new Request(await connectionPool.getInstance()) diff --git a/tslib/yarn.lock b/tslib/yarn.lock index 82df87b91d..1e6c6e1496 100644 --- a/tslib/yarn.lock +++ b/tslib/yarn.lock @@ -81,6 +81,26 @@ "@azure/core-client" "^1.3.0" "@azure/core-rest-pipeline" "^1.3.0" +"@azure/core-http@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-3.0.4.tgz#024b2909bbc0f2fce08c74f97a21312c4f42e922" + integrity sha512-Fok9VVhMdxAFOtqiiAtg74fL0UJkt0z3D+ouUUxcRLzZNBioPRAMJFVxiWoJljYpXsRi4GDQHzQHDc9AiYaIUQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/core-util" "^1.1.1" + "@azure/logger" "^1.0.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.3" + form-data "^4.0.0" + node-fetch "^2.6.7" + process "^0.11.10" + tslib "^2.2.0" + tunnel "^0.0.6" + uuid "^8.3.0" + xml2js "^0.5.0" + "@azure/core-lro@^2.2.0": version "2.7.2" resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.7.2.tgz#787105027a20e45c77651a98b01a4d3b01b75a08" @@ -128,6 +148,14 @@ https-proxy-agent "^7.0.0" tslib "^2.6.2" +"@azure/core-tracing@1.0.0-preview.13": + version "1.0.0-preview.13" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz#55883d40ae2042f6f1e12b17dd0c0d34c536d644" + integrity sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ== + dependencies: + "@opentelemetry/api" "^1.0.1" + tslib "^2.2.0" + "@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" @@ -174,14 +202,14 @@ tslib "^2.2.0" uuid "^8.3.0" -"@azure/functions@^3.5.0": - version "3.5.1" - resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-3.5.1.tgz#98ac5c18f84835fd5821de404c52abbd458871f6" - integrity sha512-6UltvJiuVpvHSwLcK/Zc6NfUwlkDLOFFx97BHCJzlWNsfiWwzwmTsxJXg4kE/LemKTHxPpfoPE+kOJ8hAdiKFQ== +"@azure/functions@^4.5.0": + version "4.5.0" + resolved "https://registry.yarnpkg.com/@azure/functions/-/functions-4.5.0.tgz#7aea7b0b4f2ae86dcc4d1663909e846ea2cca066" + integrity sha512-WNCiOHMQEZpezxgThD3o2McKEjUEljtQBvdw4X4oE5714eTw76h33kIj0660ZJGEnxYSx4dx18oAbg5kLMs9iQ== dependencies: - iconv-lite "^0.6.3" + cookie "^0.6.0" long "^4.0.0" - uuid "^8.3.0" + undici "^5.13.0" "@azure/identity@^3.4.1": version "3.4.2" @@ -284,21 +312,16 @@ rhea-promise "^3.0.0" tslib "^2.2.0" -"@azure/storage-blob@^12.23.0": - version "12.24.0" - resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.24.0.tgz#d4ae1e29574b4a19d90eaf082cfde95f996d3f9b" - integrity sha512-l8cmWM4C7RoNCBOImoFMxhTXe1Lr+8uQ/IgnhRNMpfoA9bAFWoLG4XrWm6O5rKXortreVQuD+fc1hbzWklOZbw== +"@azure/storage-blob@~12.18.0": + version "12.18.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e" + integrity sha512-BzBZJobMoDyjJsPRMLNHvqHycTGrT8R/dtcTx9qUFcqwSRfGVK9A/cZ7Nx38UQydT9usZGbaDCN75QRNjezSAA== dependencies: "@azure/abort-controller" "^1.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-client" "^1.6.2" - "@azure/core-http-compat" "^2.0.0" + "@azure/core-http" "^3.0.0" "@azure/core-lro" "^2.2.0" "@azure/core-paging" "^1.1.1" - "@azure/core-rest-pipeline" "^1.10.1" - "@azure/core-tracing" "^1.1.2" - "@azure/core-util" "^1.6.1" - "@azure/core-xml" "^1.3.2" + "@azure/core-tracing" "1.0.0-preview.13" "@azure/logger" "^1.0.0" events "^3.0.0" tslib "^2.2.0" @@ -730,6 +753,11 @@ resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-8.4.1.tgz#5d5e8aee8fce48f5e189bf730ebd1f758f491451" integrity sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg== +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@humanwhocodes/config-array@^0.11.14": version "0.11.14" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" @@ -1035,6 +1063,11 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@opentelemetry/api@^1.0.1": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1", "@opentelemetry/api@^1.7.0": version "1.8.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" @@ -1314,6 +1347,14 @@ "@types/tedious" "*" tarn "^3.0.1" +"@types/node-fetch@^2.5.0": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node@*": version "22.2.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-22.2.0.tgz#7cf046a99f0ba4d628ad3088cb21f790df9b0c5b" @@ -1400,6 +1441,13 @@ resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== +"@types/tunnel@^0.0.3": + version "0.0.3" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.3.tgz#f109e730b072b3136347561fc558c9358bb8c6e9" + integrity sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA== + dependencies: + "@types/node" "*" + "@types/ua-parser-js@^0.7.36": version "0.7.39" resolved "https://registry.yarnpkg.com/@types/ua-parser-js/-/ua-parser-js-0.7.39.tgz#832c58e460c9435e4e34bb866e85e9146e12cdbb" @@ -2183,7 +2231,7 @@ cookie-signature@1.0.6: resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.6.0: +cookie@0.6.0, cookie@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== @@ -4554,6 +4602,13 @@ node-abort-controller@^3.1.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== +node-fetch@^2.6.7: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5114,6 +5169,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sax@>=0.6.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + semver@^5.3.0, semver@^5.4.1: version "5.7.2" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" @@ -5498,6 +5558,11 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + triple-beam@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984" @@ -5549,6 +5614,11 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -5657,6 +5727,13 @@ undici-types@~6.13.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.13.0.tgz#e3e79220ab8c81ed1496b5812471afd7cf075ea5" integrity sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg== +undici@^5.13.0: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -5718,6 +5795,19 @@ walker@^1.0.8: dependencies: makeerror "1.0.12" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -5823,6 +5913,19 @@ write-file-atomic@^4.0.2: imurmurhash "^0.1.4" signal-exit "^3.0.7" +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"