From 51f63b820749a79f109c5369cbbb9654e4a23a87 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Thu, 18 Jan 2024 19:33:56 +0200 Subject: [PATCH 01/21] Add LOAD_PROCESS_LOG as a valid logItemType [MRA-751] --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 22f69df5..8347edc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.0", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.1", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", @@ -2962,9 +2962,9 @@ } }, "node_modules/@natlibfi/melinda-rest-api-commons": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.0.tgz", - "integrity": "sha512-G6nlDOZYfAU3FRATNSp/nak7NarGwLfZW+1zodWObeY54WXv8YyrKKKP1zYJJK+lX2ETb4U4DTG6ia9qxK46jA==", + "version": "4.1.1-alpha.1", + "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.1-alpha.1.tgz", + "integrity": "sha512-6RE4qVK2nlmqKKWUxdecyPvTqPwpA42uNWGHWz17EHXvp58QmzQ/E+J6jEHVqHHBDBb/5T7MLF6D7bUwePyiiA==", "dependencies": { "@natlibfi/marc-record": "^8.0.2", "@natlibfi/marc-record-serializers": "^10.1.2", diff --git a/package.json b/package.json index 46f4201b..f51f29d7 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.0", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.1", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", From 8e890c472ba008dba4bd0d595be34cb10995b01d Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Thu, 18 Jan 2024 19:41:19 +0200 Subject: [PATCH 02/21] Fix logs/correlationIds [MRA-752] --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8347edc1..49642940 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.1", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.2", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", @@ -2962,9 +2962,9 @@ } }, "node_modules/@natlibfi/melinda-rest-api-commons": { - "version": "4.1.1-alpha.1", - "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.1-alpha.1.tgz", - "integrity": "sha512-6RE4qVK2nlmqKKWUxdecyPvTqPwpA42uNWGHWz17EHXvp58QmzQ/E+J6jEHVqHHBDBb/5T7MLF6D7bUwePyiiA==", + "version": "4.1.1-alpha.2", + "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.1-alpha.2.tgz", + "integrity": "sha512-iCzttaQjoVTFWcUP6yhyDeKiXAUQd6hwOwBiE02rdLjo7GB8WJ0ARKyhiR1Hf91SiioxxyiT7kwWHeYn8dA8Fw==", "dependencies": { "@natlibfi/marc-record": "^8.0.2", "@natlibfi/marc-record-serializers": "^10.1.2", diff --git a/package.json b/package.json index f51f29d7..9ee09095 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.1", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.2", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", From 4f72a689848b854f0dbcfeb5bc9d455c6b8629fe Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Thu, 18 Jan 2024 19:58:33 +0200 Subject: [PATCH 03/21] Fix checking misformatted tiemstamp [MRA-753] --- src/routes/queryUtils.js | 42 +++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/routes/queryUtils.js b/src/routes/queryUtils.js index bc9103bd..73e6f0fa 100644 --- a/src/routes/queryUtils.js +++ b/src/routes/queryUtils.js @@ -79,25 +79,30 @@ export function checkQueryParams(req, res, next) { if (!(/^\[.*\]$/u).test(timestampArrayString)) { return false; } - - const timestampArray = JSON.parse(timestampArrayString); - const invalidTimestamps = timestampArray.some(timestamp => { - if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}\.\d{3}Z/u).test(timestamp)) { - return false; - } - - if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}$/u).test(timestamp)) { + logger.debug(`TimestampArrayString: ${timestampArrayString}`); + try { + const timestampArray = JSON.parse(timestampArrayString); + const invalidTimestamps = timestampArray.some(timestamp => { + if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}\.\d{3}Z/u).test(timestamp)) { + return false; + } + + if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}$/u).test(timestamp)) { + return false; + } + + return true; + }); + + if (invalidTimestamps) { return false; } return true; - }); - - if (invalidTimestamps) { + } catch (err) { + logger.debug(`Parsing timestampArrayString ${timestampArrayString} failed: ${err.message}`); return false; } - - return true; } function checkQueueItemState(queueItemState) { @@ -112,20 +117,9 @@ export function checkQueryParams(req, res, next) { return states[queueItemState]; } - // We'd propably like to get these from commons? function checkLogItemType(logItemType) { const logItemTypes = LOG_ITEM_TYPE; - /*const logItemTypes = { - MERGE_LOG: 'MERGE_LOG', - //MATCH_VALIDATION_LOG: 'MATCH_VALIDATION_LOG', - MATCH_LOG: 'MATCH_LOG', - SPLITTER_LOG: 'SPLITTER_LOG', - LOAD_PROCESS_LOG: 'LOAD_PROCESS_LOG', - INPUT_RECORD_LOG: 'INPUT_RECORD_LOG', - RESULT_RECORD_LOG: 'RESULT_RECORD_LOG' - };*/ - if (logItemTypes[logItemType]) { return true; } From 9d129294e957b52f184e7f758c35207106dc8968 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Thu, 18 Jan 2024 20:14:56 +0200 Subject: [PATCH 04/21] Add logging for KVP-authorization [MRA-754] --- src/routes/routeUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/routeUtils.js b/src/routes/routeUtils.js index 9abab86e..2d38c1e0 100644 --- a/src/routes/routeUtils.js +++ b/src/routes/routeUtils.js @@ -7,7 +7,7 @@ import {version as uuidVersion, validate as uuidValidate} from 'uuid'; const logger = createLogger(); export function authorizeKVPOnly(req, res, next) { - logger.debug(`Checking ${JSON.stringify(req.user.id)} for KVP-authorization`); + logger.debug(`Checking ${JSON.stringify(req.user.id)} for KVP-authorization: ${req.user.authorization}`); if (req.user.authorization.includes('KVP')) { logger.debug(`We have user with KVP-authorization`); return next(); From 6fad3183b64f7a7e6f1565a61891637d41738285 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 11:47:03 +0200 Subject: [PATCH 05/21] 3.3.5-alpha.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 49642940..a694a825 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.4", + "version": "3.3.5-alpha.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.4", + "version": "3.3.5-alpha.1", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 9ee09095..697bc415 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.4", + "version": "3.3.5-alpha.1", "main": "dist/index.js", "engines": { "node": ">=18" From 017413e5c93cc6e6f7e29d945ea45bdf4fd43e0c Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 11:54:50 +0200 Subject: [PATCH 06/21] 3.3.5-alpha.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a694a825..86177735 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.1", + "version": "3.3.5-alpha.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.1", + "version": "3.3.5-alpha.2", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 697bc415..32bb47de 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.1", + "version": "3.3.5-alpha.2", "main": "dist/index.js", "engines": { "node": ">=18" From 3129a22d23bc87d848c104703f6652a01ed258ca Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 13:20:54 +0200 Subject: [PATCH 07/21] Add security and logItemTypes to swagger [MRA-754] --- src/api.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/api.yaml b/src/api.yaml index 6c1cdb46..fed3884e 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -410,6 +410,8 @@ paths: summary: Add a record to a bulk job tags: - /bulk/ + security: + - httpBasic: [] responses: '200': description: OK @@ -425,6 +427,8 @@ paths: summary: Retrieve current state of a job tags: - /bulk/ + security: + - httpBasic: [] responses: '200': description: OK @@ -432,6 +436,8 @@ paths: summary: Update state of a job tags: - /bulk/ + security: + - httpBasic: [] responses: '200': description: OK @@ -510,6 +516,8 @@ paths: summary: Query job logs tags: - /logs/ + security: + - httpBasic: [] responses: '200': description: Array of job logs @@ -525,6 +533,8 @@ paths: allOf: - default: MERGE_LOG - $ref: '#/components/schemas/logItemType' + security: + - httpBasic: [] responses: '200': description: Array of job log ids @@ -543,10 +553,14 @@ paths: responses: '200': description: A job log item + security: + - httpBasic: [] put: summary: Protect a job log tags: - /logs/ + security: + - httpBasic: [] responses: '200': description: Array of job logs @@ -554,6 +568,8 @@ paths: summary: Delete a job log tags: - /logs/ + security: + - httpBasic: [] responses: '200': description: Array of job logs @@ -568,6 +584,10 @@ components: enum: - MERGE_LOG - MATCH_LOG + - LOAD_PROCESS_LOG + - SPLITTER_LOG + - INPUT_RECORD_LOG + - RESULT_RECORD_LOG jobLogId: description: Id for a job log (uuid) type: string From f77c499efda282d1427773841d92697233dde3aa Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 13:29:18 +0200 Subject: [PATCH 08/21] 3.3.5-alpha.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 86177735..a67c48a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.2", + "version": "3.3.5-alpha.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.2", + "version": "3.3.5-alpha.3", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 32bb47de..4e7ef28c 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.2", + "version": "3.3.5-alpha.3", "main": "dist/index.js", "engines": { "node": ">=18" From 87adf147ac478eecf835956a5d84fe71f918fa07 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 14:08:54 +0200 Subject: [PATCH 09/21] Update swagger * add logs/catalogers * add logs/correlationIds * update descriptions --- src/api.yaml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index fed3884e..ab9dadf3 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -520,10 +520,10 @@ paths: - httpBasic: [] responses: '200': - description: Array of job logs + description: Array of correlationIds for jobs that have logs available /logs/list: get: - summary: List job logs + summary: List correlationIds that have job logs available tags: - /logs/ parameters: @@ -533,11 +533,27 @@ paths: allOf: - default: MERGE_LOG - $ref: '#/components/schemas/logItemType' + - name: expanded + description: Get expanded info on jobs that have logs instead just correlationIds + in: query + schema: + type: boolean + default: false + security: + - httpBasic: [] + responses: + '200': + description: Array of correlationIds or expanded info on jobs that have logs available + /logs/catalogers: + get: + summary: List catalogers that have triggered jobs that have logs availabel + tags: + - /logs/ security: - httpBasic: [] responses: '200': - description: Array of job log ids + description: Array of catalogers that have triggered jobs that have logs available /logs/{id}: parameters: - name: id From 1aafd1e3ac70bead2ad029a9e1c58098ce6fde69 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 14:51:00 +0200 Subject: [PATCH 10/21] Update swagger --- src/api.yaml | 42 +++++++++++++++++++++++++++++++++++++----- src/interfaces/logs.js | 4 ++++ src/routes/logs.js | 2 ++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index ab9dadf3..8848271d 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -511,16 +511,47 @@ paths: description: The credentials are not authorized for this operation '404': description: The record does not exist - /logs/: + /logs: get: summary: Query job logs tags: - /logs/ + parameters: + - name: correlationId + description: Queue item identifier (correlationId) for job + in: query + required: false + schema: + $ref: '#/components/schemas/correlationId' + - name: blobSequence + in: query + required: false + schema: + type: integer + - name: logItemType + in: query + required: false + schema: + allOf: + - default: MERGE_LOG + - $ref: '#/components/schemas/logItemType' + - name: limit + in: query + required: false + schema: + type: integer + default: 5 + - name: skip + in: query + required: false + schema: + type: integer + default: 0 security: - httpBasic: [] responses: '200': - description: Array of correlationIds for jobs that have logs available + description: Array of JobLogItems /logs/list: get: summary: List correlationIds that have job logs available @@ -536,6 +567,7 @@ paths: - name: expanded description: Get expanded info on jobs that have logs instead just correlationIds in: query + required: false schema: type: boolean default: false @@ -546,7 +578,7 @@ paths: description: Array of correlationIds or expanded info on jobs that have logs available /logs/catalogers: get: - summary: List catalogers that have triggered jobs that have logs availabel + summary: List catalogers that have triggered jobs that have logs available tags: - /logs/ security: @@ -557,13 +589,13 @@ paths: /logs/{id}: parameters: - name: id - description: Id for a job log + description: correlationId for a job in: path required: true schema: $ref: '#/components/schemas/jobLogId' get: - summary: Retrieve a job log + summary: Retrieve one MERGE_LOG -type of logItem for a job tags: - /logs/ responses: diff --git a/src/interfaces/logs.js b/src/interfaces/logs.js index 7daa93cd..18ecf04e 100644 --- a/src/interfaces/logs.js +++ b/src/interfaces/logs.js @@ -12,6 +12,10 @@ export default async function ({mongoUri}) { return {getLogs, doLogsQuery, getListOfCatalogers, getListOfCorrelationIds, getListOfLogs, getExpandedListOfLogs, protectLog, removeLog}; + // routes/getLogs -> interfaces -> getLogs -> mongoLog queryByIds + // DEVELOP: currently return *one MERGE_LOG* for correlationId if such exists + // getLogs reads *just* correlationId, and returns one MERGE_LOG for it + // this is kinda useless async function getLogs(params) { logger.debug(`getLogs: params: ${JSON.stringify(params)}`); logger.debug(`Getting action logs for ${params.correlationId}`); diff --git a/src/routes/logs.js b/src/routes/logs.js index 9f60f669..3f26e049 100644 --- a/src/routes/logs.js +++ b/src/routes/logs.js @@ -38,6 +38,8 @@ export default async function ({mongoUri}) { } } + // routes/getLogs -> interfaces -> getLogs -> mongoLog queryByIds + // DEVELOP: currently return *one MERGE_LOG* for correlationId if such exists async function getLogs(req, res, next) { logger.verbose('routes/logs getLogs'); try { From 01b194ed71984666213ee2ff8f59dc7e45a61389 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 14:51:15 +0200 Subject: [PATCH 11/21] 3.3.5-alpha.4 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a67c48a6..51ba9d57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.3", + "version": "3.3.5-alpha.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.3", + "version": "3.3.5-alpha.4", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 4e7ef28c..77c13ab3 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.3", + "version": "3.3.5-alpha.4", "main": "dist/index.js", "engines": { "node": ">=18" From 290904c80076cf56947bdcc3882805f5f102c002 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 15:02:12 +0200 Subject: [PATCH 12/21] Update swagger --- src/api.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/api.yaml b/src/api.yaml index 8848271d..ca7bfa77 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -586,6 +586,17 @@ paths: responses: '200': description: Array of catalogers that have triggered jobs that have logs available + /logs/correlationIds: + get: + summary: List correlationIds that have logs available + tags: + - /logs/ + security: + - httpBasic: [] + responses: + '200': + description: Array of correlationIds that have logs available + /logs/{id}: parameters: - name: id From 71594ee9dc0c08c921a7aa329b2ff017349b8d84 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 15:29:28 +0200 Subject: [PATCH 13/21] Update swagger --- src/api.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/api.yaml b/src/api.yaml index ca7bfa77..62260a3d 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -277,6 +277,41 @@ paths: enum: - OLD - NEW + - name: noStream + description: Start bulk job as noStream/batch bulk job and wait for records + in: query + required: false + schema: + type: boolean + default: false + - name: noop + description: Do not create/update the record + in: query + schema: + type: boolean + default: false + - name: unique + description: Do not create the record if there are duplicates in the datastore + in: query + schema: + type: boolean + default: true + - name: merge + description: >- + Merge incoming record to datastore record if a duplicate record is + found + in: query + schema: + type: boolean + default: false + - name: skipNoChangeUpdates + description: >- + Do not update the datastore record if update would result in no + changes + in: query + schema: + type: boolean + default: false - name: pRejectFile description: Error log file location in: query From 8922710c83ffce9951391cdc1acb654c35c1574e Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 15:31:50 +0200 Subject: [PATCH 14/21] Update swagger --- src/api.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index 62260a3d..87d6f50c 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -413,7 +413,7 @@ paths: - /bulk/ parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: query required: true schema: @@ -436,7 +436,7 @@ paths: /bulk/record/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: @@ -453,7 +453,7 @@ paths: /bulk/state/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: @@ -479,7 +479,7 @@ paths: /bulk/content/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: @@ -521,7 +521,7 @@ paths: - /prio/ parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: query required: false schema: From 4cb536b0e5900e137687416353100ffc1dd81021 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 16:04:36 +0200 Subject: [PATCH 15/21] Update swagger --- src/api.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index 87d6f50c..71786227 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -650,23 +650,37 @@ paths: security: - httpBasic: [] put: - summary: Protect a job log + summary: Protect/unprotect a job's / a job's blobSequence's logs tags: - /logs/ + parameters: + - name: blobSequence + in: query + required: false + schema: + type: integer security: - httpBasic: [] responses: '200': - description: Array of job logs + description: Mongo response object for update delete: - summary: Delete a job log + summary: Delete a job's logs tags: - /logs/ + parameters: + - name: force + description: Force deletion of protected logs + in: query + required: false + schema: + type: boolean + default: false security: - httpBasic: [] responses: '200': - description: Array of job logs + description: Mongo response object for deletion components: securitySchemes: httpBasic: From f8afc11d2fbd1b1fea4d4a9a7625eafa036c7970 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 16:04:46 +0200 Subject: [PATCH 16/21] 3.3.5-alpha.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 51ba9d57..bb230405 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.4", + "version": "3.3.5-alpha.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.4", + "version": "3.3.5-alpha.5", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 77c13ab3..5cf018af 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.4", + "version": "3.3.5-alpha.5", "main": "dist/index.js", "engines": { "node": ">=18" From 2dbfff66f3cea3b1878ad459ceb6a004a607a019 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 18:42:44 +0200 Subject: [PATCH 17/21] Update swagger --- src/api.yaml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index 71786227..26c2ab8b 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -586,10 +586,15 @@ paths: - httpBasic: [] responses: '200': - description: Array of JobLogItems + description: Array of JobLogItems, or empty array if not logItems are found. /logs/list: get: - summary: List correlationIds that have job logs available + summary: >- + Get list of correlationIds (or expanded info) for jobs that have logs available. + Note that logItemType defaults to MERGE_LOG, if its not given as a parameter. + Note that most other query parameters are only usable with expanded = true. + Note that expanded is not currently usable with LOAD_PROCESS_REPORT or + SPLITTER_LOG logItemType. tags: - /logs/ parameters: @@ -606,6 +611,29 @@ paths: schema: type: boolean default: false + - name: logItemTypes + description: Comma-separated list of logItemTypes (see '#/components/schemas/logItemType') + example: 'INPUT_RECORD_LOG,MATCH_LOG' + in: query + schema: + type: string + - name: catalogers + description: Comma-separated list of catalogers + example: 'TEST1234,FOOBA0000,ZOO-TEST' + in: query + required: false + schema: + type: string + - name: creationTime + description: >- + String array of one or two YYYY-MM-YY-formatted timestamps. + First date is dateAfter, second is dateBefore for + filtering logs by creationTime. + example: '["2023-12-01","2023-12-31"]' + in: query + required: false + schema: + type: string security: - httpBasic: [] responses: From ea279e47ba0d80cb3582046bf034e7a1297631e9 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 18:42:51 +0200 Subject: [PATCH 18/21] 3.3.5-alpha.6 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index bb230405..b27b8585 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.5", + "version": "3.3.5-alpha.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.5", + "version": "3.3.5-alpha.6", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index 5cf018af..a363a600 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.5", + "version": "3.3.5-alpha.6", "main": "dist/index.js", "engines": { "node": ">=18" From d2b9333ca88195a2a7053f70ca9c5de40e59e458 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 19:08:59 +0200 Subject: [PATCH 19/21] Update swagger --- src/api.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api.yaml b/src/api.yaml index 26c2ab8b..d8ec6066 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -589,7 +589,8 @@ paths: description: Array of JobLogItems, or empty array if not logItems are found. /logs/list: get: - summary: >- + summary: Get list of correlationIds (or expanded info) for jobs that have logs available. + description: >- Get list of correlationIds (or expanded info) for jobs that have logs available. Note that logItemType defaults to MERGE_LOG, if its not given as a parameter. Note that most other query parameters are only usable with expanded = true. From 6e704e3ad3434770540f45df2e2c521c59f8f5eb Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 20:04:01 +0200 Subject: [PATCH 20/21] Update swagger --- src/api.yaml | 95 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/src/api.yaml b/src/api.yaml index d8ec6066..5fa8a009 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -8,7 +8,7 @@ tags: - name: /bulk/ description: >- Operate on bibliographic records in bulk format. Also admin operations for - operatiing on bib records in bulk format. + operating on bib records in bulk format. - name: /prio/ description: Admin operations for operating on single bibliographic records - name: /logs/ @@ -259,6 +259,12 @@ paths: /bulk/: post: summary: Create a bulk job for operating on several records + description: >- + A bulk job can be created: +
  • as a streamBulk job (default), which requires a request body that contains record(s) for the job, or +
  • as a batchBulk (noStreamBulk) job, where initial POST does not require a request body + + In case of batchBulk jobs the records are added one by one to the job by POST requests to bulk/record/{correlationId} endpoint, and the job is started by updating it's state by a PUT request to bulk/state/{correlationId} endpoint tags: - /bulk/ parameters: @@ -387,6 +393,48 @@ paths: required: false schema: $ref: '#/components/schemas/correlationId' + - name: showAll + in: query + required: false + schema: + type: boolean + default: false + - name: showOperationSettings + in: query + required: false + schema: + type: boolean + default: false + - name: showRecordLoadParams + in: query + required: false + schema: + type: boolean + default: false + - name: showImportJobState + in: query + required: false + schema: + type: boolean + default: false + - name: recordsAsReport + in: query + required: false + schema: + type: boolean + default: false + - name: noRecords + in: query + required: false + schema: + type: boolean + default: false + - name: noIds + in: query + required: false + schema: + type: boolean + default: false security: - httpBasic: [] responses: @@ -441,12 +489,30 @@ paths: required: true schema: $ref: '#/components/schemas/correlationId' - put: + post: summary: Add a record to a bulk job tags: - /bulk/ security: - httpBasic: [] + requestBody: + description: Contains a single record + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/marcRecord' + application/xml: + schema: + type: string + example: + $ref: '#/components/examples/MARCXML' + application/marc: + schema: + type: string + format: binary + example: + $ref: '#/components/examples/ISO2709' responses: '200': description: OK @@ -469,6 +535,19 @@ paths: description: OK put: summary: Update state of a job + description: >- + Update state (queueItemState) of an existing job.
    + Usually used to start a batchBulk job by updating it's state to PENDING_VALIDATION.
    + Note that other manual state changes may cause unexpected problems in the bulk job. + parameters: + - name: status + description: >- + Target queueItemState. + in: query + required: true + schema: + type: string + default: 'PENDING_VALIDATION' tags: - /bulk/ security: @@ -591,9 +670,9 @@ paths: get: summary: Get list of correlationIds (or expanded info) for jobs that have logs available. description: >- - Get list of correlationIds (or expanded info) for jobs that have logs available. - Note that logItemType defaults to MERGE_LOG, if its not given as a parameter. - Note that most other query parameters are only usable with expanded = true. + Get list of correlationIds (or expanded info) for jobs that have logs available.
    + Note that *logItemType* defaults to MERGE_LOG, if its not given as a parameter.
    + Note that most other query parameters are only usable with *expanded* = true.
    Note that expanded is not currently usable with LOAD_PROCESS_REPORT or SPLITTER_LOG logItemType. tags: @@ -614,20 +693,20 @@ paths: default: false - name: logItemTypes description: Comma-separated list of logItemTypes (see '#/components/schemas/logItemType') - example: 'INPUT_RECORD_LOG,MATCH_LOG' + example: 'MERGE_LOG,MATCH_LOG' in: query schema: type: string - name: catalogers description: Comma-separated list of catalogers - example: 'TEST1234,FOOBA0000,ZOO-TEST' + example: 'TEST1234,FOOBA0000' in: query required: false schema: type: string - name: creationTime description: >- - String array of one or two YYYY-MM-YY-formatted timestamps. + String array of one or two YYYY-MM-YY-formatted timestamps.
    First date is dateAfter, second is dateBefore for filtering logs by creationTime. example: '["2023-12-01","2023-12-31"]' From b1b6cbfcf3c2be4e42c82791ec17538f8bc91257 Mon Sep 17 00:00:00 2001 From: Minttu Hurme Date: Fri, 19 Jan 2024 20:05:58 +0200 Subject: [PATCH 21/21] 3.3.5-alpha.7 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b27b8585..51e31c66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.6", + "version": "3.3.5-alpha.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.5-alpha.6", + "version": "3.3.5-alpha.7", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", diff --git a/package.json b/package.json index a363a600..4eecf9a3 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.5-alpha.6", + "version": "3.3.5-alpha.7", "main": "dist/index.js", "engines": { "node": ">=18"