diff --git a/package-lock.json b/package-lock.json index 22f69df5..51e31c66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,19 @@ { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.4", + "version": "3.3.5-alpha.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@natlibfi/melinda-rest-api-http", - "version": "3.3.4", + "version": "3.3.5-alpha.7", "license": "AGPL-3.0+", "dependencies": { "@babel/runtime": "^7.23.8", "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.0", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.2", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", @@ -2962,9 +2962,9 @@ } }, "node_modules/@natlibfi/melinda-rest-api-commons": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.0.tgz", - "integrity": "sha512-G6nlDOZYfAU3FRATNSp/nak7NarGwLfZW+1zodWObeY54WXv8YyrKKKP1zYJJK+lX2ETb4U4DTG6ia9qxK46jA==", + "version": "4.1.1-alpha.2", + "resolved": "https://registry.npmjs.org/@natlibfi/melinda-rest-api-commons/-/melinda-rest-api-commons-4.1.1-alpha.2.tgz", + "integrity": "sha512-iCzttaQjoVTFWcUP6yhyDeKiXAUQd6hwOwBiE02rdLjo7GB8WJ0ARKyhiR1Hf91SiioxxyiT7kwWHeYn8dA8Fw==", "dependencies": { "@natlibfi/marc-record": "^8.0.2", "@natlibfi/marc-record-serializers": "^10.1.2", diff --git a/package.json b/package.json index 46f4201b..4eecf9a3 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "url": "git@github.com:natlibfi/melinda-rest-api-http.git" }, "license": "AGPL-3.0+", - "version": "3.3.4", + "version": "3.3.5-alpha.7", "main": "dist/index.js", "engines": { "node": ">=18" @@ -34,7 +34,7 @@ "@natlibfi/marc-record-serializers": "^10.1.2", "@natlibfi/melinda-backend-commons": "^2.2.6", "@natlibfi/melinda-commons": "^13.0.11", - "@natlibfi/melinda-rest-api-commons": "^4.1.0", + "@natlibfi/melinda-rest-api-commons": "^4.1.1-alpha.2", "@natlibfi/passport-melinda-aleph": "^2.0.4", "@natlibfi/sru-client": "^6.0.8", "body-parser": "^1.20.2", diff --git a/src/api.yaml b/src/api.yaml index 6c1cdb46..5fa8a009 100644 --- a/src/api.yaml +++ b/src/api.yaml @@ -8,7 +8,7 @@ tags: - name: /bulk/ description: >- Operate on bibliographic records in bulk format. Also admin operations for - operatiing on bib records in bulk format. + operating on bib records in bulk format. - name: /prio/ description: Admin operations for operating on single bibliographic records - name: /logs/ @@ -259,6 +259,12 @@ paths: /bulk/: post: summary: Create a bulk job for operating on several records + description: >- + A bulk job can be created: +
  • as a streamBulk job (default), which requires a request body that contains record(s) for the job, or +
  • as a batchBulk (noStreamBulk) job, where initial POST does not require a request body + + In case of batchBulk jobs the records are added one by one to the job by POST requests to bulk/record/{correlationId} endpoint, and the job is started by updating it's state by a PUT request to bulk/state/{correlationId} endpoint tags: - /bulk/ parameters: @@ -277,6 +283,41 @@ paths: enum: - OLD - NEW + - name: noStream + description: Start bulk job as noStream/batch bulk job and wait for records + in: query + required: false + schema: + type: boolean + default: false + - name: noop + description: Do not create/update the record + in: query + schema: + type: boolean + default: false + - name: unique + description: Do not create the record if there are duplicates in the datastore + in: query + schema: + type: boolean + default: true + - name: merge + description: >- + Merge incoming record to datastore record if a duplicate record is + found + in: query + schema: + type: boolean + default: false + - name: skipNoChangeUpdates + description: >- + Do not update the datastore record if update would result in no + changes + in: query + schema: + type: boolean + default: false - name: pRejectFile description: Error log file location in: query @@ -352,6 +393,48 @@ paths: required: false schema: $ref: '#/components/schemas/correlationId' + - name: showAll + in: query + required: false + schema: + type: boolean + default: false + - name: showOperationSettings + in: query + required: false + schema: + type: boolean + default: false + - name: showRecordLoadParams + in: query + required: false + schema: + type: boolean + default: false + - name: showImportJobState + in: query + required: false + schema: + type: boolean + default: false + - name: recordsAsReport + in: query + required: false + schema: + type: boolean + default: false + - name: noRecords + in: query + required: false + schema: + type: boolean + default: false + - name: noIds + in: query + required: false + schema: + type: boolean + default: false security: - httpBasic: [] responses: @@ -378,7 +461,7 @@ paths: - /bulk/ parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: query required: true schema: @@ -401,22 +484,42 @@ paths: /bulk/record/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: $ref: '#/components/schemas/correlationId' - put: + post: summary: Add a record to a bulk job tags: - /bulk/ + security: + - httpBasic: [] + requestBody: + description: Contains a single record + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/marcRecord' + application/xml: + schema: + type: string + example: + $ref: '#/components/examples/MARCXML' + application/marc: + schema: + type: string + format: binary + example: + $ref: '#/components/examples/ISO2709' responses: '200': description: OK /bulk/state/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: @@ -425,20 +528,37 @@ paths: summary: Retrieve current state of a job tags: - /bulk/ + security: + - httpBasic: [] responses: '200': description: OK put: summary: Update state of a job + description: >- + Update state (queueItemState) of an existing job.
    + Usually used to start a batchBulk job by updating it's state to PENDING_VALIDATION.
    + Note that other manual state changes may cause unexpected problems in the bulk job. + parameters: + - name: status + description: >- + Target queueItemState. + in: query + required: true + schema: + type: string + default: 'PENDING_VALIDATION' tags: - /bulk/ + security: + - httpBasic: [] responses: '200': description: OK /bulk/content/{id}: parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: path required: true schema: @@ -480,7 +600,7 @@ paths: - /prio/ parameters: - name: id - description: Queue item identifier + description: Queue item identifier (correlationId) in: query required: false schema: @@ -505,17 +625,56 @@ paths: description: The credentials are not authorized for this operation '404': description: The record does not exist - /logs/: + /logs: get: summary: Query job logs tags: - /logs/ + parameters: + - name: correlationId + description: Queue item identifier (correlationId) for job + in: query + required: false + schema: + $ref: '#/components/schemas/correlationId' + - name: blobSequence + in: query + required: false + schema: + type: integer + - name: logItemType + in: query + required: false + schema: + allOf: + - default: MERGE_LOG + - $ref: '#/components/schemas/logItemType' + - name: limit + in: query + required: false + schema: + type: integer + default: 5 + - name: skip + in: query + required: false + schema: + type: integer + default: 0 + security: + - httpBasic: [] responses: '200': - description: Array of job logs + description: Array of JobLogItems, or empty array if not logItems are found. /logs/list: get: - summary: List job logs + summary: Get list of correlationIds (or expanded info) for jobs that have logs available. + description: >- + Get list of correlationIds (or expanded info) for jobs that have logs available.
    + Note that *logItemType* defaults to MERGE_LOG, if its not given as a parameter.
    + Note that most other query parameters are only usable with *expanded* = true.
    + Note that expanded is not currently usable with LOAD_PROCESS_REPORT or + SPLITTER_LOG logItemType. tags: - /logs/ parameters: @@ -525,38 +684,111 @@ paths: allOf: - default: MERGE_LOG - $ref: '#/components/schemas/logItemType' + - name: expanded + description: Get expanded info on jobs that have logs instead just correlationIds + in: query + required: false + schema: + type: boolean + default: false + - name: logItemTypes + description: Comma-separated list of logItemTypes (see '#/components/schemas/logItemType') + example: 'MERGE_LOG,MATCH_LOG' + in: query + schema: + type: string + - name: catalogers + description: Comma-separated list of catalogers + example: 'TEST1234,FOOBA0000' + in: query + required: false + schema: + type: string + - name: creationTime + description: >- + String array of one or two YYYY-MM-YY-formatted timestamps.
    + First date is dateAfter, second is dateBefore for + filtering logs by creationTime. + example: '["2023-12-01","2023-12-31"]' + in: query + required: false + schema: + type: string + security: + - httpBasic: [] + responses: + '200': + description: Array of correlationIds or expanded info on jobs that have logs available + /logs/catalogers: + get: + summary: List catalogers that have triggered jobs that have logs available + tags: + - /logs/ + security: + - httpBasic: [] responses: '200': - description: Array of job log ids + description: Array of catalogers that have triggered jobs that have logs available + /logs/correlationIds: + get: + summary: List correlationIds that have logs available + tags: + - /logs/ + security: + - httpBasic: [] + responses: + '200': + description: Array of correlationIds that have logs available + /logs/{id}: parameters: - name: id - description: Id for a job log + description: correlationId for a job in: path required: true schema: $ref: '#/components/schemas/jobLogId' get: - summary: Retrieve a job log + summary: Retrieve one MERGE_LOG -type of logItem for a job tags: - /logs/ responses: '200': description: A job log item + security: + - httpBasic: [] put: - summary: Protect a job log + summary: Protect/unprotect a job's / a job's blobSequence's logs tags: - /logs/ + parameters: + - name: blobSequence + in: query + required: false + schema: + type: integer + security: + - httpBasic: [] responses: '200': - description: Array of job logs + description: Mongo response object for update delete: - summary: Delete a job log + summary: Delete a job's logs tags: - /logs/ + parameters: + - name: force + description: Force deletion of protected logs + in: query + required: false + schema: + type: boolean + default: false + security: + - httpBasic: [] responses: '200': - description: Array of job logs + description: Mongo response object for deletion components: securitySchemes: httpBasic: @@ -568,6 +800,10 @@ components: enum: - MERGE_LOG - MATCH_LOG + - LOAD_PROCESS_LOG + - SPLITTER_LOG + - INPUT_RECORD_LOG + - RESULT_RECORD_LOG jobLogId: description: Id for a job log (uuid) type: string diff --git a/src/interfaces/logs.js b/src/interfaces/logs.js index 7daa93cd..18ecf04e 100644 --- a/src/interfaces/logs.js +++ b/src/interfaces/logs.js @@ -12,6 +12,10 @@ export default async function ({mongoUri}) { return {getLogs, doLogsQuery, getListOfCatalogers, getListOfCorrelationIds, getListOfLogs, getExpandedListOfLogs, protectLog, removeLog}; + // routes/getLogs -> interfaces -> getLogs -> mongoLog queryByIds + // DEVELOP: currently return *one MERGE_LOG* for correlationId if such exists + // getLogs reads *just* correlationId, and returns one MERGE_LOG for it + // this is kinda useless async function getLogs(params) { logger.debug(`getLogs: params: ${JSON.stringify(params)}`); logger.debug(`Getting action logs for ${params.correlationId}`); diff --git a/src/routes/logs.js b/src/routes/logs.js index 9f60f669..3f26e049 100644 --- a/src/routes/logs.js +++ b/src/routes/logs.js @@ -38,6 +38,8 @@ export default async function ({mongoUri}) { } } + // routes/getLogs -> interfaces -> getLogs -> mongoLog queryByIds + // DEVELOP: currently return *one MERGE_LOG* for correlationId if such exists async function getLogs(req, res, next) { logger.verbose('routes/logs getLogs'); try { diff --git a/src/routes/queryUtils.js b/src/routes/queryUtils.js index bc9103bd..73e6f0fa 100644 --- a/src/routes/queryUtils.js +++ b/src/routes/queryUtils.js @@ -79,25 +79,30 @@ export function checkQueryParams(req, res, next) { if (!(/^\[.*\]$/u).test(timestampArrayString)) { return false; } - - const timestampArray = JSON.parse(timestampArrayString); - const invalidTimestamps = timestampArray.some(timestamp => { - if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}\.\d{3}Z/u).test(timestamp)) { - return false; - } - - if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}$/u).test(timestamp)) { + logger.debug(`TimestampArrayString: ${timestampArrayString}`); + try { + const timestampArray = JSON.parse(timestampArrayString); + const invalidTimestamps = timestampArray.some(timestamp => { + if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}\.\d{3}Z/u).test(timestamp)) { + return false; + } + + if ((/^\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}$/u).test(timestamp)) { + return false; + } + + return true; + }); + + if (invalidTimestamps) { return false; } return true; - }); - - if (invalidTimestamps) { + } catch (err) { + logger.debug(`Parsing timestampArrayString ${timestampArrayString} failed: ${err.message}`); return false; } - - return true; } function checkQueueItemState(queueItemState) { @@ -112,20 +117,9 @@ export function checkQueryParams(req, res, next) { return states[queueItemState]; } - // We'd propably like to get these from commons? function checkLogItemType(logItemType) { const logItemTypes = LOG_ITEM_TYPE; - /*const logItemTypes = { - MERGE_LOG: 'MERGE_LOG', - //MATCH_VALIDATION_LOG: 'MATCH_VALIDATION_LOG', - MATCH_LOG: 'MATCH_LOG', - SPLITTER_LOG: 'SPLITTER_LOG', - LOAD_PROCESS_LOG: 'LOAD_PROCESS_LOG', - INPUT_RECORD_LOG: 'INPUT_RECORD_LOG', - RESULT_RECORD_LOG: 'RESULT_RECORD_LOG' - };*/ - if (logItemTypes[logItemType]) { return true; } diff --git a/src/routes/routeUtils.js b/src/routes/routeUtils.js index 9abab86e..2d38c1e0 100644 --- a/src/routes/routeUtils.js +++ b/src/routes/routeUtils.js @@ -7,7 +7,7 @@ import {version as uuidVersion, validate as uuidValidate} from 'uuid'; const logger = createLogger(); export function authorizeKVPOnly(req, res, next) { - logger.debug(`Checking ${JSON.stringify(req.user.id)} for KVP-authorization`); + logger.debug(`Checking ${JSON.stringify(req.user.id)} for KVP-authorization: ${req.user.authorization}`); if (req.user.authorization.includes('KVP')) { logger.debug(`We have user with KVP-authorization`); return next();