diff --git a/api/lib/entities/harvest-session.dto.js b/api/lib/entities/harvest-session.dto.js index 4c74c8820..a50f06bb1 100644 --- a/api/lib/entities/harvest-session.dto.js +++ b/api/lib/entities/harvest-session.dto.js @@ -24,7 +24,7 @@ const schema = { }), beginDate: Joi.string().regex(/^\d{4}-\d{2}$/), endDate: Joi.string().regex(/^\d{4}-\d{2}$/), - reportTypes: Joi.array().items(Joi.string()), + reportTypes: Joi.array().items(Joi.string().lowercase().trim()), timeout: Joi.number(), allowFaulty: Joi.boolean(), downloadUnsupported: Joi.boolean(), diff --git a/api/lib/entities/harvest-session.service.js b/api/lib/entities/harvest-session.service.js index 0118222b0..fe12f6c3f 100644 --- a/api/lib/entities/harvest-session.service.js +++ b/api/lib/entities/harvest-session.service.js @@ -41,6 +41,7 @@ const { queryToPrismaFilter } = require('../controllers/utils'); /* eslint-enable max-len */ const DEFAULT_HARVESTED_REPORTS = new Set(config.get('counter.defaultHarvestedReports')); +const defaultHarvestedReports = Array.from(DEFAULT_HARVESTED_REPORTS).map((r) => r.toLowerCase()); module.exports = class HarvestSessionService extends BasePrismaService { /** @type {BasePrismaService.TransactionFnc} */ @@ -367,7 +368,7 @@ module.exports = class HarvestSessionService extends BasePrismaService { } // Get report types - let reportTypes = Array.from(new Set(session.reportTypes)); + let reportTypes = Array.from(new Set(session.reportTypes)).map((r) => r?.toLowerCase?.()); /** @type {(SushiCredentials & { endpoint: SushiEndpoint, institution: Institution })[]} */ let credentialsToHarvest = []; @@ -430,7 +431,12 @@ module.exports = class HarvestSessionService extends BasePrismaService { } // Get supported reports - const { supportedReportsUpdatedAt } = endpoint; + const { + supportedReportsUpdatedAt, + ignoredReports, + additionalReports, + } = endpoint; + const oneMonthAgo = subMonths(new Date(), 1); let supportedReports = []; @@ -465,14 +471,26 @@ module.exports = class HarvestSessionService extends BasePrismaService { } if (reportTypes.includes('all')) { - reportTypes = Array.from(DEFAULT_HARVESTED_REPORTS); + reportTypes = defaultHarvestedReports; } - const supportedReportsSet = new Set(supportedReports); + const supportedReportsSet = new Set([ + // If there's no support list available, assume the default list is supported + ...(supportedReports.length > 0 ? supportedReports : defaultHarvestedReports), + ...additionalReports, + ]); + const ignoredReportsSet = new Set(ignoredReports); + + if (!session.downloadUnsupported) { + // Filter supported reports based on session params + if (supportedReportsSet.size > 0) { + reportTypes = reportTypes.filter((reportId) => supportedReportsSet.has(reportId)); + } - // Filter supported report based on session params - if (!session.downloadUnsupported && supportedReportsSet.size > 0) { - reportTypes = reportTypes.filter((reportId) => supportedReportsSet.has(reportId)); + // Remove reports that should be ignored + if (ignoredReportsSet.size > 0) { + reportTypes = reportTypes.filter((reportId) => !ignoredReportsSet.has(reportId)); + } } // Add jobs diff --git a/api/lib/entities/sushi-endpoints.dto.js b/api/lib/entities/sushi-endpoints.dto.js index 60cba1d68..4a0b5a861 100644 --- a/api/lib/entities/sushi-endpoints.dto.js +++ b/api/lib/entities/sushi-endpoints.dto.js @@ -32,6 +32,9 @@ const schema = { testedReport: Joi.string().allow('').empty(null).lowercase(), tags: Joi.array().items(Joi.string()), + ignoredReports: Joi.array().items(Joi.string().lowercase()), + additionalReports: Joi.array().items(Joi.string().lowercase()), + credentials: Joi.array().items(Joi.object()), disabledUntil: Joi.date().allow(null), diff --git a/api/lib/services/processors/harvest/steps/insert.js b/api/lib/services/processors/harvest/steps/insert.js index f75805e91..95a55c2b9 100644 --- a/api/lib/services/processors/harvest/steps/insert.js +++ b/api/lib/services/processors/harvest/steps/insert.js @@ -56,6 +56,8 @@ function list2object(list, opts = {}) { * @param {import('..').ProcessorStepParam} params */ module.exports = async function process(params) { + const now = new Date(); + const { task: { data: task, @@ -166,6 +168,8 @@ module.exports = async function process(params) { X_Endpoint_ID: credentials.endpoint.id, X_Package: credentials.packages, X_Tags: credentials.tags, + X_Harvested_At: now, + X_Endpoint_Name: credentials.endpoint.vendor, X_Endpoint_Tags: credentials.endpoint.tags, X_Harvest_ID: sessionId, diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index 8b9053d4e..7ded3ebe2 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -410,6 +410,10 @@ model SushiEndpoint { paramSeparator String? /// List report IDs that are supported by the endpoint supportedReports String[] + /// List of report IDs that should be ignored, even if the endpoint indicates that they are supported + ignoredReports String[] + /// Additional report IDs to be added to the list of supported reports provided by the endpoint + additionalReports String[] /// Date on which the list of supported reports was last updated supportedReportsUpdatedAt DateTime? /// Report used when testing endpoint diff --git a/front/components/EndpointForm.vue b/front/components/EndpointForm.vue index ad716c0d1..9581a9d0c 100644 --- a/front/components/EndpointForm.vue +++ b/front/components/EndpointForm.vue @@ -63,7 +63,38 @@ :items="supportedReports" placeholder="pr" outlined - /> + > + + + + @@ -125,6 +156,73 @@ + + {{ $t('endpoints.harvestedReports') }} + + + +

{{ $t('endpoints.ignoredReportsDesc') }}

+ + + + + + + +

{{ $t('endpoints.additionalReportsDesc') }}

+ + + + + + + + {{ $t('endpoints.queryParameters') }} @@ -227,6 +325,7 @@ export default { saving: false, valid: false, formTitle: '', + supportedReportsSearch: '', supportedReports: [], @@ -234,6 +333,8 @@ export default { id: null, params: [], tags: [], + ignoredReports: [], + additionalReports: [], vendor: '', sushiUrl: '', description: '', @@ -277,6 +378,12 @@ export default { this.endpointForm.id = data.id; this.endpointForm.params = Array.isArray(data.params) ? data.params : []; this.endpointForm.tags = Array.isArray(data.tags) ? data.tags : []; + this.endpointForm.ignoredReports = ( + Array.isArray(data.ignoredReports) ? data.ignoredReports : [] + ); + this.endpointForm.additionalReports = ( + Array.isArray(data.additionalReports) ? data.additionalReports : [] + ); this.endpointForm.vendor = data.vendor || ''; this.endpointForm.sushiUrl = data.sushiUrl || ''; diff --git a/front/locales/en.json b/front/locales/en.json index c9e4c0d1f..994e15980 100755 --- a/front/locales/en.json +++ b/front/locales/en.json @@ -210,6 +210,11 @@ "requireRequestorId": "Requestor ID is required", "requireApiKey": "An API key is required", "isSushiCompliant": "The endpoint is SUSHI compliant", + "harvestedReports": "Harvested reports", + "ignoredReportsDesc": "The reports listed below will be considered as unsupported, even if they are present in the list provided by the endpoint.", + "ignoredReports": "Ignored reports", + "additionalReportsDesc": "The reports listed below will be considered as supported, even if they are not present in the list provided by the endpoint.", + "additionalReports": "Additional reports", "ignoreReportValidation": "Ignore report validation", "description": "Description", "paramSeparatorDesc": "ezMESURE automatically adds {0}, {1} and {2} parameters while harvesting. If the endpoint does not accept the standard separator {separator} for this multivalued fields, please enter the separator here.", @@ -583,6 +588,7 @@ "checkCredentials": "Check credentials", "supportedReports": "Supported reports", "supportedReportsOnPlatform": "Reports supported by the publisher platform", + "supportedReportsUnavailable": "The list of supported reports is not available for this publisher platform.", "failedToFetchReports": "Failed to fetch the list of available reports", "onlyShowMasterReports": "Only show master reports" }, @@ -870,5 +876,7 @@ "invalidate": "Invalidate", "reason": "Reason: {reason}", "groupBy": "Group by", - "indeterminate": "indeterminate" + "indeterminate": "indeterminate", + "noMatchFor": "No results matching \"{search}\". Press {key} to add it.", + "enterKey": "enter" } diff --git a/front/locales/fr.json b/front/locales/fr.json index 7aa8a7d5e..a10e8de9f 100755 --- a/front/locales/fr.json +++ b/front/locales/fr.json @@ -210,6 +210,11 @@ "requireRequestorId": "Un requestor ID est nécessaire", "requireApiKey": "Une clé d'API est nécessaire", "isSushiCompliant": "Le point d'accès est conforme SUSHI", + "harvestedReports": "Rapports moissonnés", + "ignoredReportsDesc": "Les rapports saisis ci-dessous seront considérés comme non-supportés, même s'ils sont présents dans la liste fournie par le point d'accès.", + "ignoredReports": "Rapports ignorés", + "additionalReportsDesc": "Les rapports saisis ci-dessous seront considérés comme supportés, même s'ils ne se trouvent pas dans la liste fournie par le point d'accès.", + "additionalReports": "Rapports additionnels", "ignoreReportValidation": "Ignorer la validation du rapport", "description": "Description", "paramSeparatorDesc": "ezMESURE ajoute automatiquement les paramètres {0}, {1} et {2} lors du moissonnage. Si le point d'accès n'accepte pas le séparateur standard {separator} pour ces champs multivalués, renseignez le séparateur ici.", @@ -584,6 +589,7 @@ "checkCredentials": "Vérifier les identifiants", "supportedReports": "Rapports supportés", "supportedReportsOnPlatform": "Rapports supportés par la plateforme éditeur", + "supportedReportsUnavailable": "La liste des rapports supportés n'est pas disponible pour cette plateforme éditeur.", "failedToFetchReports": "Impossible de récupérer la liste des rapports disponibles", "onlyShowMasterReports": "Afficher uniquement les rapports maîtres" }, @@ -870,5 +876,7 @@ "invalidate": "Invalider", "reason": "Raison : {reason}", "groupBy": "Regrouper par", - "indeterminate": "indéterminée" + "indeterminate": "indéterminée", + "noMatchFor": "Aucune correspondance pour \"{search}\". Appuyez sur {key} pour l'ajouter.", + "enterKey": "entrée" } \ No newline at end of file