From 0a47d480ce131f3ac9c821eff2b7928a5bb9fb8b Mon Sep 17 00:00:00 2001 From: Roman Kalyakin Date: Wed, 15 Jan 2025 19:00:39 +0100 Subject: [PATCH] More permissions tests (#487) * more permissions tests * refactoring and permission integration tests for images * updated redaction policy * make linter happy --- src/hooks/redaction.ts | 2 +- src/models/articles.model.js | 8 +- src/models/images.model.js | 94 ++++--- src/models/text-reuse-passages.model.js | 6 +- src/services/admin/admin.class.ts | 11 +- src/services/articles/articles.hooks.js | 6 +- .../contentItemRedactionPolicyWebApp.yml | 18 +- src/services/images/images.hooks.js | 32 ++- src/services/images/images.service.js | 21 +- .../resources/imageRedactionPolicyWebApp.yml | 17 ++ .../search-exporter/search-exporter.class.js | 2 +- src/services/search/search.hooks.js | 4 +- .../text-reuse-clusters.class.ts | 5 +- .../text-reuse-clusters.hooks.js | 6 +- .../text-reuse-passages.hooks.js | 8 +- .../getContentItemsPermissionsDetails.ts | 11 +- src/util/bigint.ts | 6 + .../integration/use_cases/permissions.test.ts | 265 +++++++++++++++--- 18 files changed, 387 insertions(+), 135 deletions(-) create mode 100644 src/services/images/resources/imageRedactionPolicyWebApp.yml diff --git a/src/hooks/redaction.ts b/src/hooks/redaction.ts index d0d3c0bd..c6ab1d25 100644 --- a/src/hooks/redaction.ts +++ b/src/hooks/redaction.ts @@ -142,7 +142,7 @@ const webappAuthBitmapExtractor = (redactable: Redactable, kind: keyof Authoriza * - the request is made through the app (not the public API) * - user bitmap does not align with the content item bitmap */ -export const webAppTranscriptRedactionCondition: RedactCondition = (context, redactable) => { +export const webAppExploreRedactionCondition: RedactCondition = (context, redactable) => { return ( !inPublicApi(context, redactable) && !bitmapsAlign(context, redactable, x => webappAuthBitmapExtractor(x, 'explore')) diff --git a/src/models/articles.model.js b/src/models/articles.model.js index a4b180f9..5aea1317 100644 --- a/src/models/articles.model.js +++ b/src/models/articles.model.js @@ -1,3 +1,5 @@ +import { OpenPermissions } from '../util/bigint' + const { DataTypes } = require('sequelize') const lodash = require('lodash') const config = require('@feathersjs/configuration')()() @@ -725,9 +727,9 @@ class Article extends BaseArticle { // permissions bitmaps // if it's not defined, set max permissions for compatibility // with old Solr version - bitmapExplore: BigInt(doc.rights_bm_explore_l ?? Number.MAX_SAFE_INTEGER), - bitmapGetTranscript: BigInt(doc.rights_bm_get_tr_l ?? Number.MAX_SAFE_INTEGER), - bitmapGetImages: BigInt(doc.rights_bm_get_img_l ?? Number.MAX_SAFE_INTEGER), + bitmapExplore: BigInt(doc.rights_bm_explore_l ?? OpenPermissions), + bitmapGetTranscript: BigInt(doc.rights_bm_get_tr_l ?? OpenPermissions), + bitmapGetImages: BigInt(doc.rights_bm_get_img_l ?? OpenPermissions), }) if (!doc.pp_plain) { diff --git a/src/models/images.model.js b/src/models/images.model.js index 5c7824cb..41a9bbca 100644 --- a/src/models/images.model.js +++ b/src/models/images.model.js @@ -1,11 +1,13 @@ -const Page = require('./pages.model'); -const Issue = require('./issues.model'); -const Newspaper = require('./newspapers.model'); -const Article = require('./articles.model'); -const { getExternalFragment } = require('../hooks/iiif.js'); +import { OpenPermissions } from '../util/bigint' + +const Page = require('./pages.model') +const Issue = require('./issues.model') +const Newspaper = require('./newspapers.model') +const Article = require('./articles.model') +const { getExternalFragment } = require('../hooks/iiif.js') class Image { - constructor ({ + constructor({ uid = '', type = 'image', coords = [], @@ -17,44 +19,52 @@ class Image { isFront = false, pages = [], article = null, + // permissions bitmaps + bitmapExplore = undefined, + bitmapGetTranscript = undefined, + bitmapGetImages = undefined, } = {}) { - this.uid = String(uid); - this.year = parseInt(year, 10); - this.type = String(type); - this.coords = coords; - this.pages = pages; - this.isFront = Boolean(isFront); + this.uid = String(uid) + this.year = parseInt(year, 10) + this.type = String(type) + this.coords = coords + this.pages = pages + this.isFront = Boolean(isFront) - this.title = String(title); + this.title = String(title) if (date instanceof Date) { - this.date = date; + this.date = date } else { - this.date = new Date(date); + this.date = new Date(date) } if (newspaper instanceof Newspaper) { - this.newspaper = newspaper; + this.newspaper = newspaper } else { - this.newspaper = new Newspaper(newspaper); + this.newspaper = new Newspaper(newspaper) } if (issue instanceof Issue) { - this.issue = issue; + this.issue = issue } else { - this.issue = new Issue(issue); + this.issue = new Issue(issue) } if (article instanceof Article) { - this.article = article; + this.article = article } else if (typeof article === 'string') { - this.article = article; + this.article = article } + + this.bitmapExplore = bitmapExplore + this.bitmapGetTranscript = bitmapGetTranscript + this.bitmapGetImages = bitmapGetImages } - assignIIIF () { + assignIIIF() { this.regions = this.pages.map(page => ({ pageUid: page.uid, coords: this.coords, iiifFragment: getExternalFragment(page.iiif, { coords: this.coords, dim: '250,' }), - })); + })) } /** @@ -62,8 +72,8 @@ class Image { * * @return {function} {Image} image instance. */ - static solrFactory () { - return (doc) => { + static solrFactory() { + return doc => { const img = new Image({ uid: doc.id, newspaper: new Newspaper({ @@ -72,12 +82,13 @@ class Image { issue: new Issue({ uid: doc.meta_issue_id_s, }), - pages: Array.isArray(doc.page_nb_is) - ? doc.page_nb_is.map(num => new Page({ - uid: `${doc.meta_issue_id_s}-p${String(num).padStart(4, '0')}`, - num, - })) - : [], + pages: (Array.isArray(doc.page_nb_is) ? doc.page_nb_is : []).map( + num => + new Page({ + uid: `${doc.meta_issue_id_s}-p${String(num).padStart(4, '0')}`, + num, + }) + ), title: Article.getUncertainField(doc, 'title'), type: doc.item_type_s, year: doc.meta_year_i, @@ -85,13 +96,19 @@ class Image { coords: doc.coords_is, isFront: doc.front_b, article: doc.linked_art_s, - }); - return img; - }; + // permissions bitmaps + // if it's not defined, set max permissions for compatibility + // with old Solr version + bitmapExplore: BigInt(doc.rights_bm_explore_l ?? OpenPermissions), + bitmapGetTranscript: BigInt(doc.rights_bm_get_tr_l ?? OpenPermissions), + bitmapGetImages: BigInt(doc.rights_bm_get_img_l ?? OpenPermissions), + }) + return img + } } } -module.exports = Image; +module.exports = Image module.exports.SOLR_FL = [ 'id', 'coords_is', @@ -108,4 +125,9 @@ module.exports.SOLR_FL = [ 'item_type_s', 'title_txt_fr', '_version_', -]; + // permissions bitmaps + // see https://github.com/search?q=org%3Aimpresso%20rights_bm_explore_l&type=code + 'rights_bm_explore_l', + 'rights_bm_get_tr_l', + 'rights_bm_get_img_l', +] diff --git a/src/models/text-reuse-passages.model.js b/src/models/text-reuse-passages.model.js index c8c2ce53..35bee5f4 100644 --- a/src/models/text-reuse-passages.model.js +++ b/src/models/text-reuse-passages.model.js @@ -1,3 +1,5 @@ +import { OpenPermissions } from '../util/bigint' + const { invert } = require('lodash') const { solrDocsMapCallbackFn } = require('../util/solr/fields') @@ -103,8 +105,8 @@ class TextReusePassage { this.pageNumbers = pageNumbers this.collections = collections - this.bitmapExplore = BigInt(bitmapExplore ?? Number.MAX_SAFE_INTEGER) - this.bitmapGetTranscript = BigInt(bitmapGetTranscript ?? Number.MAX_SAFE_INTEGER) + this.bitmapExplore = BigInt(bitmapExplore ?? OpenPermissions) + this.bitmapGetTranscript = BigInt(bitmapGetTranscript ?? OpenPermissions) } static CreateFromSolr(fieldsToPropsMapper = SolrFieldsToPropsMapper) { diff --git a/src/services/admin/admin.class.ts b/src/services/admin/admin.class.ts index 6f0a15bb..5206fc90 100644 --- a/src/services/admin/admin.class.ts +++ b/src/services/admin/admin.class.ts @@ -7,7 +7,8 @@ import { import { getUserAccountsWithAvailablePermissions, UserAccount } from '../../useCases/getUsersPermissionsDetails' interface FindResponse { - permissionsDetails: ContentItemPermissionsDetails + contentItemsPermissionsDetails: ContentItemPermissionsDetails + imagesPermissionsDetails: ContentItemPermissionsDetails userAccounts: UserAccount[] } interface FindParams {} @@ -18,8 +19,12 @@ export class Service implements IService { constructor(private readonly app: ImpressoApplication) {} async find(params?: Params): Promise { - const permissionsDetails = await getContentItemsPermissionsDetails(this.app.service('simpleSolrClient')) + const [contentItemsPermissionsDetails, imagesPermissionsDetails] = await Promise.all([ + getContentItemsPermissionsDetails(this.app.service('simpleSolrClient'), 'Search'), + getContentItemsPermissionsDetails(this.app.service('simpleSolrClient'), 'Images'), + ]) + const userAccounts = await getUserAccountsWithAvailablePermissions(this.app.get('sequelizeClient')!) - return { permissionsDetails, userAccounts } satisfies FindResponse + return { contentItemsPermissionsDetails, imagesPermissionsDetails, userAccounts } satisfies FindResponse } } diff --git a/src/services/articles/articles.hooks.js b/src/services/articles/articles.hooks.js index ab5b0eed..8f6e913c 100644 --- a/src/services/articles/articles.hooks.js +++ b/src/services/articles/articles.hooks.js @@ -4,7 +4,7 @@ import { redactResponse, redactResponseDataItem, publicApiTranscriptRedactionCondition, - webAppTranscriptRedactionCondition, + webAppExploreRedactionCondition, inPublicApi, } from '../../hooks/redaction' import { loadYamlFile } from '../../util/yaml' @@ -107,7 +107,7 @@ export default { obfuscate(), transformResponseDataItem(transformContentItem, inPublicApi), redactResponseDataItem(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition), - redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition), + redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppExploreRedactionCondition), ], get: [ // save here cache, flush cache here @@ -120,7 +120,7 @@ export default { obfuscate(), transformResponse(transformContentItem, inPublicApi), redactResponse(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition), - redactResponse(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition), + redactResponse(contentItemRedactionPolicyWebApp, webAppExploreRedactionCondition), ], create: [], update: [], diff --git a/src/services/articles/resources/contentItemRedactionPolicyWebApp.yml b/src/services/articles/resources/contentItemRedactionPolicyWebApp.yml index e5e129f8..02447c2f 100644 --- a/src/services/articles/resources/contentItemRedactionPolicyWebApp.yml +++ b/src/services/articles/resources/contentItemRedactionPolicyWebApp.yml @@ -1,7 +1,21 @@ # yaml-language-server: $schema=../../../schema/common/redactionPolicy.json name: content-item-redaction-policy items: - - jsonPath: $.title - valueConverterName: redact + # NOTE: decided to show title and fragment so that guest users can see at least something + # - jsonPath: $.title + # valueConverterName: redact - jsonPath: $.excerpt valueConverterName: redact + - jsonPath: $.content + valueConverterName: redact + # NOTE: decided to show title and fragment so that guest users can see at least something + # - jsonPath: $.matches.[*].fragment + # valueConverterName: redact + - jsonPath: $.regions.[*].iiifFragment + valueConverterName: contextNotAllowedImage + - jsonPath: $.pages.[*].iiif + valueConverterName: contextNotAllowedImage + - jsonPath: $.pages.[*].iiifThumbnail + valueConverterName: contextNotAllowedImage + - jsonPath: $.pages.[*].iiifFragment + valueConverterName: contextNotAllowedImage \ No newline at end of file diff --git a/src/services/images/images.hooks.js b/src/services/images/images.hooks.js index d2f86c18..5ce71e1d 100644 --- a/src/services/images/images.hooks.js +++ b/src/services/images/images.hooks.js @@ -1,19 +1,24 @@ +import { loadYamlFile } from '../../util/yaml' +import { redactResponse, redactResponseDataItem, webAppExploreRedactionCondition } from '../../hooks/redaction' + // const { authenticate } = require('@feathersjs/authentication').hooks; const { - utils, validate, validateEach, queryWithCommonParams, displayQueryParams, REGEX_UID, -} = require('../../hooks/params'); -const { - qToSolrFilter, filtersToSolrQuery, -} = require('../../hooks/search'); + utils, + validate, + validateEach, + queryWithCommonParams, + displayQueryParams, + REGEX_UID, +} = require('../../hooks/params') +const { qToSolrFilter, filtersToSolrQuery } = require('../../hooks/search') -const { - resolveFacets, - resolveQueryComponents, -} = require('../../hooks/search-info'); +const { resolveFacets, resolveQueryComponents } = require('../../hooks/search-info') -const { eachFilterValidator } = require('../search/search.validators'); +const { eachFilterValidator } = require('../search/search.validators') -module.exports = { +export const imageRedactionPolicyWebApp = loadYamlFile(`${__dirname}/resources/imageRedactionPolicyWebApp.yml`) + +export default { before: { all: [ // authenticate('jwt') @@ -92,8 +97,9 @@ module.exports = { resolveFacets(), displayQueryParams(['queryComponents', 'filters']), resolveQueryComponents(), + redactResponseDataItem(imageRedactionPolicyWebApp, webAppExploreRedactionCondition), ], - get: [], + get: [redactResponse(imageRedactionPolicyWebApp, webAppExploreRedactionCondition)], create: [], update: [], patch: [], @@ -109,4 +115,4 @@ module.exports = { patch: [], remove: [], }, -}; +} diff --git a/src/services/images/images.service.js b/src/services/images/images.service.js index ea1495e4..a14b7acb 100644 --- a/src/services/images/images.service.js +++ b/src/services/images/images.service.js @@ -1,16 +1,19 @@ // Initializes the `images` service on path `/images` -const createService = require('./images.class.js'); -const hooks = require('./images.hooks'); +import hooks from './images.hooks' +const createService = require('./images.class.js') module.exports = function (app) { // Initialize our service with any options it requires - app.use('/images', createService({ - app, - name: 'images', - })); + app.use( + '/images', + createService({ + app, + name: 'images', + }) + ) // Get our initialized service so that we can register hooks - const service = app.service('images'); + const service = app.service('images') - service.hooks(hooks); -}; + service.hooks(hooks) +} diff --git a/src/services/images/resources/imageRedactionPolicyWebApp.yml b/src/services/images/resources/imageRedactionPolicyWebApp.yml new file mode 100644 index 00000000..2540b1e9 --- /dev/null +++ b/src/services/images/resources/imageRedactionPolicyWebApp.yml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=../../../schema/common/redactionPolicy.json +name: image-redaction-policy +items: + - jsonPath: $.title + valueConverterName: redact + - jsonPath: $.article.[*].title + valueConverterName: redact + - jsonPath: $.article.[*].excerpt + valueConverterName: redact + - jsonPath: $.article.[*].content + valueConverterName: redact + - jsonPath: $.regions.[*].iiifFragment + valueConverterName: contextNotAllowedImage + - jsonPath: $.pages.[*].iiifThumbnail + valueConverterName: contextNotAllowedImage + - jsonPath: $.pages.[*].iiif + valueConverterName: contextNotAllowedImage diff --git a/src/services/search-exporter/search-exporter.class.js b/src/services/search-exporter/search-exporter.class.js index 024f7f20..effda6f4 100644 --- a/src/services/search-exporter/search-exporter.class.js +++ b/src/services/search-exporter/search-exporter.class.js @@ -30,7 +30,7 @@ class Service { return client .run({ - task: task, + task, args: [ // user id params.user.id, diff --git a/src/services/search/search.hooks.js b/src/services/search/search.hooks.js index 513a5562..503dbd03 100644 --- a/src/services/search/search.hooks.js +++ b/src/services/search/search.hooks.js @@ -4,7 +4,7 @@ import { redactResponseDataItem, inPublicApi, publicApiTranscriptRedactionCondition, - webAppTranscriptRedactionCondition, + webAppExploreRedactionCondition, } from '../../hooks/redaction' import { transformResponseDataItem, transformResponse, renameQueryParameters } from '../../hooks/transformation' import { transformBaseFind } from '../../transformers/base' @@ -120,7 +120,7 @@ module.exports = { protect('content'), transformResponseDataItem(transformContentItem, inPublicApi), redactResponseDataItem(contentItemRedactionPolicy, publicApiTranscriptRedactionCondition), - redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppTranscriptRedactionCondition), + redactResponseDataItem(contentItemRedactionPolicyWebApp, webAppExploreRedactionCondition), ], get: [], create: [], diff --git a/src/services/text-reuse-clusters/text-reuse-clusters.class.ts b/src/services/text-reuse-clusters/text-reuse-clusters.class.ts index 48fba88a..63dc7734 100644 --- a/src/services/text-reuse-clusters/text-reuse-clusters.class.ts +++ b/src/services/text-reuse-clusters/text-reuse-clusters.class.ts @@ -11,6 +11,7 @@ import { NewspapersService } from '../newspapers/newspapers.class' import { SimpleSolrClient } from '../../internalServices/simpleSolr' import { getToSelect } from '../../util/solr/adapters' import { MediaSources } from '../media-sources/media-sources.class' +import { OpenPermissions } from '../../util/bigint' const { mapValues, groupBy, values, uniq, clone, get } = require('lodash') const { NotFound } = require('@feathersjs/errors') @@ -50,8 +51,8 @@ function buildResponseClusters( ({ id, text: textSample, permissionBitmapExplore, permissionsBitmapGetTranscript }) => ({ cluster: clustersById[id], textSample, - bitmapExplore: BigInt(permissionBitmapExplore ?? Number.MAX_SAFE_INTEGER), - bitmapGetTranscript: BigInt(permissionsBitmapGetTranscript ?? Number.MAX_SAFE_INTEGER), + bitmapExplore: BigInt(permissionBitmapExplore ?? OpenPermissions), + bitmapGetTranscript: BigInt(permissionsBitmapGetTranscript ?? OpenPermissions), }) ) return results diff --git a/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js b/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js index d7a5cfe5..6db40a5f 100644 --- a/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js +++ b/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js @@ -6,7 +6,7 @@ import { parseFilters } from '../../util/queryParameters' import { redactResponse, redactResponseDataItem, - webAppTranscriptRedactionCondition, + webAppExploreRedactionCondition, publicApiTranscriptRedactionCondition, inPublicApi, } from '../../hooks/redaction' @@ -55,14 +55,14 @@ module.exports = { all: [], get: [ transformResponse(transformTextReuseCluster, inPublicApi), - redactResponse(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponse(trPassageRedactionPolicy, webAppExploreRedactionCondition), redactResponse(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], find: [ renameTopLevelField(['clusters', 'data'], inPublicApi), transformResponse(transformBaseFind, inPublicApi), transformResponseDataItem(transformTextReuseCluster, inPublicApi), - redactResponseDataItem(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponseDataItem(trPassageRedactionPolicy, webAppExploreRedactionCondition), redactResponseDataItem(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], // find: [validateWithSchema('services/text-reuse-clusters/schema/find/response.json', 'result')], diff --git a/src/services/text-reuse-passages/text-reuse-passages.hooks.js b/src/services/text-reuse-passages/text-reuse-passages.hooks.js index 9677a486..4b270ccd 100644 --- a/src/services/text-reuse-passages/text-reuse-passages.hooks.js +++ b/src/services/text-reuse-passages/text-reuse-passages.hooks.js @@ -6,7 +6,7 @@ import { parseFilters } from '../../util/queryParameters' import { redactResponse, redactResponseDataItem, - webAppTranscriptRedactionCondition, + webAppExploreRedactionCondition, publicApiTranscriptRedactionCondition, inPublicApi, } from '../../hooks/redaction' @@ -15,7 +15,7 @@ import { transformResponseDataItem, transformResponse } from '../../hooks/transf import { transformTextReusePassage } from '../../transformers/textReuse' import { transformBaseFind } from '../../transformers/base' -const trPassageRedactionPolicy = loadYamlFile(`${__dirname}/resources/trPassageRedactionPolicy.yml`) +export const trPassageRedactionPolicy = loadYamlFile(`${__dirname}/resources/trPassageRedactionPolicy.yml`) // import { validateParameters } from '../../util/openapi' // import { docs } from './text-reuse-passages.schema' @@ -43,13 +43,13 @@ module.exports = { after: { get: [ transformResponse(transformTextReusePassage, inPublicApi), - redactResponse(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponse(trPassageRedactionPolicy, webAppExploreRedactionCondition), redactResponse(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], find: [ transformResponse(transformBaseFind, inPublicApi), transformResponseDataItem(transformTextReusePassage, inPublicApi), - redactResponseDataItem(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponseDataItem(trPassageRedactionPolicy, webAppExploreRedactionCondition), redactResponseDataItem(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], }, diff --git a/src/useCases/getContentItemsPermissionsDetails.ts b/src/useCases/getContentItemsPermissionsDetails.ts index ace9db57..29863471 100644 --- a/src/useCases/getContentItemsPermissionsDetails.ts +++ b/src/useCases/getContentItemsPermissionsDetails.ts @@ -89,9 +89,10 @@ interface SampleSolrDocument { } export const getContentItemsPermissionsDetails = async ( - solrClient: SimpleSolrClient + solrClient: SimpleSolrClient, + index: keyof typeof SolrNamespaces ): Promise => { - const response = await solrClient.select(SolrNamespaces.Search, { + const response = await solrClient.select(SolrNamespaces[index], { body: getBitmapsFacets, }) @@ -108,7 +109,7 @@ export const getContentItemsPermissionsDetails = async ( async ({ scope, permissions }) => ({ scope, - permissions: await Promise.all(permissions.map(withSample(scope, solrClient))), + permissions: await Promise.all(permissions.map(withSample(scope, solrClient, index))), }) satisfies ScopedPermissions ) ) @@ -131,9 +132,9 @@ const toPermissionDetails = (buckets: Bucket[]): Omit + (scope: PermissionsScope, solrClient: SimpleSolrClient, index: keyof typeof SolrNamespaces) => async (details: Omit): Promise => { - const response = await solrClient.select(SolrNamespaces.Search, { + const response = await solrClient.select(SolrNamespaces[index], { body: getSample(ScopeToField[scope], details.bitmap), }) diff --git a/src/util/bigint.ts b/src/util/bigint.ts index 87e63e3f..75ceb82d 100644 --- a/src/util/bigint.ts +++ b/src/util/bigint.ts @@ -32,3 +32,9 @@ const Zero = BigInt(0) export const bitmapsAlign = (bitmap: bigint, mask: bigint): boolean => { return (bitmap & mask) != Zero } + +/** + * A bitmap that allows all permissions. + * Useful to assign to resources that do not declare any permissions. + */ +export const OpenPermissions = BigInt('0b' + [...Array(64)].map(() => '1').join('')) diff --git a/test/integration/use_cases/permissions.test.ts b/test/integration/use_cases/permissions.test.ts index 013a04d9..3eef00e9 100644 --- a/test/integration/use_cases/permissions.test.ts +++ b/test/integration/use_cases/permissions.test.ts @@ -15,12 +15,17 @@ import { contentItemRedactionPolicy, contentItemRedactionPolicyWebApp, } from '../../../src/services/articles/articles.hooks' +import { trPassageRedactionPolicy } from '../../../src/services/text-reuse-passages/text-reuse-passages.hooks' +import { imageRedactionPolicyWebApp } from '../../../src/services/images/images.hooks' import { DefaultConverters, RedactionPolicy } from '../../../src/util/redaction' import { JSONPath } from 'jsonpath-plus' +import { SolrNamespaces } from '../../../src/solr' interface RedactionTestContext { scope: PermissionsScope + contentItemNamespace: keyof typeof SolrNamespaces + contentItemId: string contentItemPermissions: bigint contentSample: PermissionDetails['sample'] @@ -41,30 +46,41 @@ const buildSlimUser = (context: RedactionTestContext): SlimUser => ({ const buildTestMatrix = ( contentItemDetails: ContentItemPermissionsDetails, + imageDetails: ContentItemPermissionsDetails, userAccounts: UserAccount[] ): RedactionTestContext[] => { - const items = contentItemDetails.permissions.map(scopeItem => { - return scopeItem.permissions.map(permissionItem => { - return userAccounts.map(userAccount => { - const contentBitmap = permissionItem.bitmap.valueOf() // as bigint - const userBitmap = userAccount.bitmap.valueOf() // as bigint + const groups: { + namespace: keyof typeof SolrNamespaces + permissions: ContentItemPermissionsDetails['permissions'] + }[] = [ + { namespace: 'Search', permissions: contentItemDetails.permissions }, + { namespace: 'Images', permissions: imageDetails.permissions }, + ] + const items = groups.map(({ namespace, permissions }) => { + return permissions.map(scopeItem => { + return scopeItem.permissions.map(permissionItem => { + return userAccounts.map(userAccount => { + const contentBitmap = permissionItem.bitmap.valueOf() // as bigint + const userBitmap = userAccount.bitmap.valueOf() // as bigint - return { - scope: scopeItem.scope, + return { + scope: scopeItem.scope, - contentItemId: permissionItem.sample.id, - contentItemPermissions: contentBitmap, - contentSample: permissionItem.sample, + contentItemNamespace: namespace, + contentItemId: permissionItem.sample.id, + contentItemPermissions: contentBitmap, + contentSample: permissionItem.sample, - userAccountId: userAccount.sample_user_id, - userAccountPermissions: userBitmap, + userAccountId: userAccount.sample_user_id, + userAccountPermissions: userBitmap, - accessAllowed: bitmapsAlign(contentBitmap, userBitmap), - } satisfies RedactionTestContext + accessAllowed: bitmapsAlign(contentBitmap, userBitmap), + } satisfies RedactionTestContext + }) }) }) }) - return items.flat(3) as RedactionTestContext[] + return items.flat(4) as RedactionTestContext[] } const getPaths = (policy: RedactionPolicy, object: any, redactionExpectation: 'redacted' | 'notRedacted') => { @@ -93,61 +109,218 @@ const getPaths = (policy: RedactionPolicy, object: any, redactionExpectation: 'r return paths } +const runner = async ( + testCases: RedactionTestContext[], + getter: (ctx: RedactionTestContext) => Promise, + redactionPolicy: RedactionPolicy, + assertions?: (result: any, ctx: RedactionTestContext) => void, + inspect?: 'none' | 'redacted' | 'notRedacted' +) => { + let resultReceivedCounter = 0 + + const awaitables = testCases.map(async (testCase, idx) => { + const result = await getter(testCase) + resultReceivedCounter++ + + process.stdout.write(`Got result ${resultReceivedCounter} of ${testCases.length}...\r`) + + const paths = getPaths(redactionPolicy, result, testCase.accessAllowed ? 'notRedacted' : 'redacted') + + // inspect redaction + if (inspect != 'none') { + if ( + (!testCase.accessAllowed && inspect === 'redacted') || + (testCase.accessAllowed && inspect === 'notRedacted') + ) { + console.log( + safeStringifyJson( + { + testCase, + result, + }, + 2 + ) + ) + } + } + + const failMessage = ` + Test case item ${testCase.contentItemId} is supposed to be ${testCase.accessAllowed ? 'not redacted' : 'redacted'} for user ${testCase.userAccountId}, + but these paths are ${testCase.accessAllowed ? 'redacted' : 'not redacted'}: ${paths.join(',')}. + Content bitmap:\t${bigIntToBitString(testCase.contentItemPermissions)} + User bitmap:\t${bigIntToBitString(testCase.userAccountPermissions)} + Content: ${safeStringifyJson(result, 2)} + ` + + assert.strictEqual(paths.length, 0, failMessage) + assertions?.(result, testCase) + }) + + await Promise.all(awaitables) +} + describe('Bitmap permissions', function () { - this.timeout(30000) + this.timeout(300000) - let isPublicApi = false let testMatrix: RedactionTestContext[] = [] before(async () => { - isPublicApi = app.get('isPublicApi') ?? false + if (app.get('cache')?.enabled) assert.fail('Cache is enabled. Disable it to run the test without cache.') const solrCilent = app.service('simpleSolrClient') const sequelize = app.get('sequelizeClient')! - const [userAccounts, contentPermissionsDetails] = await Promise.all([ + const [userAccounts, contentPermissionsDetails, imagesPermissionsDetails] = await Promise.all([ getUserAccountsWithAvailablePermissions(sequelize), - getContentItemsPermissionsDetails(solrCilent), + getContentItemsPermissionsDetails(solrCilent, 'Search'), + getContentItemsPermissionsDetails(solrCilent, 'Images'), ]) - testMatrix = buildTestMatrix(contentPermissionsDetails, userAccounts) - const totalContentItemPermissions = contentPermissionsDetails.permissions.reduce( - (acc, item) => acc + item.permissions.length, - 0 - ) + testMatrix = buildTestMatrix(contentPermissionsDetails, imagesPermissionsDetails, userAccounts) + const totalContentItemPermissions = [ + ...contentPermissionsDetails.permissions, + ...imagesPermissionsDetails.permissions, + ].reduce((acc, item) => acc + item.permissions.length, 0) console.log( `${totalContentItemPermissions} various content item permissions across ${contentPermissionsDetails.permissions.length} scopes found` ) console.log(`${userAccounts.length} various user accounts permissions found`) - console.log(`${testMatrix.length} test cases to run`) + console.log(`${testMatrix.length} test cases in total`) }) - it('Web App: get article', async () => { - if (isPublicApi) return + describe('Web app', () => { + if (app.get('isPublicApi')) { + console.log('Skipping web app tests because this is a public API') + return + } - const redactionPolicy = contentItemRedactionPolicyWebApp - const getTranscriptCases = testMatrix.filter(test => test.scope === 'explore') - const service = app.service('articles') + it('Get article', async () => { + const getTranscriptCases = testMatrix.filter( + test => test.scope === 'explore' && test.contentItemNamespace === 'Search' + ) + const service = app.service('articles') - const indices = Array.from({ length: getTranscriptCases.length }, (_, i) => i) + await runner( + getTranscriptCases, + async testCase => { + const params = { user: buildSlimUser(testCase), authenticated: true } + return await service.get(testCase.contentItemId, params) + }, + contentItemRedactionPolicyWebApp, + undefined, + 'none' + ) + }) - for await (const idx of indices) { - const testCase = getTranscriptCases[idx] - console.log(`Testing case ${idx} of ${getTranscriptCases.length}...`) - const params = { user: buildSlimUser(testCase) } - const result = await service.get(testCase.contentItemId, params) + it('Search', async () => { + const getTranscriptCases = testMatrix.filter( + test => test.scope === 'explore' && test.contentItemNamespace === 'Search' + ) + const service = app.service('search') - const paths = getPaths(redactionPolicy, result, testCase.accessAllowed ? 'notRedacted' : 'redacted') + await runner( + getTranscriptCases, + async testCase => { + const params = { + user: buildSlimUser(testCase), + authenticated: true, + query: { + sq: '*:*', + filters: [{ q: [testCase.contentItemId], type: 'uid' }], + }, + } + return await service.find(params) + }, + contentItemRedactionPolicyWebApp, + (result, testCase) => { + assert.strictEqual( + result.data.length, + 1, + `search for ${testCase.contentItemId} yielded ${result.data.length}. Should be 1` + ) + }, + 'none' + ) + }) - const failMessage = ` - Content item ${testCase.contentItemId} is supposed to be ${testCase.accessAllowed ? 'not redacted' : 'redacted'} for user ${testCase.userAccountId}, - but these paths are ${testCase.accessAllowed ? 'redacted' : 'not redacted'}: ${paths.join(',')}. - Content bitmap:\t${bigIntToBitString(testCase.contentItemPermissions)} - User bitmap:\t${bigIntToBitString(testCase.userAccountPermissions)} - Content: ${safeStringifyJson(result, 2)} - ` + it('Get image', async () => { + const getTranscriptCases = testMatrix.filter( + test => test.scope === 'explore' && test.contentItemNamespace === 'Images' + ) + const service = app.service('images') + + await runner( + getTranscriptCases, + async testCase => { + const params = { user: buildSlimUser(testCase), authenticated: true } + return await service.get(testCase.contentItemId, params) + }, + imageRedactionPolicyWebApp, + undefined, + 'none' + ) + }) + + // TODO when the data is ready + // it('Web App: text reuse passages', async () => { + // // get samples code needs to be updated + // // right now it gets samples from the main search index only. + // }) + }) - assert.strictEqual(paths.length, 0, failMessage) + describe('Public API', () => { + if (!app.get('isPublicApi')) { + console.log('Skipping public API tests because this is not a public API') + return } + + it('Get content item', async () => { + const getTranscriptCases = testMatrix.filter( + test => test.scope === 'get_transcript' && test.contentItemNamespace === 'Search' + ) + const service = app.service('content-items') + + await runner( + getTranscriptCases, + async testCase => { + const params = { user: buildSlimUser(testCase), authenticated: true } + return await service.get(testCase.contentItemId, params) + }, + contentItemRedactionPolicy, + undefined, + 'none' + ) + }) + + it('Search', async () => { + const getTranscriptCases = testMatrix.filter( + test => test.scope === 'get_transcript' && test.contentItemNamespace === 'Search' + ) + const service = app.service('search') + + await runner( + getTranscriptCases, + async testCase => { + const params = { + user: buildSlimUser(testCase), + authenticated: true, + query: { + sq: '*:*', + filters: [{ q: [testCase.contentItemId], type: 'uid' }], + }, + } + return await service.find(params) + }, + contentItemRedactionPolicy, + (result, testCase) => { + assert.strictEqual( + result.data.length, + 1, + `search for ${testCase.contentItemId} yielded ${result.data.length}. Should be 1` + ) + }, + 'none' + ) + }) }) })