Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🎨 Add deces-update index #442

Merged
merged 2 commits into from
Feb 9, 2025
+300 −81
Merged

🎨 Add deces-update index #442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36841a7
🎨 Rebase index update
cristianpb Jan 17, 2025
a87465e
✍ Take into account comments from review
cristianpb Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/src/buildRequest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ const buildFrom = (current: number, resultsPerPage: number) => {
return (current - 1) * resultsPerPage;
}

const referenceSort: any = {
export const referenceSort: any = {
score: "_score",
firstName: "PRENOM.raw",
lastName: "NOM.raw",
Expand Down
75 changes: 22 additions & 53 deletions backend/src/controllers/search.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ import express from 'express';
import { writeFile, access, mkdir, createReadStream } from 'fs';
import { promisify } from 'util';
import { resultsHeader, jsonPath, prettyString } from '../processStream';
import { runRequest, runBulkRequest } from '../runRequest';
import { runRequest } from '../runRequest';
import { buildRequest } from '../buildRequest';
import { RequestInput, RequestBody } from '../models/requestInput';
import { StrAndNumber, Modification, UpdateRequest, UpdateUserRequest, Review, ReviewsStringified, statusAuthMap, PersonCompare } from '../models/entities';
import { buildResult, buildResultSingle, Result, ErrorResponse } from '../models/result';
import { buildResult, Result, ErrorResponse } from '../models/result';
import { format } from '@fast-csv/format';
import { ScoreResult, personFromRequest } from '../score';
import { updatedFields } from '../updatedIds';
import { getAllUpdates, getAuthorUpdates, updatedFields, resultsFromUpdates, cleanRawUpdates, addModification } from '../updatedIds';
import { sendUpdateConfirmation } from '../mail';
// import getDataGouvCatalog from '../getDataGouvCatalog';

Expand Down Expand Up @@ -262,16 +262,21 @@ export class SearchController extends Controller {
@Body() updateRequest: UpdateRequest,
@Request() request: express.Request
): Promise<any> {
const date = new Date(Date.now()).toISOString()
try {
await this.storeProof(request);
} catch (err) {
this.setStatus(400);
return { msg: err.message };
}
// get user & rights from Security
await this.handleFile(request);
const author = (request as any).user && (request as any).user.user
const isAdmin = (request as any).user && (request as any).user.scopes && (request as any).user.scopes.includes('admin');
const requestInput = new RequestInput({id});
const requestBuild = buildRequest(requestInput);
const result = await runRequest(requestBuild, requestInput.scroll);
const builtResult = buildResult(result.data, requestInput)
if (builtResult.response.persons.length > 0) {
const date = new Date(Date.now()).toISOString()
const bytes = forge.random.getBytesSync(24);
const randomId = forge.util.bytesToHex(bytes);
if (!isAdmin) {
Expand All @@ -293,7 +298,7 @@ export class SearchController extends Controller {
return { msg: 'Proof must be provided' }
}
delete updateRequest.proof;
const correctionData: Modification = {
const modification: Modification = {
id: randomId,
date,
proof,
Expand All @@ -302,17 +307,15 @@ export class SearchController extends Controller {
fields: updateRequest
};
if (message) {
correctionData.message = message;
modification.message = message;
}
try {
await accessAsync(`${process.env.PROOFS}/${id}`);
} catch(err) {
await mkdirAsync(`${process.env.PROOFS}/${id}`, { recursive: true });
const success = await addModification(id, modification, date);
if (success) {
return { msg: "Update stored" };
} else {
this.setStatus(500);
return { msg: "Update failed" };
}
await writeFileAsync(`${process.env.PROOFS}/${id}/${date}_${id}.json`, JSON.stringify(correctionData));
if (!updatedFields[id]) { updatedFields[id] = [] }
updatedFields[id].push(correctionData);
return { msg: "Update stored" };
} else {
if (!updatedFields[id]) {
this.setStatus(406);
Expand Down Expand Up @@ -383,47 +386,13 @@ export class SearchController extends Controller {
public async updateList(@Request() request: express.Request): Promise<any> {
const author = (request as any).user && (request as any).user.user
const isAdmin = (request as any).user && (request as any).user.scopes && (request as any).user.scopes.includes('admin');
let updates:any = {};
if (isAdmin) {
updates = {...updatedFields};
} else {
Object.keys(updatedFields).forEach((id:any) => {
let filter = false;
const modifications = updatedFields[id].map((m:any) => {
const modif:any = {...m}
if (modif.author !== author) {
modif.author = modif.author.substring(0,2)
+ '...' + modif.author.replace(/@.*/,'').substring(modif.author.replace(/@.*/,'').length-2)
+ '@' + modif.author.replace(/.*@/,'');
modif.message = undefined;
modif.review = undefined;
} else {
filter=true
}
return modif;
});
if (filter) {
updates[id] = modifications;
}
})
}
const updates:any = isAdmin ? getAllUpdates() : getAuthorUpdates(author);
if (Object.keys(updates).length === 0) return []
const bulkRequest = Object.keys(updates).map((id: any) =>
[JSON.stringify({index: "deces"}), JSON.stringify(buildRequest(new RequestInput({id})))]
);
const msearchRequest = bulkRequest.map((x: any) => x.join('\n\r')).join('\n\r') + '\n';
const result = await runBulkRequest(msearchRequest);
return result.data.responses.map((r:any) => buildResultSingle(r.hits.hits[0]))
.filter((r:any) => Object.keys(r).length > 0)
.map((r:any) => {
delete r.score;
delete r.scores;
r.modifications = updates[r.id];
return r;
});
const rawUpdates = await resultsFromUpdates(updates);
return await cleanRawUpdates(rawUpdates, updates);
}

private async handleFile(request: express.Request): Promise<any> {
private async storeProof(request: express.Request): Promise<any> {
const storage = multer.diskStorage({
destination: void(async (req: any, file: any, cb: any) => {
if (file.mimetype !== 'application/pdf') {
Expand Down
9 changes: 7 additions & 2 deletions backend/src/controllers/search.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { SearchController } from './search.controller';
import { PersonCompare } from '../models/entities';
import express from 'express';
import { describe, expect, it } from 'vitest'
import { initUpdateIndex } from '../updatedIds';
import { describe, expect, it, beforeAll } from 'vitest'

beforeAll(async () => {
await initUpdateIndex();
})

describe('search.controller.ts - GET request', () => {
const controller = new SearchController()
Expand Down Expand Up @@ -90,7 +95,7 @@ describe('search.controller.ts - POST id', () => {
it('update id', async () => {
const body = {
'author_id': '[email protected]',
lastName: 'Aiph7u',
lastName: 'Aeboox9e',
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Test data inconsistency detected

The lastName value should remain as 'Aiph7u' to maintain consistency with server.spec.ts, as both tests are validating the same functionality with the same author_id ('[email protected]').

  • backend/src/controllers/search.spec.ts: Revert lastName from 'Aeboox9e' back to 'Aiph7u'
🔗 Analysis chain

Verify test data change rationale.

The modification of the test data from 'Aiph7u' to 'Aeboox9e' seems unrelated to the PR objective.

Let's check if this change is related to any other modifications:

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Search for occurrences of both old and new test values
rg -g '*.ts' -g '*.js' "Aiph7u|Aeboox9e" ./backend/

Length of output: 256

Script:

#!/bin/bash
# Get context around the test data usage in both files
echo "=== search.spec.ts ==="
rg -B 5 -A 5 "Aeboox9e" ./backend/src/controllers/search.spec.ts

echo -e "\n=== server.spec.ts ==="
rg -B 5 -A 5 "Aiph7u" ./backend/src/server.spec.ts

Length of output: 1295

proof: 'https://somwhere.in.the.internet',
}
const req = {
Expand Down
15 changes: 11 additions & 4 deletions backend/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
import { app } from './server';
import { initUpdateIndex, updateFieldsToIndex, getAllUpdates } from './updatedIds';

const port = 8080;

app.listen( port, () => {
// eslint-disable-next-line no-console
console.log( `server started at http://localhost:${ port }` );
} );
(async () => {
await initUpdateIndex();
const updates = getAllUpdates();
await updateFieldsToIndex(updates);

app.listen(port, () => {
// eslint-disable-next-line no-console
console.log(`server started at http://localhost:${port}`);
});
})();
1 change: 1 addition & 0 deletions backend/src/models/entities.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ export interface Modification {
}

export interface Person {
index?: string;
score?: number;
source?: string;
sourceLine?: number;
Expand Down
25 changes: 22 additions & 3 deletions backend/src/models/result.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ export interface ResultRawES {
}

export interface ResultRawHit {
_index: 'deces'|'deces-updates';
_score: number;
_id: string;
_source: {
Expand Down Expand Up @@ -215,6 +216,21 @@ export const buildResult = (result: ResultRawES, requestInput: RequestInput): Re
}
})
let filteredResults = result.hits.hits.map(buildResultSingle)

filteredResults
.forEach((value, index, self) => {
const firstIndex = self.findIndex((item) => item.id === value.id);
if (index !== firstIndex) {
// Prioritize 'deces-updates' index over 'deces'
const shouldRemoveFirst = self[firstIndex].index === 'deces' && value.index === 'deces-updates';
if (shouldRemoveFirst) {
self.splice(firstIndex, 1);
} else if (value.index === 'deces') {
self.splice(index, 1);
}
}
})

scoreResults(filteredRequest, filteredResults, {dateFormatA: filteredRequest.dateFormat})
if (requestInput.sort && Object.values(requestInput.sort.value).map(x => Object.keys(x))[0].includes('score')) {
if (Object.values(requestInput.sort.value).find(x => x.score).score === 'asc') {
Expand Down Expand Up @@ -244,6 +260,7 @@ export const buildResult = (result: ResultRawES, requestInput: RequestInput): Re
export const buildResultSingle = (item: ResultRawHit): Person|undefined => {
if (item === undefined) return {}
const result: Person = {
index: item._index,
score: item._score,
// source: dataCatalog[item._source.SOURCE],
source: item._source.SOURCE,
Expand Down Expand Up @@ -299,9 +316,11 @@ export const buildResultSingle = (item: ResultRawHit): Person|undefined => {
const update: any = {...u};
// WIP quick n dirty anonymization
const { author } = u;
update.author = author && author.substring(0,2)
+ '...' + author.replace(/@.*/,'').substring(author.replace(/@.*/,'').length-2)
+ '@' + author.replace(/.*@/,'');
update.author = author ?
(author.length > 8 ?
author.substring(0,2) + '...' + author.replace(/@.*/,'').substring(author.replace(/@.*/,'').length-2)
: '...')
+ '@' + author.replace(/.*@/,'') : "";
return update;
});
}
Expand Down
19 changes: 9 additions & 10 deletions backend/src/processStream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,15 @@ import iconv from 'iconv-lite';

import timer from './timer';

const log = (json:any) => {
loggerStream.write(JSON.stringify({
"backend": {
"server-date": new Date(Date.now()).toISOString(),
...json
}
}));
}

const timerRunBulkRequest = timer(runBulkRequest, 'runBulkRequest', 1);

export const validFields: string[] = ['q', 'firstName', 'lastName', 'legalName', 'sex', 'birthDate', 'birthCity', 'birthLocationCode', 'birthPostalCode', 'birthDepartment', 'birthCountry',
Expand All @@ -27,15 +36,6 @@ export const validFields: string[] = ['q', 'firstName', 'lastName', 'legalName',
const validFieldsForColumnsCount = ['firstName', 'lastName', 'legalName', 'sex', 'birthDate', 'birthCity', 'birthLocationCode', 'birthPostalCode', 'birthDepartment', 'birthCountry',
'birthGeoPoint', 'deathDate', 'deathCity', 'deathLocationCode', 'deathPostalCode', 'deathDepartment', 'deathCountry', 'deathGeoPoint', 'deathAge', 'lastSeenAliveDate']

const log = (json:any) => {
loggerStream.write(JSON.stringify({
"backend": {
"server-date": new Date(Date.now()).toISOString(),
...json
}
}));
}

const formatJob = (job:any) => {
const duration: number = job && job.processedOn && job.finishedOn && (job.finishedOn - job.processedOn) / 1000;
return job && {
Expand Down Expand Up @@ -134,7 +134,6 @@ const JsonParseStream = () => {
objectMode: true,
transform(row: any, encoding: string, callback: (e?: any) => void) {
try {
// loggerStream.write(`jsonParse ${row}\n`);
this.push(JSON.parse(row));
callback();
} catch(e) {
Expand Down
4 changes: 2 additions & 2 deletions backend/src/runRequest.ts
View file
Open in desktop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, vu le endpoint updates !

Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: st
if (body.scroll_id) {
endpoint = '_search/scroll'
} else if (scroll) {
endpoint = `deces/_search?scroll=${scroll}`
endpoint = `deces,deces-updates/_search?scroll=${scroll}`
} else {
endpoint = 'deces/_search'
endpoint = 'deces,deces-updates/_search'
}
const response = await axios(`http://elasticsearch:9200/${endpoint}`, {
method: 'post',
Expand Down
46 changes: 41 additions & 5 deletions backend/src/server.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,18 @@ import { Person } from './models/entities';
import { promisify } from 'util';
import { parseString } from '@fast-csv/parse';
import { writeToBuffer } from '@fast-csv/format';
import { initUpdateIndex } from './updatedIds';
import fs from "fs";
import { describe, expect, it, test } from 'vitest'
import { describe, expect, it, test, beforeAll } from 'vitest'
import supertest from 'supertest';
const server = supertest(app)
const finishedAsync:any = promisify(finished);


beforeAll(async () => {
await initUpdateIndex();
})

const csv2Buffer = async (filePath: string, nrows: number) => {
let data = '';
let index: number;
Expand Down Expand Up @@ -41,8 +47,8 @@ describe('server.ts - Express application', () => {
expect(res.body.msg).toEqual("OK");
});

describe('/id/{id}', () => {
it('search', async () => {
describe.sequential('/id/{id}', () => {
test.sequential('search', async () => {
let res = await server
.get(apiPath('search'))
.query({q: 'Georges Duboeuf'})
Expand All @@ -55,19 +61,49 @@ describe('server.ts - Express application', () => {
expect(res.body.response.persons[0].links.wikidata).to.include('Q3102639');
});

it('update', async () => {
test.sequential('update add modification to updates', async () => {
const token = await server
.post(apiPath(`auth`))
.send({user:'[email protected]', password: 'magicPass'})
const buf = Buffer.from('weird pdf', 'base64')
const res = await server
let res = await server
.post(apiPath(`id/VhfumwT3QnUq`))
.set('Authorization', `Bearer ${token.body.access_token as string}`)
.field('author_id', '[email protected]')
.field('lastName', 'Aiph7u')
.attach('pdf', buf, 'file.pdf')
expect(res.status).toBe(200);
expect(res.body.msg).to.equal('Update stored');

await new Promise(f => setTimeout(f, 1000));
res = await server
.get(apiPath('search'))
.query({ lastName: 'Aiph7u' })
expect(res.status).toBe(200);
expect(res.body.response.persons.length).to.above(0);
expect(res.body.response.persons[0].name.first).to.include('Georges');
});

test.sequential('update get all updates (admin)', async () => {
const token = await server
.post(apiPath(`auth`))
.send({user:process.env.BACKEND_TOKEN_USER, password: process.env.BACKEND_TOKEN_PASSWORD})
const res = await server
.get(apiPath('updated'))
.set('Authorization', `Bearer ${token.body.access_token as string}`)
expect(res.status).toBe(200);
expect(res.body.length).to.above(0);
});

test.sequential('update get author updates', async () => {
const token = await server
.post(apiPath(`auth`))
.send({user:'[email protected]', password: 'magicPass'})
const res = await server
.get(apiPath('updated'))
.set('Authorization', `Bearer ${token.body.access_token as string}`)
expect(res.status).toBe(200);
expect(res.body.length).to.above(0);
});
})

Expand Down
3 changes: 2 additions & 1 deletion backend/src/tsoa.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ import { generateRoutes, generateSpec, ExtendedRoutesConfig, ExtendedSpecConfig
flow: "password",
tokenUrl: "/deces/api/v1/auth",
scopes: {
admin: "administration access"
admin: "administration access",
user: "user access"
Comment on lines +19 to +20
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Remove unused 'admin' scope definition

The 'admin' scope is defined in tsoa.ts but not used anywhere in the codebase. All endpoints currently use only the 'user' scope. Consider:

  • Removing the unused 'admin' scope definition, or
  • Adding a TODO comment if admin functionality is planned for future implementation
🔗 Analysis chain

LGTM! Verify scope requirements.

The addition of the 'user' scope alongside 'admin' enhances access control granularity. This is particularly relevant for managing access to the new updates index.

Let's verify the scope usage across the codebase:

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Search for JWT scope usage in controllers and routes
rg -g '*.ts' -A 3 '@Security\(.*\)' ./backend/src/

Length of output: 2969

Script:

#!/bin/bash
# Search for any references to admin scope or other security decorators
rg -g '*.ts' -i '(admin.*access|@security|authentication|authorization)' ./backend/src/

Length of output: 6828

}
},
}
Expand Down
Loading
Loading