feat: récupérer la liste des conventions collectives depuis un lien e…

…xterne

scripts/check.js

@@ -6,32 +6,42 @@ import unistUtilFind from "unist-util-find";
 import { getAgreement, getAgreements } from "../src";
 
 log.enableColor();
-const INDEXED_AGREEMENTS = getAgreements();
-
-/**
- * @param {KaliData.IndexedAgreement} convention
- *
- * @returns {boolean}
- */
-const containsUrl = convention => typeof convention.url === "string";
-
-/**
- * @param {KaliData.AgreementArticle | KaliData.AgreementSection} node
- *
- * @returns {boolean}
- */
-const hasSectionWithNoChild = node =>
-    node.type === "section" && Array.isArray(node.children) && node.children.length === 0;
-
-const agrementsWithUrl = INDEXED_AGREEMENTS.filter(containsUrl);
-agrementsWithUrl.forEach(({ id }) => {
-    const agreement = getAgreement(id);
-    const agreementAdditionalSections = agreement.children.slice(1);
-    const emptyNodes = unistUtilFind(agreementAdditionalSections, hasSectionWithNoChild);
-
-    if (emptyNodes !== undefined) {
-        log.error("check()", `${id} has ${emptyNodes.length} empty nodes.`);
-
-        process.exit(-1);
+
+async function main() {
+    const agreements = await getAgreements();
+
+    /**
+     * @param {KaliData.IndexedAgreement} convention
+     *
+     * @returns {boolean}
+     */
+    const containsUrl = convention => typeof convention.url === "string";
+
+    /**
+     * @param {KaliData.AgreementArticle | KaliData.AgreementSection} node
+     *
+     * @returns {boolean}
+     */
+    const hasSectionWithNoChild = node =>
+        node.type === "section" && Array.isArray(node.children) && node.children.length === 0;
+
+    const agrementsWithUrl = agreements.filter(containsUrl);
+    for (const { id } of agrementsWithUrl) {
+        const agreement = await getAgreement(id);
+        const agreementAdditionalSections = agreement.children.slice(1);
+        const emptyNodes = unistUtilFind(agreementAdditionalSections, hasSectionWithNoChild);
+
+        if (emptyNodes !== undefined) {
+            log.error("check()", `${id} has ${emptyNodes.length} empty nodes.`);
+
+            process.exit(-1);
+        }
     }
+}
+
+main().catch(error => {
+    log.error("check()", `Failed: ${error}`);
+    console.error(error);
+
+    process.exit(-1);
 });

scripts/clean.js

@@ -10,8 +10,6 @@ log.enableColor();
 
 const writeFile = promisify(fs.writeFile);
 
-const INDEXED_AGREEMENTS = getAgreements();
-
 const t0 = Date.now();
 
 function cleanAgreements(agreementId) {
@@ -28,7 +26,7 @@ async function saveFile(container) {
         path.join(__dirname, "..", "data", `${container.data.id}.json`),
         JSON.stringify(container, 0, 2),
     );
-    log.verbose("fetch()", `Updating ${container.data.id}.json`);
+    log.verbose("clean()", `Updating ${container.data.id}.json`);
 }
 
 function toFix(value, nb = 2) {
@@ -40,22 +38,24 @@ function toFix(value, nb = 2) {
 async function main() {
     const pipelineClean = pPipe(cleanAgreements, saveFile);
 
+    const INDEXED_AGREEMENTS = await getAgreements();
+
     const notSupportedCcnList = INDEXED_AGREEMENTS.filter(
         convention => !!convention.url && !convention.fetchArticles,
     );
 
     const pClean = notSupportedCcnList.map(({ id }) => {
         return pipelineClean(id).catch(error => {
-            log.error("main()", `clean failed for ${id}`);
+            log.error("clean()", `clean failed for ${id}`);
             throw error;
         });
     });
     await Promise.all(pClean);
-    log.info("fetch()", `Clean done in ${toFix((Date.now() - t0) / 1000)} s`);
+    log.info("clean()", `Clean done in ${toFix((Date.now() - t0) / 1000)} s`);
 }
 
 main().catch(error => {
-    log.error("fetch()", `Failed in ${toFix((Date.now() - t0) / 1000)} s (${error})`);
+    log.error("clean()", `Failed in ${toFix((Date.now() - t0) / 1000)} s (${error})`);
 
     process.exit(-1);
 });

scripts/fetch.js

@@ -13,8 +13,6 @@ import astify, { cleanAst, isValidSection } from "./libs/astify";
 log.enableColor();
 const writeFile = promisify(fs.writeFile);
 
-const INDEXED_AGREEMENTS = getAgreements();
-
 const { OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET } = process.env;
 
 if (OAUTH_CLIENT_ID === undefined || OAUTH_CLIENT_SECRET === undefined) {
@@ -92,13 +90,14 @@ function toFix(value, nb = 2) {
 async function main() {
     const pipeline = pPipe(fetchKaliCont, fetchAdditionalText, astify, cleanAst, saveFile);
 
-    const ccnList = INDEXED_AGREEMENTS.filter(
+    const indexedAgreements = await getAgreements();
+    const ccnList = indexedAgreements.filter(
         convention => !!convention.url && convention.fetchArticles,
     );
 
     const pResults = ccnList.map(({ id }) => {
         return pipeline(id).catch(error => {
-            log.error("main()", `pipeline failed for ${id}`);
+            log.error("fetch()", `pipeline failed for ${id}`);
             throw error;
         });
     });

scripts/list.js

@@ -12,22 +12,36 @@ import { getAgreements } from "../src";
 
 log.enableColor();
 
-const clean = str => ("" + (str || "")).replace(/
/g, "").replace(/\n/g, " ").trim();
-const cleanDate = dte => (dte && dte.split("T")[0].split("-").reverse().join("/")) || "-";
-
-const sourceLines = [`# Agreement References`, ``, `ID | IDCC | Title | Published At`, `-|-|-|-`];
-getAgreements().forEach(convention => {
-    sourceLines.push(
-        `${clean(convention.id)} | ${clean(convention.num)} | ${clean(
-            convention.shortTitle,
-        )} | ${cleanDate(convention.date_publi)}`,
-    );
+async function main() {
+    const clean = str => ("" + (str || "")).replace(/
/g, "").replace(/\n/g, " ").trim();
+    const cleanDate = dte => (dte && dte.split("T")[0].split("-").reverse().join("/")) || "-";
+
+    const sourceLines = [
+        `# Agreement References`,
+        ``,
+        `ID | IDCC | Title | Published At`,
+        `-|-|-|-`,
+    ];
+    const agreements = await getAgreements();
+    agreements.forEach(convention => {
+        sourceLines.push(
+            `${clean(convention.id)} | ${clean(convention.num)} | ${clean(
+                convention.shortTitle,
+            )} | ${cleanDate(convention.date_publi)}`,
+        );
+    });
+
+    log.info("list()", `Updating REFERENCES.md…`);
+
+    const filePath = path.join(__dirname, "..", "REFERENCES.md");
+    const source = sourceLines.join("\n");
+    fs.writeFileSync(filePath, source);
+
+    log.info("list()", `Done.`);
+}
+
+main().catch(error => {
+    log.error("list()", `Failed: ${error}`);
+
+    process.exit(-1);
 });
-
-log.info("list()", `Updating REFERENCES.md…`);
-
-const filePath = path.join(__dirname, "..", "REFERENCES.md");
-const source = sourceLines.join("\n");
-fs.writeFileSync(filePath, source);
-
-log.info("list()", `Done.`);

scripts/match.js

@@ -14,35 +14,48 @@ import getArticlePath from "./helpers/getArticlePath";
 
 log.enableColor();
 
-log.info("match()", `Indexing articles…`);
-const agreementsIndex = getAgreements();
-
-const articlesIndex = agreementsIndex.flatMap(({ id: agreementId }) => {
-    if (/-\d+$/.test(agreementId)) {
-        return [];
-    }
-    console.warn("getAgreement", agreementId);
-
-    const agreement = getAgreement(agreementId);
-    const agreementWithFlatArticles =
-        /** @type {{ type: "root", children: KaliData.AgreementArticle }} */
-        (unistUtilFlatFilter(agreement, "article"));
-    if (agreementWithFlatArticles === null || !Array.isArray(agreementWithFlatArticles.children)) {
-        return [];
+async function main() {
+    log.info("match()", `Indexing articles…`);
+    const agreementsIndex = await getAgreements();
+
+    const articlesIndex = [];
+    for (const { id: agreementId } of agreementsIndex) {
+        if (/-\d+$/.test(agreementId)) {
+            continue;
+        }
+        console.warn("getAgreement", agreementId);
+
+        const timeStart = new Date().getTime();
+        const agreement = await getAgreement(agreementId);
+        const agreementWithFlatArticles =
+            /** @type {{ type: "root", children: KaliData.AgreementArticle }} */
+            (unistUtilFlatFilter(agreement, "article"));
+        if (
+            agreementWithFlatArticles === null ||
+            !Array.isArray(agreementWithFlatArticles.children)
+        ) {
+            continue;
+        }
+
+        const agreementArticles = agreementWithFlatArticles.children.map(
+            ({ data: { cid: articleCid, id: articleId } }) => ({
+                agreementId,
+                articleCid,
+                articleId,
+                path: getArticlePath(agreement, articleCid),
+            }),
+        );
+
+        articlesIndex.push.apply(articlesIndex, agreementArticles);
     }
 
-    const agreementArticles = agreementWithFlatArticles.children.map(
-        ({ data: { cid: articleCid, id: articleId } }) => ({
-            agreementId,
-            articleCid,
-            articleId,
-            path: getArticlePath(agreement, articleCid),
-        }),
-    );
+    const articlesIndexFilePath = path.join(__dirname, "..", "data", "articles", "index.json");
+    log.info("match()", `Writing ${articlesIndexFilePath}…`);
+    fs.writeFileSync(articlesIndexFilePath, JSON.stringify(articlesIndex, null, 2));
+}
 
-    return agreementArticles;
-});
+main().catch(error => {
+    log.error("match()", `Failed: ${error}`);
 
-const articlesIndexFilePath = path.join(__dirname, "..", "data", "articles", "index.json");
-log.info("match()", `Writing ${articlesIndexFilePath}…`);
-fs.writeFileSync(articlesIndexFilePath, JSON.stringify(articlesIndex, null, 2));
+    process.exit(-1);
+});

src/index.d.ts

@@ -5,7 +5,7 @@
  *
  * @see https://github.com/syntax-tree/unist
  */
-export function getAgreement(agreementIdOrIdcc: number | string): KaliData.Agreement;
+export function getAgreement(agreementIdOrIdcc: number | string): Promise<KaliData.Agreement>;
 
 /**
  * Get a flat unist array of all the articles an agreement contains.
@@ -20,12 +20,12 @@ export function getAgreementArticlesWithPath(
 /**
  * Convert any agreement ID or IDCC into a normalized agreement ID.
  */
-export function getAgreementIdFromIdOrIdcc(agreementIdOrIdcc: number | string): string;
+export function getAgreementIdFromIdOrIdcc(agreementIdOrIdcc: number | string): Promise<string>;
 
 /**
  * Get the full list of indexed agreements.
  */
-export function getAgreements(): KaliData.IndexedAgreement[];
+export function getAgreements(): Promise<KaliData.IndexedAgreement[]>;
 
 /**
  * Get the full list of indexed articles.
@@ -48,7 +48,7 @@ export function getIndexedArticle(articleIdOrCid: string): KaliData.IndexedArtic
 /**
  * Check if an agreement is available.
  */
-export function hasAgreement(agreementIdOrIdcc: number | string): boolean;
+export function hasAgreement(agreementIdOrIdcc: number | string): Promise<boolean>;
 
 /**
  * Check if an article is available.
@@ -161,7 +161,6 @@ type AgreementArticleDataLinkUpdate = {
 };
 
 type IndexedAgreement = {
-    active?: boolean;
     /** Publication ISO date */
     date_publi?: string;
     effectif?: number;

src/index.js

@@ -9,14 +9,12 @@ const getArticles = require("./libs/getArticles");
 const getIndexedArticle = require("./libs/getIndexedArticle");
 const hasAgreement = require("./libs/hasAgreement");
 const hasArticle = require("./libs/hasArticle");
-const getAgreementsWithNoId = require("./libs/getAgreementsWithNoId");
 
 module.exports = {
     getAgreement,
     getAgreementArticlesWithPath,
     getAgreementIdFromIdOrIdcc,
     getAgreements,
-    getAgreementsWithNoId,
     getArticleWithPath,
     getArticles,
     getIndexedArticle,

src/libs/__tests__/getAgreements.test.js

@@ -1,7 +1,7 @@
 import getAgreements from "../getAgreements";
 
 describe(`libs/getAgreements()`, () => {
-    it(`should return more than one agreement`, () => {
-        expect(getAgreements().length).toBeGreaterThan(1);
+    it(`should return more than one agreement`, async () => {
+        expect((await getAgreements()).length).toBeGreaterThan(1);
     });
 });

src/libs/getAgreement.js

@@ -10,10 +10,10 @@ const hasAgreement = require("./hasAgreement");
  *
  * @param {number | string} agreementIdOrIdcc
  *
- * @returns {KaliData.Agreement}
+ * @returns {Promise<KaliData.Agreement>}
  */
-function getAgreement(agreementIdOrIdcc) {
-    const agreementId = getAgreementIdFromIdOrIdcc(agreementIdOrIdcc);
+async function getAgreement(agreementIdOrIdcc) {
+    const agreementId = await getAgreementIdFromIdOrIdcc(agreementIdOrIdcc);
     if (!hasAgreement(agreementId)) {
         throw new Error(`No agreement found with this ID (${agreementId}).`);
     }

src/libs/getAgreementIdFromIdOrIdcc.js

@@ -1,8 +1,5 @@
 // @ts-check
-
-const getAgreements = require("../libs/getAgreements");
-
-const INDEXED_AGREEMENTS = getAgreements();
+const getAgreements = require("./getAgreements");
 
 /**
  * @param {number | string} idcc
@@ -26,11 +23,13 @@ const isAgreementId = id => typeof id === "string" && /^KALICONT\d{12}$/.test(id
  *
  * @returns {string}
  */
-function getAgreementIdFromIdOrIdcc(agreementIdOrIdcc) {
+async function getAgreementIdFromIdOrIdcc(agreementIdOrIdcc) {
+    const agreements = await getAgreements();
+
     if (isAgreementIdcc(agreementIdOrIdcc)) {
         const idcc = Number(agreementIdOrIdcc);
         const matchIdcc = ({ num }) => num === idcc;
-        const maybeAgreement = INDEXED_AGREEMENTS.find(matchIdcc);
+        const maybeAgreement = agreements.find(matchIdcc);
         if (maybeAgreement === undefined) {
             throw new Error(`No agreement found with this IDCC (${idcc}).`);
         }