Unverified Commit 04863241 authored by Ivan Gabriele's avatar Ivan Gabriele Committed by GitHub
Browse files

feat(app): use new api for legal references (#1080)

parent fae9a084
......@@ -8,6 +8,10 @@
},
"rules": {
"no-console": ["error", { "allow": ["info", "error", "warn"] }],
"padding-line-between-statements": [
"error",
{ "blankLine": "always", "prev": "*", "next": "return" }
],
"jest/no-disabled-tests": "error",
"prettier/prettier": "error",
"sort-requires/sort-requires": "error"
......
......@@ -78,7 +78,6 @@ to update Unite Tests snapshots, you can run `yarn test:update`.
This repository comes with multiple useful npm scripts (run via `yarn <script>`):
- `data:generate`: Generate (update) DILA-related data (Labor Code & Agreements).
- `db:backup`: Generate a database dump.
- `db:migrate` Migrate database schema.
- `db:migrate:make`: Create a new database migration file.
......@@ -104,7 +103,6 @@ This repository comes with multiple useful npm scripts (run via `yarn <script>`)
```json
{
"coverage-gutters.coverageReportFileName": "packages/contrib/coverage/**/index.html",
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnPaste": false,
"editor.formatOnSave": true,
......
-------------------------------------- UP --------------------------------------
ALTER TABLE api.answers_references
DROP COLUMN is_skipped;
------------------------------------- DOWN -------------------------------------
ALTER TABLE api.answers_references
ADD COLUMN is_skipped boolean NOT NULL DEFAULT FALSE;
const getMigrationQuery = require("../../../scripts/db/getMigrationQuery");
exports.up = async knex => {
await knex.raw(
getMigrationQuery("20200623083723_remove_answers_references_table_is_skipped_field").up(),
);
};
exports.down = async knex => {
await knex.raw(
getMigrationQuery("20200623083723_remove_answers_references_table_is_skipped_field").down(),
);
};
......@@ -150,7 +150,10 @@ exports.seed = async knex => {
.eq("answer_id", answer.id)
.get("/answers_references");
answersReferences = answersReferences.concat(foundAnswerReferences);
answersReferences = answersReferences
.concat(foundAnswerReferences)
// eslint-disable-next-line no-unused-vars
.map(({ is_skipped, ...answerReference }) => answerReference);
continue;
}
......
No preview for this file type
......@@ -5,7 +5,6 @@
"license": "Apache-2.0",
"private": true,
"scripts": {
"data:generate": "node ./scripts/data/generateLaborCodeArticles.js && node ./scripts/data/generateIndex.js",
"db:backup": "node -r dotenv/config ./scripts/db/backup.js",
"db:migrate": "knex migrate:latest",
"db:migrate:make": "node ./scripts/db/generateMigration.js",
......@@ -41,7 +40,6 @@
"devDependencies": {
"@socialgouv/eslint-config-react-strict": "0.33.0",
"@socialgouv/eslint-config-strict": "0.33.0",
"@socialgouv/kali-data": "1.73.0",
"@testing-library/jest-dom": "5.10.1",
"@types/cucumber": "6.0.1",
"@types/jest": "26.0.0",
......
This diff is collapsed.
This diff is collapsed.
......@@ -9,14 +9,10 @@
"start": "node ./src"
},
"dependencies": {
"@inspired-beings/log": "2.0.0",
"@socialgouv/kali-data": "1.73.0",
"fuse.js": "6.2.1",
"html-to-text": "5.1.1",
"http-proxy": "1.18.1",
"jsonwebtoken": "8.5.1",
"knex": "0.21.1",
"node-cache": "5.1.1",
"npmlog": "4.1.2",
"pg": "8.2.1"
},
"devDependencies": {
......
// @ts-check
const url = require("url");
const answerWithError = require("../helpers/answerWithError");
const findAgreementArticles = require("../libs/findAgreementArticles");
const findLaborCodeArticles = require("../libs/findLaborCodeArticles");
const getArticleById = require("../libs/getArticleById");
const { COMMON_HEADERS, LEGAL_REFERENCE_CATEGORY } = require("../constants");
const LEGAL_REFERENCE_CATEGORYS = Object.values(LEGAL_REFERENCE_CATEGORY);
class LegalReference {
/**
* @param {import("http").IncomingMessage} req
* @param {import("http").ServerResponse} res
*/
get(req, res) {
try {
const path = String(url.parse(String(req.url)).pathname);
const id = path.substr(path.lastIndexOf("/") + 1);
const body = getArticleById(id);
if (body === null) {
answerWithError(
"controllers/LegalReference#get()",
`Could not find any matching legal reference with {id}="${id}".`,
res,
404,
);
}
res.writeHead(200, COMMON_HEADERS);
res.end(JSON.stringify(body));
} catch (err) {
answerWithError("controllers/LegalReference#get()", err, res);
}
}
/**
* @param {import("http").IncomingMessage} req
* @param {import("http").ServerResponse} res
*
* @example
* - http://localhost:3200/legal-references?idcc=2216&type=agreement&query=12.1
* - http://localhost:3200/legal-references?idcc=2216&type=labor_code&query=L1234-5
*/
index(req, res) {
try {
const queryParams = url.parse(String(req.url), true).query;
const { category, idcc, query } = queryParams;
switch (true) {
case typeof category === "undefined":
throw new Error("The `category` query parameter is mandatory.");
case typeof query === "undefined":
throw new Error("The `query` query parameter is mandatory.");
case typeof category !== "string":
throw new Error("The `category` query parameter must be a {string}.");
case typeof query !== "string":
throw new Error("The `query` query parameter must be a {string}.");
case !LEGAL_REFERENCE_CATEGORYS.includes(String(category)):
// eslint-disable-next-line no-case-declarations
const types = `"${LEGAL_REFERENCE_CATEGORYS.join(`", "`)}"`;
throw new Error(`The \`type\` category parameter must be one of: ${types}.`);
case category === LEGAL_REFERENCE_CATEGORY.AGREEMENT && typeof idcc === "undefined":
throw new Error("The `idcc` query parameter is mandatory.");
case category === LEGAL_REFERENCE_CATEGORY.AGREEMENT && typeof idcc !== "string":
throw new Error("The `idcc` query parameter must be a {string}.");
}
let body;
switch (String(category)) {
case LEGAL_REFERENCE_CATEGORY.AGREEMENT:
body = findAgreementArticles(String(idcc), String(query));
break;
case LEGAL_REFERENCE_CATEGORY.LABOR_CODE:
body = findLaborCodeArticles(String(query));
break;
}
res.writeHead(200, COMMON_HEADERS);
res.end(JSON.stringify(body));
} catch (err) {
answerWithError("controllers/LegalReference#index()", err, res);
}
}
}
module.exports = new LegalReference();
const convertHtmlToPlainText = require("../convertHtmlToPlainText");
describe("helpers/convertHtmlToPlainText()", () => {
it(`should return the expected result`, () => {
const source = `<p><br/>A <a alt='alternative' href='/path?a=b&c=d'>link</a>, a text.</p>`;
const expected = `A link, a text.`;
const result = convertHtmlToPlainText(source);
expect(result.length).toBeGreaterThan(0);
expect(result).toStrictEqual(expected);
});
});
// @ts-check
const log = require("@inspired-beings/log");
const log = require("npmlog");
log.enableColor();
const { COMMON_HEADERS } = require("../constants");
/**
......@@ -41,7 +42,7 @@ function answerWithError(path, error, res, code) {
}
if (code === undefined || code === 400 || code >= 500) {
bodyJson.errors.map(error => log.err(`[api] [${path}] Error: %s`, error));
bodyJson.errors.map(error => log.err(`[api] [${path}]`, "Error: %s", error));
}
}
......
const NodeCache = require("node-cache");
module.exports = new NodeCache();
// @ts-check
const htmlToText = require("html-to-text");
// https://github.com/werk85/node-html-to-text#options
const HTML_TO_TEXT_OPTIONS = {
ignoreHref: true,
ignoreImage: true,
wordwrap: 60,
};
/**
* @param {string} source
*
* @returns {string}
*/
function convertHtmlToPlainText(source) {
return htmlToText
.fromString(source, HTML_TO_TEXT_OPTIONS)
.trim()
.replace(/\n{3,}/g, "\n\n");
}
module.exports = convertHtmlToPlainText;
const http = require("http");
const httpProxy = require("http-proxy");
const log = require("@inspired-beings/log");
const log = require("npmlog");
const answerWithError = require("./helpers/answerWithError");
const logAction = require("./hooks/logAction");
const route = require("./middlewares/route");
log.enableColor();
const NODE_ENV = process.env.NODE_ENV !== undefined ? process.env.NODE_ENV : "development";
const { API_PORT, DEV_POSTGREST_PORT } = process.env;
let { POSTGREST_URI } = process.env;
......@@ -18,9 +18,6 @@ const proxy = httpProxy.createProxyServer().on("proxyReq", logAction);
http
.createServer((req, res) => {
try {
const isRouted = route(req, res);
if (isRouted) return;
proxy.web(req, res, { target: POSTGREST_URI });
} catch (err) {
answerWithError("index.js", err, res);
......@@ -28,4 +25,4 @@ http
})
.listen(API_PORT);
log.info(`[api] [index.js] Info: Listening on %s (%s).`, API_PORT, NODE_ENV);
log.info("[api] [index.js]", "Listening on %s (%s).", API_PORT, NODE_ENV);
const findAgreementArticles = require("../findAgreementArticles");
describe("libs/findAgreementArticles()", () => {
it(`should find "Article 04.05.1" (0029)`, () => {
const result = findAgreementArticles("0029", "Article 04.05.1");
expect(result.length).toBeGreaterThan(0);
expect(result[0].id).toStrictEqual("KALIARTI000029952604");
});
it(`should find "Avenant n° 1 Ouvriers et collaborateurs du 11 février 1971 Article 27" (0044)`, () => {
const result = findAgreementArticles(
"0044",
"Avenant n° 1 Ouvriers et collaborateurs du 11 février 1971 Article 27",
);
expect(result.length).toBeGreaterThan(0);
expect(result[0].id).toStrictEqual("KALIARTI000005846394");
});
});
const findLaborCodeArticles = require("../findLaborCodeArticles");
describe("libs/findLaborCodeArticles()", () => {
// TODO Find what's wrong here.
it(`should find "L1223-5`, () => {
const result = findLaborCodeArticles("L1223-5");
expect(result.length).toBeGreaterThan(0);
expect(result[0].id).toStrictEqual("LEGIARTI000006900872");
});
it(`should find "R1111.1`, () => {
const result = findLaborCodeArticles("R1111.1");
expect(result.length).toBeGreaterThan(0);
expect(result[0].id).toStrictEqual("LEGIARTI000018538086");
});
});
// @ts-check
/** @type {typeof Fuse.default} */
const FuseJs = /** @type {*} */ (require("fuse.js"));
const Agreement = require("../services/Agreement");
/**
* @typedef {LegalReference.Article} ArticleWithScore
* @property {number} score
*/
/**
* @param {string} idcc
* @param {string} query
*
* @returns {ArticleWithScore[]}
*/
function findAgreementArticles(idcc, query) {
const allArticles = Agreement.getArticles(idcc);
let cleanQuery = query
.replace(/\s+-|-\s+/g, " ")
.replace(/[^a-z0-9áâàäéêèëíîìïóôòöúûùüç\s-]/gi, " ")
.replace(/\s+/g, " ")
.trim()
.toLocaleLowerCase();
cleanQuery = "'" + cleanQuery.replace(/\s/g, " '");
if (!cleanQuery.includes("article") && /'\d{1,2}$/.test(cleanQuery)) {
cleanQuery = cleanQuery.replace(/'(\d{1,2}$)/, " '» 'Article $1$");
}
/** @type {Fuse.default.IFuseOptions<LegalReference.Article>} */
const fuseJsOptions = {
distance: 999,
findAllMatches: false,
includeMatches: false,
includeScore: true,
isCaseSensitive: false,
keys: ["fullText"],
minMatchCharLength: 1,
shouldSort: true,
threshold: 0.5,
useExtendedSearch: true,
};
const fuseJs = new FuseJs(allArticles, fuseJsOptions);
const foundArticles =
/** @type {Fuse.default.FuseResult<LegalReference.Article>[]} */
(fuseJs.search(cleanQuery));
const foundArticlesWithScore = foundArticles
.slice(0, 10)
.map(({ item, score }) => ({ ...item, score }));
return foundArticlesWithScore;
}
module.exports = findAgreementArticles;
// @ts-check
/** @type {typeof Fuse.default} */
const FuseJs = /** @type {*} */ (require("fuse.js"));
const LaborCode = require("../services/LaborCode");
/**
* @typedef {LegalReference.Article} ArticleWithScore
* @property {number} score
*/
/**
* @param {string} query
*
* @returns {ArticleWithScore[]}
*/
function findLaborCodeArticles(query) {
const allArticles = LaborCode.getArticles();
/** @type {Fuse.default.IFuseOptions<LegalReference.Article>} */
const fuseJsOptions = {
distance: 0,
includeScore: true,
keys: ["index"],
shouldSort: true,
};
const fuseJs = new FuseJs(allArticles, fuseJsOptions);
const foundArticles =
/** @type {Fuse.default.FuseResult<LegalReference.Article>[]} */
(fuseJs.search(query));
const foundArticlesWithScore = foundArticles
.slice(0, 10)
.map(({ item, score }) => ({ ...item, score }));
return foundArticlesWithScore;
}
module.exports = findLaborCodeArticles;
// @ts-check
const Agreement = require("../services/Agreement");
const LaborCode = require("../services/LaborCode");
const INDEX = /** @type {LegalReference.ArticleIndex[]} */ (require("../../data/index.json"));
/**
* @param {string} articleId
*
* @returns {?LegalReference.Article}
*/
function getArticleById(articleId) {
/** @type {LegalReference.Article[]} */
let articles;
if (articleId.startsWith("KALI")) {
const maybeIndex = INDEX.find(({ id }) => id === articleId);
if (typeof maybeIndex === "undefined") {
return null;
}
const { agreementId } = maybeIndex;
articles = Agreement.getArticles(agreementId);
} else {
articles = LaborCode.getArticles();
}
const maybeArticle = articles.find(({ id }) => id === articleId);
if (typeof maybeArticle === "undefined") {
return null;
}
return maybeArticle;
}
module.exports = getArticleById;
const answerWithError = require("../helpers/answerWithError");
const LegalReferenceController = require("../controllers/LegalReference");
const { COMMON_HEADERS } = require("../constants");
/**
* @param {import("http").IncomingMessage} req
* @param {import("http").ServerResponse} res
*/
function route(req, res) {
try {
switch (true) {
case /^\/legal-references($|\?)/.test(req.url) && req.method === "OPTIONS":
case /^\/legal-references\/[0-9a-z]+/i.test(req.url) && req.method === "OPTIONS":
res.writeHead(200, COMMON_HEADERS);
res.end();
return true;
case /^\/legal-references($|\?)/.test(req.url) && req.method === "GET":
LegalReferenceController.index(req, res);
return true;
case /^\/legal-references\/[0-9a-z]+/i.test(req.url) && req.method === "GET":
LegalReferenceController.get(req, res);
return true;
default:
return false;
}
} catch (err) {
answerWithError("middlewares/route()", err, res);
}
}
module.exports = route;
// @ts-check
const cache = require("../helpers/cache");
const convertHtmlToPlainText = require("../helpers/convertHtmlToPlainText");
const { LEGAL_REFERENCE_CATEGORY } = require("../constants");
const CACHE_TTL = 4 * 60 * 60; // => 4h
const AGREEMENTS_INDEX =
/** @type {import("@socialgouv/kali-data").IndexAgreement[]} */
require("@socialgouv/kali-data/data/index.json");
/**
* @param {[LegalReference.Article[], string | undefined]} prev
* @param {import("@socialgouv/kali-data").AgreementArticleOrSection} articleOrSection
*
* @returns {[LegalReference.Article[], string | undefined]}
*/
function normalizeMainArticles([normalizedArticles, lastSectionTitle], articleOrSection) {
const { type } = articleOrSection;
if (type === "section") {
const kaliSection =
/** @type {import("@socialgouv/kali-data").AgreementSection} */
(articleOrSection);
const { title } = kaliSection.data;
const sectionTitle = title.trim();
return [normalizedArticles, sectionTitle];
}
const kaliArticle =
/** @type {import("@socialgouv/kali-data").AgreementArticle} */
(articleOrSection);
const { cid, etat, id, num, surtitre } = kaliArticle.data;
const content = convertHtmlToPlainText(kaliArticle.data.content);
const index = num !== null ? num : null;
const title = lastSectionTitle !== undefined ? lastSectionTitle : null;
const state = etat;
const subtitle = surtitre !== undefined ? surtitre.trim() : null;
const fullText = `${index !== null ? `Article ${index}` : ""}${
title !== null ? ` (${title})` : ""
}`;
/** @type {LegalReference.Article} */
const article = {
agreementId: "",
cid,
content,
fullText,
id,
index,
isAnnex: false,