Commit d0cf1044 authored by Julien Bouquillon's avatar Julien Bouquillon 🐫

feat: add fiches vdd

parent 280ac9ee
......@@ -6,6 +6,7 @@ RUN mkdir -p /tmp/clones/socialgouv
RUN git clone https://github.com/SocialGouv/legi-data /tmp/clones/socialgouv/legi-data
RUN git clone https://github.com/SocialGouv/kali-data /tmp/clones/socialgouv/kali-data
RUN git clone https://github.com/SocialGouv/fiches-vdd /tmp/clones/socialgouv/fiches-vdd
WORKDIR /app
......
......@@ -4,6 +4,7 @@
"main": "index.js",
"license": "MIT",
"dependencies": {
"@socialgouv/fiches-vdd": "^1.0.165",
"@socialgouv/kali-data": "^1.3.18",
"@socialgouv/legi-data": "^1.1.15",
"classnames": "^2.2.6",
......
import {
sync,
getLatestChanges,
getPreviousSha,
getJsonFile,
getJsonDiff
getJsonDiff,
getFilesChanged
} from "@veille/git";
import serialExec from "promise-serial-exec";
import memoizee from "memoizee";
......@@ -27,58 +26,31 @@ const compareKaliArticles = (tree1, tree2) =>
art1.data.etat !== art2.data.etat
);
// exec a compare function on the last diff of given JSON file
const getFileDiff = async ({ compare, cloneDir, path, sha }) => {
try {
const previousSha = await getPreviousSha(cloneDir, path, sha);
const tree1 = await getJsonFile({
cloneDir,
path,
oid: previousSha
});
if (!tree1) {
console.log("cannot load1", path, previousSha);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha
});
if (!tree2) {
console.log("cannot load2", path, sha);
}
const changes = compare(tree1, tree2);
return {
...tree2.data,
changes
};
} catch (e) {
console.log("e", e);
return {};
}
};
const getFileDiffKali = async (path, sha) =>
getFileDiff({
const getTreeDiffKali = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/kali-data`,
compare: compareKaliArticles,
compareFn: compareKaliArticles,
path,
sha
});
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const getFileDiffLegi = async (path, sha) =>
getFileDiff({
const getTreeDiffLegi = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/legi-data`,
compare: compareLegiArticles,
compareFn: compareLegiArticles,
path,
sha
});
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const legiPattern = /^data\/LEGITEXT000006072050\.json$/;
const kaliPattern = /^data\/KALI(?:CONT|ARTI)\d+\.json$/;
const fichesVddPattern = /^data\/[^/]+\/.*.json$/;
// add file change details to some commit
const commitMap = ({ source, filterPath, getFileDiff }) => async commit =>
......@@ -96,38 +68,116 @@ const commitMap = ({ source, filterPath, getFileDiff }) => async commit =>
const legiCommitMap = commitMap({
source: "LEGI",
filterPath: file => file.path.match(legiPattern),
getFileDiff: getFileDiffLegi
getFileDiff: getTreeDiffLegi
});
const kaliCommitMap = commitMap({
source: "KALI",
filterPath: file => file.path.match(kaliPattern),
getFileDiff: getFileDiffKali
getFileDiff: getTreeDiffKali
});
// const fichesVddCommitMap = commitMap({
// source: "FICHES-SP",
// filterPath: file => file.path.match(fichesVDDPattern)
// //getFileDiff: getFileDiffFichesVdd
// });
// const memoizedLegiCommitMap = memoizee(legiCommitMap, {
// normalizer: args => args[0].hash,
// promise: true
// });
// const memoizedKaliCommitMap = memoizee(kaliCommitMap, {
// normalizer: args => args[0].hash,
// promise: true
// });
// commit details never change, lets memoize them
const memoizedLegiCommitMap = memoizee(legiCommitMap, {
normalizer: args => args[0].hash,
promise: true
});
const memoizeCommitMap = commitMap =>
memoizee(commitMap, {
normalizer: args => args[0].hash,
promise: true
});
const memoizedKaliCommitMap = memoizee(kaliCommitMap, {
normalizer: args => args[0].hash,
promise: true
});
const getFicheMeta = (fiche, name) =>
fiche &&
fiche.children &&
fiche.children.length &&
fiche.children[0].children.find(c => c.name === name);
const getFicheMetaText = (fiche, name) => {
const node = getFicheMeta(fiche, name);
return (
node &&
node.children &&
node.children.length &&
node.children[0] &&
node.children[0].text
);
};
const getFicheTitle = data => getFicheMetaText(data, "dc:title");
const getFicheSubject = data => getFicheMetaText(data, "dc:subject");
const getFicheAriane = data => {
const fil = getFicheMeta(data, "FilDAriane");
return (
fil &&
fil.children &&
fil.children.length &&
fil.children.map(c => c.children[0].text).join(" > ")
);
};
// .map(e => e.children[0].text)
// .join(" > ");
const addVddData = path => {
const fiche = require(`@socialgouv/fiches-vdd/${path}`);
return {
path,
data: {
id: fiche.id,
title: getFicheTitle(fiche),
subject: getFicheSubject(fiche),
theme: getFicheAriane(fiche)
}
};
};
const repos = {
"socialgouv/legi-data": {
url: `https://github.com/socialgouv/legi-data.git`,
cloneDir: `/tmp/clones/socialgouv/legi-data`,
filterPath: path => path.match(legiPattern),
commitMap: memoizedLegiCommitMap
commitMap: memoizeCommitMap(legiCommitMap)
},
"socialgouv/kali-data": {
url: `https://github.com/socialgouv/kali-data.git`,
cloneDir: `/tmp/clones/socialgouv/kali-data`,
filterPath: path => path.match(kaliPattern),
commitMap: memoizedKaliCommitMap
commitMap: memoizeCommitMap(kaliCommitMap)
},
"socialgouv/fiches-vdd": {
url: `https://github.com/socialgouv/fiches-vdd.git`,
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
filterPath: path => path.match(fichesVddPattern),
commitMap: async commit => ({
...commit,
...(await getFilesChanged({
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
hash: commit.hash
}).then(changes => {
return {
source: "FICHES-SP",
...commit,
changes: {
added: changes.added.map(addVddData),
removed: changes.removed.map(addVddData),
modified: changes.modified.map(addVddData)
}
};
}))
})
}
};
......@@ -139,7 +189,6 @@ const memoizedGetLatestChanges = memoizee(getLatestChanges, {
// /api/git/[owner]/[repo]/latest
const latest = async (req, res) => {
const { owner, repo } = req.query;
//console.log("latest", owner, repo);
const repoPath = `${owner}/${repo}`;
const repoConf = repos[repoPath];
......@@ -166,7 +215,9 @@ const latest = async (req, res) => {
console.log("get diffs for these commits");
const changesWithDiffs = await serialExec(
// add some metadata to each commit
changes.map(change => () => repoConf.commitMap(change))
changes.map(change => () =>
repoConf.commitMap ? repoConf.commitMap(change) : Promise.resolve(change)
)
);
const t3 = new Date();
console.log(t3 - t2);
......
......@@ -152,7 +152,7 @@ const hasChanges = file =>
file.changes.modified.length > 0 ||
file.changes.removed.length > 0);
const ChangesGroup = ({ changes, source, label }) =>
const ChangesGroup = ({ changes, label, renderChange }) =>
changes.length ? (
<React.Fragment>
<thead>
......@@ -162,34 +162,26 @@ const ChangesGroup = ({ changes, source, label }) =>
</th>
</tr>
</thead>
<tbody>
{changes.map(f => (
<FileChangeDetail source={source} key={f.path} {...f} />
))}
</tbody>
<tbody>{changes.map(renderChange)}</tbody>
</React.Fragment>
) : null;
const ChangesDetails = ({ source, changes }) =>
const ChangesTable = ({ changes, renderChange }) =>
(changes && (
<Table size="sm" striped>
{/*<thead>
<tr>
<th>#</th>
<th>First Name</th>
<th>Last Name</th>
<th>Username</th>
</tr>
</thead>*/}
<ChangesGroup source={source} changes={changes.added} label="Nouveaux" />
<ChangesGroup
source={source}
changes={changes.added}
label="Nouveaux"
renderChange={renderChange}
/>
<ChangesGroup
changes={changes.modified}
label="Modifiés"
renderChange={renderChange}
/>
<ChangesGroup
source={source}
changes={changes.removed}
renderChange={renderChange}
label="Supprimés"
/>
</Table>
......@@ -199,7 +191,7 @@ const ChangesDetails = ({ source, changes }) =>
const getLegiId = path =>
path.replace(/^data\/((?:LEGITEXT|KALICONT)\d+)\.json/, "$1");
const getFileUrl = (source, path) => {
const getLegiFranceUrl = (source, path) => {
if (source === "LEGI") {
return `https://www.legifrance.gouv.fr/affichCode.do?cidTexte=${getLegiId(
path
......@@ -210,10 +202,19 @@ const getFileUrl = (source, path) => {
)}`;
}
};
const getFicheSpUrl = fiche => {
if (fiche.path.match(/associations/)) {
return `https://www.service-public.fr/associations/vosdroits/${fiche.data.id}`;
} else if (fiche.path.match(/particuliers/)) {
return `https://www.service-public.fr/particuliers/vosdroits/${fiche.data.id}`;
} else if (fiche.path.match(/entreprises/)) {
return `https://www.service-public.fr/professionnels-entreprises/vosdroits/${fiche.data.id}`;
}
};
const Page = ({ query, changes }) => {
//console.log("query", query);
//console.log("changes", changes);
console.log("changes", changes);
return (
<div className="container">
<Jumbotron style={{ padding: 30 }}>
......@@ -279,13 +280,35 @@ const Page = ({ query, changes }) => {
</Badge>
</a>
</h4>
{change.source === "FICHES-SP" && (
<ChangesTable
source={change.source}
changes={change.changes}
renderChange={change => (
<tr>
<td width="100" align="center">
{change.data.subject}
</td>
<td>
<a
target="_blank"
href={getFicheSpUrl(change)}
rel="noopener noreferrer"
>
{change.data.title}
</a>
</td>
</tr>
)}
/>
)}
{change.files.filter(hasChanges).map(file => (
<Card key={file.path} style={{ marginBottom: 20 }}>
<CardHeader>
<a
rel="noopener noreferrer"
target="_blank"
href={getFileUrl(change.source, file.path)}
href={getLegiFranceUrl(change.source, file.path)}
style={{ color: "black" }}
className="h4"
>
......@@ -293,9 +316,16 @@ const Page = ({ query, changes }) => {
</a>
</CardHeader>
<CardBody>
<ChangesDetails
<ChangesTable
source={change.source}
changes={file.changes}
renderChange={change2 => (
<FileChangeDetail
source={change.source}
key={change2.path}
{...change2}
/>
)}
/>
</CardBody>
</Card>
......
......@@ -46,7 +46,7 @@ const addContext = node => ({
});
// dont include children in final results
const stripChildren = ({ children, ...props }) => props;
const stripChildren = node => node; //({ children, ...props }) => props;
// return diffed articles nodes
const compareArticles = (tree1, tree2, comparator) => {
......
const Git = require("simple-git/promise");
const getFilesChangedByFilter = ({ cloneDir, hash, diffFilter }) => {
const git = Git(cloneDir);
return git
.show([
"--name-only",
`--diff-filter=${diffFilter}`,
"--pretty=format:",
hash
])
.then(res =>
res
.trim()
.split("\n")
.filter(Boolean)
);
};
// get files changed by commit and status
const getFilesChanged = async ({ cloneDir, hash }) => {
const added = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "A"
});
const modified = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "MTR"
});
const removed = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "D"
});
return {
added,
modified,
removed
};
};
module.exports = { getFilesChanged };
const { getJsonFile } = require("./getJsonFile");
const { getPreviousSha } = require("./getPreviousSha");
// compare a JSON at two different SHA
const getJsonDiff = async ({ compare, cloneDir, path, sha1, sha2 }) => {
const tree1 = await getJsonFile({
cloneDir,
path,
oid: sha1
});
if (!tree1) {
console.log("cannot load1", path, sha1);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha2
});
if (!tree2) {
console.log("cannot load2", path, sha2);
// exec a compare function on the last diff of given JSON file
const getJsonDiff = async ({ compareFn, cloneDir, path, sha }) => {
try {
const previousSha = await getPreviousSha(cloneDir, path, sha);
const tree1 = await getJsonFile({
cloneDir,
path,
oid: previousSha
});
if (!tree1) {
console.log("cannot load1", path, previousSha);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha
});
if (!tree2) {
console.log("cannot load2", path, sha);
}
return {
tree1,
tree2,
changes: compareFn(tree1, tree2)
};
} catch (e) {
console.log("e", e);
return {};
}
return tree1 && tree2 && compare(tree1, tree2);
};
module.exports = { getJsonDiff };
......@@ -4,6 +4,7 @@ const { getJsonFile } = require("./getJsonFile");
const { getLatestChanges } = require("./getLatestChanges");
const { getRepoHistory } = require("./getRepoHistory");
const { getPreviousSha } = require("./getPreviousSha");
const { getFilesChanged } = require("./getFilesChanged");
const { sync } = require("./sync");
module.exports = {
......@@ -13,5 +14,6 @@ module.exports = {
getLatestChanges,
getRepoHistory,
getPreviousSha,
getFilesChanged,
sync
};
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment