Unverified Commit 6fdf60b5 authored by Julien Bouquillon's avatar Julien Bouquillon 🐫 Committed by GitHub

Merge pull request #2 from SocialGouv/tests

feat: various
parents b4ebd7e1 6e18a3e4
......@@ -6,6 +6,7 @@ RUN mkdir -p /tmp/clones/socialgouv
RUN git clone https://github.com/SocialGouv/legi-data /tmp/clones/socialgouv/legi-data
RUN git clone https://github.com/SocialGouv/kali-data /tmp/clones/socialgouv/kali-data
RUN git clone https://github.com/SocialGouv/fiches-vdd /tmp/clones/socialgouv/fiches-vdd
WORKDIR /app
......
# veille-git
API + UI pour reporter les changements de contenus sur des repos GIT.
## Dev
```
yarn
yarn dev
```
Actuellement les repos GIT sont récupérés via le `Dockerfile`, donc mis à jour à chaque déploiement.
# Todo :
- continuous deployment with @renovate + @socialgouv
......@@ -7,7 +7,8 @@
"scripts": {
"test": "exit 0",
"lint": "exit 0",
"build": "yarn workspace @veille/frontend build"
"build": "yarn workspace @veille/frontend build",
"dev": "yarn workspace @veille/frontend dev"
},
"workspaces": [
"packages/*"
......
module.exports = {
compress: true
};
......@@ -4,14 +4,21 @@
"main": "index.js",
"license": "MIT",
"dependencies": {
"@socialgouv/fiches-vdd": "^1.0.165",
"@socialgouv/kali-data": "^1.3.18",
"@socialgouv/legi-data": "^1.1.15",
"classnames": "^2.2.6",
"diff": "1.2",
"html-text": "^1.0.1",
"isomorphic-unfetch": "^3.0.0",
"memoizee": "^0.4.14",
"next": "^9.2.2",
"promise-serial-exec": "^1.0.0",
"prop-types": "^15.7.2",
"react": "^16.13.0",
"react-dom": "^16.13.0",
"react-feather": "^2.0.3",
"reactstrap": "^8.4.1",
"unist-util-parents": "^1.0.3",
"unist-util-select": "^3.0.1"
},
......@@ -25,7 +32,13 @@
"extends": [
"@socialgouv/eslint-config-recommended",
"@socialgouv/eslint-config-react"
]
],
"rules": {
"jsx-a11y/anchor-is-valid": "warn",
"react/prop-types": "warn",
"jsx-a11y/click-events-have-key-events": "warn",
"jsx-a11y/no-static-element-interactions": "warn"
}
},
"devDependencies": {
"@socialgouv/eslint-config-react": "^0.17.0",
......
import React from "react";
import "../src/style.css";
function MyApp({ Component, pageProps }) {
return <Component {...pageProps} />;
}
export default MyApp;
import {
sync,
getLatestChanges,
getPreviousSha,
getJsonFile,
getJsonDiff
} from "@veille/git";
import { getLatestChanges, getJsonDiff, getFilesChanged } from "@veille/git";
import serialExec from "promise-serial-exec";
import memoizee from "memoizee";
import { compareArticles } from "../../../../../src/compareArticles";
/*
Compute usable diffs from our git repos
Process file changes (ex: AST)
Add some metadata to the commits
*/
const compareLegiArticles = (tree1, tree2) =>
compareArticles(
tree1,
......@@ -27,58 +27,31 @@ const compareKaliArticles = (tree1, tree2) =>
art1.data.etat !== art2.data.etat
);
// exec a compare function on the last diff of given JSON file
const getFileDiff = async ({ compare, cloneDir, path, sha }) => {
try {
const previousSha = await getPreviousSha(cloneDir, path, sha);
const tree1 = await getJsonFile({
cloneDir,
path,
oid: previousSha
});
if (!tree1) {
console.log("cannot load1", path, previousSha);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha
});
if (!tree2) {
console.log("cannot load2", path, sha);
}
const changes = compare(tree1, tree2);
return {
...tree2.data,
changes
};
} catch (e) {
console.log("e", e);
return {};
}
};
const getFileDiffKali = async (path, sha) =>
getFileDiff({
const getTreeDiffKali = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/kali-data`,
compare: compareKaliArticles,
compareFn: compareKaliArticles,
path,
sha
});
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const getFileDiffLegi = async (path, sha) =>
getFileDiff({
const getTreeDiffLegi = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/legi-data`,
compare: compareLegiArticles,
compareFn: compareLegiArticles,
path,
sha
});
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const legiPattern = /^data\/LEGITEXT000006072050\.json$/;
const kaliPattern = /^data\/KALI(?:CONT|ARTI)\d+\.json$/;
const fichesVddPattern = /^data\/[^/]+\/.*.json$/;
// add file change details to some commit
const commitMap = ({ source, filterPath, getFileDiff }) => async commit =>
......@@ -96,45 +69,109 @@ const commitMap = ({ source, filterPath, getFileDiff }) => async commit =>
const legiCommitMap = commitMap({
source: "LEGI",
filterPath: file => file.path.match(legiPattern),
getFileDiff: getFileDiffLegi
getFileDiff: getTreeDiffLegi
});
const kaliCommitMap = commitMap({
source: "KALI",
filterPath: file => file.path.match(kaliPattern),
getFileDiff: getFileDiffKali
getFileDiff: getTreeDiffKali
});
// commit details never change, lets memoize them
const memoizedLegiCommitMap = memoizee(legiCommitMap, {
normalizer: commit => commit.hash,
async: true
});
const memoizeCommitMap = commitMap =>
memoizee(commitMap, {
normalizer: args => args[0].hash,
promise: true
});
const memoizedKaliCommitMap = memoizee(kaliCommitMap, {
normalizer: commit => commit.hash,
async: true
});
const getFicheMeta = (fiche, name) =>
fiche &&
fiche.children &&
fiche.children.length &&
fiche.children[0].children.find(c => c.name === name);
const getFicheMetaText = (fiche, name) => {
const node = getFicheMeta(fiche, name);
return (
node &&
node.children &&
node.children.length &&
node.children[0] &&
node.children[0].text
);
};
const getFicheTitle = data => getFicheMetaText(data, "dc:title");
const getFicheSubject = data => getFicheMetaText(data, "dc:subject");
const getFicheAriane = data => {
const fil = getFicheMeta(data, "FilDAriane");
return (
fil &&
fil.children &&
fil.children.length &&
fil.children.map(c => c.children[0].text).join(" > ")
);
};
const addVddData = path => {
const fiche = require(`@socialgouv/fiches-vdd/${path}`);
return {
path,
data: {
id: fiche.id,
title: getFicheTitle(fiche),
subject: getFicheSubject(fiche),
theme: getFicheAriane(fiche)
}
};
};
const repos = {
"socialgouv/legi-data": {
url: `https://github.com/socialgouv/legi-data.git`,
cloneDir: `/tmp/clones/socialgouv/legi-data`,
filterPath: path => path.match(legiPattern),
commitMap: memoizedLegiCommitMap
commitMap: memoizeCommitMap(legiCommitMap)
},
"socialgouv/kali-data": {
url: `https://github.com/socialgouv/kali-data.git`,
cloneDir: `/tmp/clones/socialgouv/kali-data`,
filterPath: path => path.match(kaliPattern),
commitMap: memoizedKaliCommitMap
commitMap: memoizeCommitMap(kaliCommitMap)
},
"socialgouv/fiches-vdd": {
url: `https://github.com/socialgouv/fiches-vdd.git`,
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
filterPath: path => path.match(fichesVddPattern),
commitMap: async commit => ({
...commit,
...(await getFilesChanged({
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
hash: commit.hash
}).then(changes => {
return {
source: "FICHES-SP",
...commit,
changes: {
added: changes.added.map(addVddData),
removed: changes.removed.map(addVddData),
modified: changes.modified.map(addVddData)
}
};
}))
})
}
};
const memoizedGetLatestChanges = memoizee(getLatestChanges, {
normalizer: args => args[0].cloneDir,
promise: true
});
// /api/git/[owner]/[repo]/latest
const latest = async (req, res) => {
const { owner, repo } = req.query;
const repoPath = `${owner}/${repo}`;
const repoConf = repos[repoPath];
......@@ -144,23 +181,29 @@ const latest = async (req, res) => {
}
const start = 0;
const limit = 3;
const limit = 1;
const t = new Date();
console.log("get latest changes", owner, repo);
const changes = (
await getLatestChanges({
await memoizedGetLatestChanges({
cloneDir: repoConf.cloneDir,
filterPath: repoConf.filterPath
})
).slice(start, limit);
//console.log("changes", changes);
const t2 = new Date();
console.log(t2 - t);
console.log("get diffs for these commits");
const changesWithDiffs = await serialExec(
// add some metadata to each commit
changes.map(change => () => repoConf.commitMap(change))
changes.map(change => () =>
repoConf.commitMap ? repoConf.commitMap(change) : Promise.resolve(change)
)
);
// todo: special case : no content has changed, only main ccn.data
// filter out changes
const t3 = new Date();
console.log(t3 - t2);
res.json(changesWithDiffs);
};
......
import React from "react";
import Link from "next/link";
import { Jumbotron, Button } from "reactstrap";
const Home = () => (
<div className="container">
<Jumbotron style={{ marginTop: "15vh" }}>
<h1>Suivi des modifications GIT</h1>
<br />
<br />
<Link href="/veille/[owner]/[repo]" as="/veille/socialgouv/legi-data">
<Button size="lg" color="primary">
Accéder au suivi
</Button>
</Link>
</Jumbotron>
</div>
);
const Home = () => <div>Home</div>;
export default Home;
import React, { useState } from "react";
const Collapsible = ({ trigger, children }) => {
const [open, setOpen] = useState(false);
return (
<div>
<span onClick={() => setOpen(!open)}>{trigger}</span>
{(open && children) || null}
</div>
);
};
export default Collapsible;
// from https://github.com/davidmason/react-stylable-diff/blob/master/lib/react-diff.js
import React, { Component } from "react";
import jsdiff from "diff";
const fnMap = {
chars: jsdiff.diffChars,
words: jsdiff.diffWords,
sentences: jsdiff.diffSentences,
json: jsdiff.diffJson
};
/**
* Display diff in a stylable form.
*
* Default is character diff. Change with props.type. Valid values
* are 'chars', 'words', 'sentences', 'json'.
*
* - Wrapping div has class 'Difference', override with props.className
* - added parts are in <ins>
* - removed parts are in <del>
* - unchanged parts are in <span>
*/
export default class Diff extends Component {
render() {
const diff = fnMap[this.props.type](this.props.inputA, this.props.inputB);
const result = diff.map((part, index) => {
if (part.added) {
return <ins key={index}>{part.value}</ins>;
}
if (part.removed) {
return <del key={index}>{part.value}</del>;
}
return <span key={index}>{part.value}</span>;
});
return (
<div style={this.props.style} className={this.props.className}>
{result}
</div>
);
}
}
Diff.defaultProps = {
inputA: "",
inputB: "",
type: "chars",
className: "Difference"
};
......@@ -10,6 +10,7 @@ const getParents = node => {
return chain;
};
// find the first parent text id to make legifrance links later
const getParentTextId = node => {
let id;
node = node.parent;
......@@ -27,6 +28,7 @@ const getParentTextId = node => {
return id;
};
// find the root text id to make legifrance links later
const getRootId = node => {
let id;
while (node) {
......@@ -36,19 +38,15 @@ const getRootId = node => {
return id;
};
const stripArticle = article => ({
...article,
parents: getParents(article),
textId: getParentTextId(article),
rootId: getRootId(article)
const addContext = node => ({
...node,
parents: getParents(node),
textId: getParentTextId(node),
rootId: getRootId(node)
});
const stripSection = section => ({
...section,
parents: getParents(section),
textId: getParentTextId(section),
rootId: getRootId(section)
});
// dont include children in final results
const stripChildren = node => node; //({ children, ...props }) => props;
// return diffed articles nodes
const compareArticles = (tree1, tree2, comparator) => {
......@@ -56,10 +54,10 @@ const compareArticles = (tree1, tree2, comparator) => {
const parentsTree2 = parents(tree2);
// all articles from tree1
const articles1 = selectAll("article", parentsTree1).map(stripArticle);
const articles1 = selectAll("article", parentsTree1).map(addContext);
const articles1cids = articles1.map(a => a.data.cid);
// all articles from tree2
const articles2 = selectAll("article", parentsTree2).map(stripArticle);
const articles2 = selectAll("article", parentsTree2).map(addContext);
const articles2cids = articles2.map(a => a.data.cid);
// new : articles in tree2 not in tree1
......@@ -85,7 +83,7 @@ const compareArticles = (tree1, tree2, comparator) => {
);
// all sections from tree1
const sections1 = selectAll("section", parentsTree1).map(stripSection);
const sections1 = selectAll("section", parentsTree1).map(addContext);
// special case, kali sections have no id, but cid
const idField = (sections1[0].data.cid && "cid") || "id";
......@@ -93,7 +91,7 @@ const compareArticles = (tree1, tree2, comparator) => {
const sections1cids = sections1.map(a => a.data[idField]);
// all sections from tree2
const sections2 = selectAll("section", parentsTree2).map(stripSection);
const sections2 = selectAll("section", parentsTree2).map(addContext);
const sections2cids = sections2.map(a => a.data[idField]);
// new : sections in tree2 not in tree1
......@@ -121,8 +119,8 @@ const compareArticles = (tree1, tree2, comparator) => {
);
const changes = {
added: [...newSections, ...newArticles],
removed: [...missingSections, ...missingArticles],
added: [...newSections, ...newArticles].map(stripChildren),
removed: [...missingSections, ...missingArticles].map(stripChildren),
modified: [
...modifiedSections.map(modif => ({
...modif,
......@@ -134,7 +132,7 @@ const compareArticles = (tree1, tree2, comparator) => {
// add the previous version in the result so we can diff later
previous: articles1.find(a => a.data.cid === modif.data.cid)
}))
]
].map(stripChildren)
};
return changes;
......
ins {
background: #c8ffc8;
}
del {
background: #ffc8b7;
}
const Git = require("simple-git/promise");
const getFilesChangedByFilter = ({ cloneDir, hash, diffFilter }) => {
const git = Git(cloneDir);
return git
.show([
"--name-only",
`--diff-filter=${diffFilter}`,
"--pretty=format:",
hash
])
.then(res =>
res
.trim()
.split("\n")
.filter(Boolean)
);
};
// get files changed by commit and status
const getFilesChanged = async ({ cloneDir, hash }) => {
const added = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "A"
});
const modified = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "MTR"
});
const removed = await getFilesChangedByFilter({
cloneDir,
hash: hash,
diffFilter: "D"
});
return {
added,
modified,
removed
};
};
module.exports = { getFilesChanged };
const { getJsonFile } = require("./getJsonFile");
const { getPreviousSha } = require("./getPreviousSha");
// compare a JSON at two different SHA
const getJsonDiff = async ({ compare, cloneDir, path, sha1, sha2 }) => {
const tree1 = await getJsonFile({
cloneDir,
path,
oid: sha1
});
if (!tree1) {
console.log("cannot load1", path, sha1);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha2
});
if (!tree2) {
console.log("cannot load2", path, sha2);
// exec a compare function on the last diff of given JSON file
const getJsonDiff = async ({ compareFn, cloneDir, path, sha }) => {
try {
const previousSha = await getPreviousSha(cloneDir, path, sha);
const tree1 = await getJsonFile({
cloneDir,
path,
oid: previousSha
});
if (!tree1) {
console.log("cannot load1", path, previousSha);
}
const tree2 = await getJsonFile({
cloneDir,
path,
oid: sha
});
if (!tree2) {
console.log("cannot load2", path, sha);
}
return {
tree1,
tree2,
changes: compareFn(tree1, tree2)
};
} catch (e) {
console.log("e", e);
return {};
}