Unverified Commit 5a7022b4 authored by Julien Bouquillon's avatar Julien Bouquillon 🐫 Committed by GitHub

feat: prerendering (#22)

* chore: add clone.sh script

* feat: implement history

* feat: add yarn start :)

* increase limits

* feat: use fiches-vdd/data/index

* fix: fiches-vdd nullify
parent b5d200d3
......@@ -4,7 +4,8 @@ include:
ref: v8.0.0
variables:
PROJECT: "veille-git"
#RANCHER_PROJECT_ID: "c-gsm8d:p-pwpk6" # "default" project id here
PROJECT: "cdtn-veille-git"
RANCHER_PROJECT_ID: "c-kk8xm:p-8n8mv" # "default" project id here
PORT: 3000
VALUES_FILE: ./.k8s/app.values.yml # Your values
DOCKER_BUILD_ARGS: "--no-cache"
......@@ -2,9 +2,9 @@ image:
repository: registry.gitlab.factory.social.gouv.fr/socialgouv/veille-git
labels:
app.kubernetes.io/part-of: veille-git
owner: veille-git
team: veille-git
app.kubernetes.io/part-of: cdtn-veille-git
owner: cdtn-veille-git
team: cdtn-veille-git
deployment:
annotations:
......@@ -22,10 +22,10 @@ deployment:
resources:
limits:
memory: 1024Mi
memory: 2048Mi
cpu: 1000m
requests:
memory: 64Mi
memory: 1024Mi
cpu: 100m
env:
......@@ -42,11 +42,11 @@ ingress:
appgw.ingress.kubernetes.io/ssl-redirect: "true"
kubernetes.io/ingress.class: "azure/application-gateway"
hosts:
- host: "veille-git.fabrique.social.gouv.fr"
- host: "cdtn-veille-git.fabrique.social.gouv.fr"
paths:
- path: /
servicePort: 3000
tls:
- hosts:
- "veille-git.fabrique.social.gouv.fr"
secretName: veille-git-crt
- "cdtn-veille-git.fabrique.social.gouv.fr"
secretName: cdtn-veille-git-crt
......@@ -2,13 +2,11 @@ FROM node:12-alpine
RUN apk add git
RUN mkdir -p /tmp/clones/socialgouv
WORKDIR /app
RUN git clone https://github.com/SocialGouv/legi-data /tmp/clones/socialgouv/legi-data
RUN git clone https://github.com/SocialGouv/kali-data /tmp/clones/socialgouv/kali-data
RUN git clone https://github.com/SocialGouv/fiches-vdd /tmp/clones/socialgouv/fiches-vdd
COPY clone.sh .
WORKDIR /app
RUN ./clone.sh
COPY package.json .
COPY yarn.lock .
......@@ -19,8 +17,7 @@ RUN yarn
COPY packages ./packages
RUN yarn workspace @veille/frontend build
RUN yarn prerender
ENTRYPOINT ["yarn", "workspace", "@veille/frontend", "start"]
......@@ -2,8 +2,17 @@
API + UI pour reporter les changements de contenus sur des repos GIT.
## API
| endpoint | usage |
| --------------------------------------- | ----------------------------- |
| `/api/git/[owner]/[repo]/history` | fetch latest commits |
| `/api/git/[owner]/[repo]/commit/[hash]` | fetch detailed commit changes |
## Dev
Utiliser [clone.sh](./clone.sh) pour récupérer les repos dans `/tmp/clones`.
```
yarn
yarn dev
......@@ -11,6 +20,12 @@ yarn dev
Actuellement les repos GIT sont récupérés via le `Dockerfile`, donc mis à jour à chaque déploiement.
## Repos
- [legi-data](https://github.com/SocialGouv/legi-data)
- [kali-data](https://github.com/SocialGouv/kali-data)
- [fiches-vdd](https://github.com/SocialGouv/fiches-vdd)
# Todo :
- continuous deployment with @renovate + @socialgouv
......
#!/bin/sh
GIT_STORAGE=/tmp/clones
mkdir -p $GIT_STORAGE || true
git clone https://github.com/SocialGouv/legi-data $GIT_STORAGE/socialgouv/legi-data || (cd $GIT_STORAGE/socialgouv/legi-data && git pull)
git clone https://github.com/SocialGouv/kali-data $GIT_STORAGE/socialgouv/kali-data || (cd $GIT_STORAGE/socialgouv/kali-data && git pull)
git clone https://github.com/SocialGouv/fiches-vdd $GIT_STORAGE/socialgouv/fiches-vdd || (cd $GIT_STORAGE/socialgouv/fiches-vdd && git pull)
......@@ -7,8 +7,10 @@
"scripts": {
"test": "exit 0",
"lint": "exit 0",
"build": "yarn workspace @veille/frontend build",
"dev": "yarn workspace @veille/frontend dev"
"build": "exit 0",
"prerender": "NODE_OPTIONS=\"--max-old-space-size=8192\" yarn workspace @veille/frontend build",
"dev": "NODE_OPTIONS=\"--max-old-space-size=8192\" yarn workspace @veille/frontend dev",
"start": "NODE_OPTIONS=\"--max-old-space-size=8192\" yarn workspace @veille/frontend start"
},
"workspaces": [
"packages/*"
......
......@@ -4,15 +4,13 @@
"main": "index.js",
"license": "MIT",
"dependencies": {
"@socialgouv/fiches-vdd": "^1.0.189",
"@socialgouv/kali-data": "^1.3.30",
"@socialgouv/legi-data": "^1.1.19",
"@socialgouv/fiches-vdd": "^1.0.196",
"classnames": "^2.2.6",
"diff": "1.2",
"html-text": "^1.0.1",
"isomorphic-unfetch": "^3.0.0",
"memoizee": "^0.4.14",
"next": "^9.2.2",
"next": "9.3.1-canary.4",
"promise-serial-exec": "^1.0.0",
"prop-types": "^15.7.2",
"react": "^16.13.0",
......
import { showCommit } from "@veille/git";
import memoizee from "memoizee";
import repos from "../../../../../../src/repos";
/*
Compute usable diffs from our git repos
Process file changes (ex: AST)
Add some metadata to the commits
*/
// commit infos can be cached
const memoizedShowCommit = memoizee(showCommit, {
normalizer: args => args[0].cloneDir + args[0].hash,
promise: true
});
//
// /api/git/[owner]/[repo]/hash
// return given commit changes
//
const getChanges = async (req, res) => {
const { owner, repo, hash = "latest" } = req.query;
const repoPath = `${owner}/${repo}`;
const repoConf = repos[repoPath];
if (!repoConf) {
// todo: conf not found; fallback ?
res.status(404).json({ error: 404 });
return;
}
const t = new Date();
console.log("get commit", owner, repo, hash);
const commit = await showCommit({
cloneDir: repoConf.cloneDir,
filterPath: repoConf.filterPath,
hash
});
const t2 = new Date();
console.log(t2 - t);
console.log("get diffs for these commits");
const changes = await repoConf.processCommit(commit);
const t3 = new Date();
console.log(t3 - t2);
res.json(changes);
};
export const getStaticPaths = async () => {
console.log("wahhhht");
return {
paths: []
};
};
export default getChanges;
import { getLatestChanges } from "@veille/git";
import memoizee from "memoizee";
import repos from "../../../../../src/repos";
const memoizedGetLatestChanges = memoizee(getLatestChanges, {
normalizer: args => args[0].cloneDir,
promise: true
});
const getHistory = async (req, res) => {
const { owner, repo } = req.query;
const repoPath = `${owner}/${repo}`;
const repoConf = repos[repoPath];
if (!repoConf) {
// todo: conf not found; fallback ?
res.status(404).json({ error: 404 });
return;
}
console.log("get latest changes", owner, repo);
const history = await getLatestChanges({
cloneDir: repoConf.cloneDir,
filterPath: repoConf.filterPath
});
res.json(history);
};
export default getHistory;
import { getLatestChanges, getJsonDiff, getFilesChanged } from "@veille/git";
import serialExec from "promise-serial-exec";
import memoizee from "memoizee";
import { compareArticles } from "../../../../../src/compareArticles";
/*
Compute usable diffs from our git repos
Process file changes (ex: AST)
Add some metadata to the commits
*/
const compareLegiArticles = (tree1, tree2) =>
compareArticles(
tree1,
tree2,
(art1, art2) =>
art1.data.texte !== art2.data.texte || art1.data.etat !== art2.data.etat
);
const compareKaliArticles = (tree1, tree2) =>
compareArticles(
tree1,
tree2,
(art1, art2) =>
art1.data.content !== art2.data.content ||
art1.data.etat !== art2.data.etat
);
const getTreeDiffKali = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/kali-data`,
compareFn: compareKaliArticles,
path,
sha
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const getTreeDiffLegi = (path, sha) =>
getJsonDiff({
cloneDir: `/tmp/clones/socialgouv/legi-data`,
compareFn: compareLegiArticles,
path,
sha
}).then(({ tree2, changes }) => ({
...tree2.data,
changes
}));
const legiPattern = /^data\/LEGITEXT000006072050\.json$/;
const kaliPattern = /^data\/KALI(?:CONT|ARTI)\d+\.json$/;
const fichesVddPattern = /^data\/[^/]+\/.*.json$/;
// add file change details to some commit
const commitMap = ({ source, filterPath, getFileDiff }) => async commit =>
await {
source,
...commit,
files: await serialExec(
commit.files.filter(filterPath).map(file => async () => ({
path: file.path,
...(await getFileDiff(file.path, commit.hash))
}))
)
};
const legiCommitMap = commitMap({
source: "LEGI",
filterPath: file => file.path.match(legiPattern),
getFileDiff: getTreeDiffLegi
});
const kaliCommitMap = commitMap({
source: "KALI",
filterPath: file => file.path.match(kaliPattern),
getFileDiff: getTreeDiffKali
});
// commit details never change, lets memoize them
const memoizeCommitMap = commitMap =>
memoizee(commitMap, {
normalizer: args => args[0].hash,
promise: true
});
const getFicheMeta = (fiche, name) =>
fiche &&
fiche.children &&
fiche.children.length &&
fiche.children[0].children.find(c => c.name === name);
const getFicheMetaText = (fiche, name) => {
const node = getFicheMeta(fiche, name);
return (
node &&
node.children &&
node.children.length &&
node.children[0] &&
node.children[0].text
);
};
const getFicheTitle = data => getFicheMetaText(data, "dc:title");
const getFicheSubject = data => getFicheMetaText(data, "dc:subject");
const getFicheAriane = data => {
const fil = getFicheMeta(data, "FilDAriane");
return (
fil &&
fil.children &&
fil.children.length &&
fil.children.map(c => c.children[0].text).join(" > ")
);
};
const addVddData = path => {
const fiche = require(`@socialgouv/fiches-vdd/${path}`);
return {
path,
data: {
id: fiche.id,
title: getFicheTitle(fiche),
subject: getFicheSubject(fiche),
theme: getFicheAriane(fiche)
}
};
};
const repos = {
"socialgouv/legi-data": {
url: `https://github.com/socialgouv/legi-data.git`,
cloneDir: `/tmp/clones/socialgouv/legi-data`,
filterPath: path => path.match(legiPattern),
commitMap: memoizeCommitMap(legiCommitMap)
},
"socialgouv/kali-data": {
url: `https://github.com/socialgouv/kali-data.git`,
cloneDir: `/tmp/clones/socialgouv/kali-data`,
filterPath: path => path.match(kaliPattern),
commitMap: memoizeCommitMap(kaliCommitMap)
},
"socialgouv/fiches-vdd": {
url: `https://github.com/socialgouv/fiches-vdd.git`,
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
filterPath: path => path.match(fichesVddPattern),
commitMap: async commit => ({
...commit,
...(await getFilesChanged({
cloneDir: `/tmp/clones/socialgouv/fiches-vdd`,
hash: commit.hash
}).then(changes => {
return {
source: "FICHES-SP",
...commit,
changes: {
added: changes.added.map(addVddData),
removed: changes.removed.map(addVddData),
modified: changes.modified.map(addVddData)
}
};
}))
})
}
};
const memoizedGetLatestChanges = memoizee(getLatestChanges, {
normalizer: args => args[0].cloneDir,
promise: true
});
// /api/git/[owner]/[repo]/latest
const latest = async (req, res) => {
const { owner, repo } = req.query;
const repoPath = `${owner}/${repo}`;
const repoConf = repos[repoPath];
if (!repoConf) {
res.status(404).json({ error: 404 });
return;
}
const start = 0;
const limit = 1;
const t = new Date();
console.log("get latest changes", owner, repo);
const changes = (
await memoizedGetLatestChanges({
cloneDir: repoConf.cloneDir,
filterPath: repoConf.filterPath
})
).slice(start, limit);
//console.log("changes", changes);
const t2 = new Date();
console.log(t2 - t);
console.log("get diffs for these commits");
const changesWithDiffs = await serialExec(
// add some metadata to each commit
changes.map(change => () =>
repoConf.commitMap ? repoConf.commitMap(change) : Promise.resolve(change)
)
);
const t3 = new Date();
console.log(t3 - t2);
res.json(changesWithDiffs);
};
export default latest;
This diff is collapsed.
import React, { useState } from "react";
const Collapsible = ({ trigger, children }) => {
export const Collapsible = ({ trigger, children }) => {
const [open, setOpen] = useState(false);
return (
<div>
......@@ -10,4 +10,3 @@ const Collapsible = ({ trigger, children }) => {
);
};
export default Collapsible;
// from https://github.com/davidmason/react-stylable-diff/blob/master/lib/react-diff.js
// adapted from https://github.com/davidmason/react-stylable-diff/blob/master/lib/react-diff.js
import React, { Component } from "react";
import React from "react";
import jsdiff from "diff";
const fnMap = {
......@@ -10,37 +10,23 @@ const fnMap = {
json: jsdiff.diffJson
};
/**
* Display diff in a stylable form.
*
* Default is character diff. Change with props.type. Valid values
* are 'chars', 'words', 'sentences', 'json'.
*
* - Wrapping div has class 'Difference', override with props.className
* - added parts are in <ins>
* - removed parts are in <del>
* - unchanged parts are in <span>
*/
export default class Diff extends Component {
render() {
const diff = fnMap[this.props.type](this.props.inputA, this.props.inputB);
const result = diff.map((part, index) => {
if (part.added) {
return <ins key={index}>{part.value}</ins>;
}
if (part.removed) {
return <del key={index}>{part.value}</del>;
}
return <span key={index}>{part.value}</span>;
});
return (
<div style={this.props.style} className={this.props.className}>
{result}
</div>
);
}
}
export const Diff = ({ style, className, type, inputA, inputB }) => {
const diff = fnMap[type](inputA, inputB);
const result = diff.map((part, index) => {
if (part.added) {
return <ins key={index}>{part.value}</ins>;
}
if (part.removed) {
return <del key={index}>{part.value}</del>;
}
return <span key={index}>{part.value}</span>;
});
return (
<div style={style} className={className}>
{result}
</div>
);
};
Diff.defaultProps = {
inputA: "",
......
import React from "react";
import { Badge } from "reactstrap";
export const HashLink = ({ className, owner, repo, hash }) => (
<a
className={className}
rel="noopener noreferrer"
target="_blank"
href={`https://github.com/${owner}/${repo}/commit/${hash}`}
>
<Badge color="light" style={{ color: "#888", fontSize: "0.5em" }}>
{hash.slice(0, 8)}
</Badge>
</a>
);
......@@ -25,7 +25,7 @@ const getParentTextId = node => {
}
node = node.parent;
}
return id;
return id || null;
};
// find the root text id to make legifrance links later
......@@ -35,14 +35,14 @@ const getRootId = node => {
id = node.data.id;
node = node.parent;
}
return id;
return id || null;
};
const addContext = node => ({
...node,
parents: getParents(node),
textId: getParentTextId(node),
rootId: getRootId(node)
textId: getParentTextId(node) || null,
rootId: getRootId(node) || null
});
// dont include children in final results
......@@ -55,30 +55,40 @@ const compareArticles = (tree1, tree2, comparator) => {
// all articles from tree1
const articles1 = selectAll("article", parentsTree1).map(addContext);
const articles1cids = articles1.map(a => a.data.cid);
const articles1cids = articles1
.map(a => a && a.data && a.data.cid)
.filter(Boolean);
// all articles from tree2
const articles2 = selectAll("article", parentsTree2).map(addContext);
const articles2cids = articles2.map(a => a.data.cid);
const articles2cids = articles2
.map(a => a && a.data && a.data.cid)
.filter(Boolean);
// new : articles in tree2 not in tree1
const newArticles = articles2.filter(
art => !articles1cids.includes(art.data.cid)
art => art && art.data && !articles1cids.includes(art.data.cid)
);
const newArticlesCids = newArticles.map(a => a.data.cid);
// supressed: articles in tree1 not in tree2
const missingArticles = articles1.filter(
art => !articles2cids.includes(art.data.cid)
art => art && art.data && !articles2cids.includes(art.data.cid)
);
// modified : articles with modified texte
const modifiedArticles = articles2.filter(
art =>
art &&
art.data &&
// exclude new articles
!newArticlesCids.includes(art.data.cid) &&
articles1.find(
// same article, different texte
art2 => art2.data.cid === art.data.cid && comparator(art, art2)
art2 =>
art2 &&
art2.data &&
art2.data.cid === art.data.cid &&
comparator(art, art2)
)
);
......@@ -86,7 +96,