Unverified Commit 7fde2ec3 authored by Lionel's avatar Lionel Committed by GitHub
Browse files

fix(data): remove fake idcc results and fix codenaf (#45)

* fix(data): remove fake idcc results

* wtf source code is in snapshots

* fix(naf): handle more code naf

* fix: filter ccn elasticsearch

* fix filter
parent 5db53f10
......@@ -5,3 +5,4 @@ python-env/
node_modules
siret-idcc.csv
api/dist
index/dist
\ No newline at end of file
......@@ -210,9 +210,7 @@ data:
\\"\\"\\"
idccs = pd.read_csv(idcc_file, usecols=[\\"SIRET\\", \\"IDCC\\"]).rename(
columns={\\"SIRET\\": \\"siret\\", \\"IDCC\\": \\"idcc\\"})
# drop unknown
unknown = idccs[\\"idcc\\"] == 9999
idccs = idccs.drop(idccs[unknown].index)
return idccs
......
......@@ -208,9 +208,7 @@ data:
\\"\\"\\"
idccs = pd.read_csv(idcc_file, usecols=[\\"SIRET\\", \\"IDCC\\"]).rename(
columns={\\"SIRET\\": \\"siret\\", \\"IDCC\\": \\"idcc\\"})
# drop unknown
unknown = idccs[\\"idcc\\"] == 9999
idccs = idccs.drop(idccs[unknown].index)
return idccs
......
......@@ -208,9 +208,7 @@ data:
\\"\\"\\"
idccs = pd.read_csv(idcc_file, usecols=[\\"SIRET\\", \\"IDCC\\"]).rename(
columns={\\"SIRET\\": \\"siret\\", \\"IDCC\\": \\"idcc\\"})
# drop unknown
unknown = idccs[\\"idcc\\"] == 9999
idccs = idccs.drop(idccs[unknown].index)
return idccs
......
......@@ -5,7 +5,7 @@
"private": true,
"main": "index.js",
"scripts": {
"dev": "node -r ts-node/register src/index.ts",
"dev": "nodemon --watch 'src/**' --ext 'ts' --ignore '__tests__' --exec 'ts-node src/index.ts'",
"build": "ncc build src/index.ts -o dist",
"start": "node dist/index.js",
"test": "jest"
......@@ -28,6 +28,7 @@
"@types/supertest": "^2.0.11",
"@vercel/ncc": "^0.28.3",
"jest": "^26.6.3",
"nodemon": "^2.0.7",
"prettier": "^2.2.1",
"supertest": "^6.1.3",
"ts-jest": "^26.5.5",
......
......@@ -159,7 +159,10 @@ export const entrepriseSearchBody = (
query: {
bool: {
filter: onlyWithConvention
? [{ term: { withIdcc: onlyWithConvention } }]
? [
{ term: { withIdcc: onlyWithConvention } },
{ range: { "idcc.number": { lt: 5001 } } },
]
: undefined,
must: [
{
......
......@@ -19,7 +19,7 @@ router.get(`${API_PREFIX}/search`, async (ctx) => {
query as string,
address as string,
parseInt(limit as string),
true,
false,
true
);
ctx.body = { entreprises };
......
This diff is collapsed.
......@@ -114,9 +114,7 @@ def read_idcc(idcc_file):
"""
idccs = pd.read_csv(idcc_file, usecols=["SIRET", "IDCC"]).rename(
columns={"SIRET": "siret", "IDCC": "idcc"})
# drop unknown
unknown = idccs["idcc"] == 9999
idccs = idccs.drop(idccs[unknown].index)
return idccs
......
This diff is collapsed.
......@@ -5,8 +5,9 @@
"main": "index.js",
"private": true,
"scripts": {
"start": "node --max_old_space_size=8192 -r ts-node/register src/index.ts",
"build": "NODE_OPTIONS='--max-old-space-size=8192' ncc build src/index.ts -o dist",
"dev": "node dist/index.js",
"start": "node -r ts-node/register src/index.ts",
"build": "ncc build src/index.ts -o dist",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
......
......@@ -169,7 +169,7 @@ export const add = async (enterprises: Enterprise[], indexName: string) => {
const batches = [];
let i = 0;
const batchSize = 100;
const batchSize = 500;
while (i < enterprises.length) {
batches.push(enterprises.slice(i, (i += batchSize)));
}
......
import { codesNaf } from "./naf";
import { getAgreements } from "@socialgouv/kali-data";
import agreements from "@socialgouv/kali-data/data/index.json";
const agreements = getAgreements();
const ccMap = new Map(agreements.map((agg) => [agg.num, agg]));
const idccs = [...ccMap.keys()];
......@@ -71,7 +70,14 @@ export const mappings = {
denominationUsuelleEtablissement: { type: "keyword" },
withIdcc: { type: "boolean" },
idcc: { type: "keyword" },
idcc: {
type: "keyword",
fields: {
number: {
type: "integer",
},
},
},
cp: { type: "keyword" },
......@@ -100,7 +106,7 @@ export const mapEnterprise = (enterprise: Enterprise) => {
// ranking feature cannot be 0
if (
!Number.parseFloat(
(enterprise.trancheEffectifsUniteLegale as unknown) as string
enterprise.trancheEffectifsUniteLegale as unknown as string
)
) {
enterprise.trancheEffectifsUniteLegale = 0.1;
......@@ -132,8 +138,8 @@ export const mapEnterprise = (enterprise: Enterprise) => {
enterprise.activitePrincipaleEtablissement,
enterprise.activitePrincipaleUniteLegale,
]
.map((c) => c.replace(".", ""))
.find((s) => s);
.map((code) => code.replace(/\w$/, ""))
.find((s) => !s.startsWith("00.00")); // 00.00Z is a temporary code
const activitePrincipale =
codeActivitePrincipale !== undefined
......@@ -145,11 +151,9 @@ export const mapEnterprise = (enterprise: Enterprise) => {
: undefined;
const withIdcc =
(enterprise.idcc !== undefined &&
enterprise.idcc !== null &&
enterprise.idcc !== "" &&
(enterprise.idcc &&
parseInt(enterprise.idcc) !== 0 &&
enterprise.idcc !== "0") ||
parseInt(enterprise.idcc) !== 9999) ||
false;
return {
......
import allCodesNaf from "@socialgouv/codes-naf";
const formatCodeNaf = (code: string) => code.replace(".", "") + "Z";
const codesEntries: [string, string][] = allCodesNaf.map(({ id, label }) => [
formatCodeNaf(id),
id,
label,
]);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment