Unverified Commit 2c4a4982 authored by Julien Bouquillon's avatar Julien Bouquillon 🐫 Committed by GitHub
Browse files

feat: use @socialgouv/codes-naf fix #5 (#23)

* feat: use @socialgouv/codes-naf fix #5

* fix(index): ensure docs are created before unaliasing
parent f8ac223a
......@@ -261,7 +261,7 @@ metadata:
apiVersion: batch/v1
kind: Job
metadata:
name: update-index
name: update-index-ccff889
annotations:
app.gitlab.com/app: socialgouv-recherche-entreprises
app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2
......
......@@ -261,7 +261,7 @@ metadata:
apiVersion: batch/v1
kind: Job
metadata:
name: update-index
name: update-index-ccff889
annotations:
app.gitlab.com/app: socialgouv-recherche-entreprises
app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2
......
......@@ -259,7 +259,7 @@ metadata:
apiVersion: batch/v1
kind: Job
metadata:
name: update-index
name: update-index-ccff889
annotations:
app.gitlab.com/app: socialgouv-recherche-entreprises
app.gitlab.com/env: recherche-entreprises-209-gitlab3-dev2
......
......@@ -185,7 +185,7 @@ const configMap = new ConfigMap({
// create the final job
const job = new Job({
metadata: {
name: "update-index",
name: `update-index-${process.env.CI_COMMIT_SHORT_SHA}`,
},
spec: {
backoffLimit: 3,
......
......@@ -3,6 +3,7 @@
"version": "1.0.0",
"description": "",
"main": "index.js",
"private": true,
"scripts": {
"start": "node --max_old_space_size=8192 -r ts-node/register src/index.ts",
"build": "NODE_OPTIONS='--max-old-space-size=8192' ncc build src/index.ts -o dist",
......@@ -10,10 +11,11 @@
},
"keywords": [],
"author": "",
"license": "ISC",
"license": "Apache-2.0",
"dependencies": {
"@babel/runtime": "^7.12.13",
"@elastic/elasticsearch": "^7.12.0",
"@socialgouv/codes-naf": "^1.0.5",
"@socialgouv/kali-data": "^2.61.0",
"fast-csv": "^4.3.6",
"p-all": "^3.0.0",
......
......@@ -179,3 +179,8 @@ export const add = async (enterprises: Enterprise[], indexName: string) => {
{ concurrency: 5 }
);
};
export const getDocsCount = async (indexName: string): Promise<number> => {
const stats = await esClient.indices.stats({index: indexName});
return stats.body._all.primaries.docs.count;
}
\ No newline at end of file
import * as fs from "fs";
import * as path from "path";
import * as csv from "fast-csv";
import { add, createIndex, updateAlias, deleteOldIndices } from "./elastic";
import {
add,
createIndex,
updateAlias,
deleteOldIndices,
getDocsCount,
} from "./elastic";
import { Enterprise } from "./enterprise";
const ASSEMBLY_FILE = process.env.ASSEMBLY_FILE || "../output/assembly.csv";
......@@ -39,6 +45,15 @@ if (require.main === module) {
// use elastic alias feature to prevent downtimes
createIndex().then(async (indexName) =>
insertEntreprises(indexName)
.then(async () => {
// ensure we have some data
const docsCount = await getDocsCount(indexName);
if (!docsCount) {
throw new Error(`No document created in index ${indexName}, skip aliasing`);
} else {
console.log(`Created ${docsCount} documents in index ${indexName}`);
}
})
.then(() => updateAlias(indexName))
.then(() => deleteOldIndices(indexName))
);
......
This diff is collapsed.
......@@ -6,7 +6,7 @@
"lib": ["es2020", "dom"],
"module": "commonjs",
"target": "es2020",
"moduleResolution": "node",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
......
......@@ -66,6 +66,11 @@
lodash.isundefined "^3.0.1"
lodash.uniq "^4.5.0"
"@socialgouv/codes-naf@^1.0.5":
version "1.0.5"
resolved "https://registry.yarnpkg.com/@socialgouv/codes-naf/-/codes-naf-1.0.5.tgz#fa64c00b6299d0d41e1e88a4422ca72cb688e08c"
integrity sha512-unWkbh6qcG/9B+Oa2LsES6+xFYRK+KMyfa+NNszaEE7JYxSAibRnICTEkRUjriufrRXREV0fiB3MR0ytk7uAMA==
"@socialgouv/kali-data@^2.61.0":
version "2.61.0"
resolved "https://registry.yarnpkg.com/@socialgouv/kali-data/-/kali-data-2.61.0.tgz#8e83bca951c99a0cf000c5e8298d309b378d6bcc"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment