diff --git a/src/scripts/generate_openfisca_tests_yaml.ts b/src/scripts/generate_openfisca_tests_yaml.ts index afa393f6054e694fb831e8f3227381d24bc33595..4e7da3087c7a7cfb345e0530f5d6ae0bc4afad80 100644 --- a/src/scripts/generate_openfisca_tests_yaml.ts +++ b/src/scripts/generate_openfisca_tests_yaml.ts @@ -12,6 +12,7 @@ import { import { nonVirtualVariablesName } from "$lib/decompositions" import type { EntityByKey, Entity, GroupEntity } from "@openfisca/json-model" import { entityByKey } from "$lib/entities" +import { spawn } from "child_process" type JsonObject = { [key: string]: any } @@ -383,6 +384,114 @@ function isKeyOrPlural(str: string): boolean { return false } +function replaceVariableValue( + jsonData: any, + variableName: string, + currentValue: any, + newValue: any, + period: string, +): any { + const entities = ["familles", "foyers_fiscaux", "menages", "individus"] + + for (const entity of entities) { + if (!jsonData.output[entity]) continue + + for (const entityKey in jsonData.output[entity]) { + const entityData = jsonData.output[entity][entityKey] + + if (entityData.hasOwnProperty(variableName)) { + const variableData = entityData[variableName] + if (typeof variableData === "object") { + if ( + variableData.hasOwnProperty(period) && + Math.trunc(Number(variableData[period])) === + Math.trunc(Number(currentValue)) + ) { + console.info( + `Replacing ${variableName} (${period}): ${variableData[period]} → ${newValue}`, + ) + variableData[period] = Number(newValue) + } else { + console.log( + "Warning : rounded value", + Math.trunc(Number(currentValue)), + "not found for", + variableName, + "\nThe cause can be multiple instance of the same variable for different entities... or bug", + ) + } + } + } + } + } + + return +} + +async function runOpenFiscaTest( + openFiscaVenvPath: string, + yamlFilePath: string, +): Promise<any> { + return new Promise<void>((resolve, reject) => { + const openfiscaExecutable = path.join(openFiscaVenvPath, "bin", "openfisca") + const args = ["test", "--country-package", "openfisca_france", yamlFilePath] + const env = { + ...process.env, + VIRTUAL_ENV: openFiscaVenvPath, + PATH: `${path.join(openFiscaVenvPath, "bin")}:${process.env.PATH}`, + PYTHONPATH: path.join( + openFiscaVenvPath, + "lib", + "python3.12", + "site-packages", + ), + } + let testOutput = "" + + const pythonProcess = spawn(openfiscaExecutable, args, { + stdio: ["inherit", "pipe", "pipe"], + env, + }) + + pythonProcess.stdout.on("data", (data) => { + testOutput += data.toString() + }) + + pythonProcess.stderr.on("data", (data) => { + console.error(`stderr: ${data}`) + reject(data) + }) + + pythonProcess.on("close", (code) => { + let returned = undefined as any + + if (code !== 0) { + returned = { passed: false } + + const testOutputLines = testOutput.split("\n") + + const regex = + /(?<variable>[a-zA-Z0-9_]+)@(?<period>\d{4}(?:-\d{2})?):\s+(?<calculated>-?\d+(?:\.\d+)?)\s+differs\s+from\s+(?<expected>-?\d+(?:\.\d+)?)/ + testOutputLines.forEach((line) => { + const match = line.match(regex) + if (match && match.groups) { + returned = { ...returned, ...match.groups } + } + }) + if (returned.variable !== undefined) { + resolve(returned) + } else + reject( + "Erreur de récupération des valeurs lors d'un appel à openfisca test", + ) + } else { + returned = { passed: true } + resolve(returned) + } + }) + }) +} + async function main() { console.info("Initializing...") @@ -459,7 +568,7 @@ async function main() { !openFiscaVariablesList.hasOwnProperty(variable) && !isKeyOrPlural(variable) ) { - console.warn( + console.info( "variable", variable, "does not exists in openFisca-France.", @@ -472,7 +581,7 @@ async function main() { } if (testCaseContainsMissingVariable) { - console.warn("Ignoring test case", testCase.id) + console.info("Ignoring test case", testCase.id) continue } @@ -567,16 +676,48 @@ async function main() { testCase, ) - const yamlOutput = YAML.dump(jsonForYamlOutput, { - noCompatMode: true, - noRefs: true, - }) + let ok = false + while (!ok) { + const yamlOutput = YAML.dump(jsonForYamlOutput, { + noCompatMode: true, + noRefs: true, + }) + const cleanedYaml = yamlOutput.replace(/'(\d{4})':/g, "$1:") + + await fs.writeFile(path.join(outdir, testCase.id + ".yml"), cleanedYaml) + + const venvPath = "/home/cafe/apps/openfisca-france/.venv" - // Remove quotes around year keys - const cleanedYaml = yamlOutput.replace(/'(\d{4})':/g, "$1:") - await fs.writeFile(path.join(outdir, testCase.id + ".yml"), cleanedYaml) - console.info("YAML file written :", outdir, testCase.id + ".yml") - console.info(processedCounter, "files processed over", testCaseCount) + try { + const openFiscaTestResult = await runOpenFiscaTest( + venvPath, + path.join(outdir, testCase.id + ".yml"), + ) + if (!openFiscaTestResult.passed) { + if ( + openFiscaTestResult.variable === undefined || + openFiscaTestResult.expected === undefined || + openFiscaTestResult.calculated === undefined || + openFiscaTestResult.period === undefined + ) { + throw new Error("Error retrieving values to update") + } + replaceVariableValue( + jsonForYamlOutput, + openFiscaTestResult.variable, + openFiscaTestResult.expected, + openFiscaTestResult.calculated, + openFiscaTestResult.period, + ) + } else { + ok = true + console.info("YAML file written :", outdir, testCase.id + ".yml") + } + } catch (error) { + console.error(error) + } + } + console.info("Processed test-case", processedCounter, "over", testCaseCount) } }