update generate test yaml script to use openfisca test for updating expected values

src/scripts/generate_openfisca_tests_yaml.ts

@@ -12,6 +12,7 @@ import {
 import { nonVirtualVariablesName } from "$lib/decompositions"
 import type { EntityByKey, Entity, GroupEntity } from "@openfisca/json-model"
 import { entityByKey } from "$lib/entities"
+import { spawn } from "child_process"
 
 type JsonObject = { [key: string]: any }
 
@@ -383,6 +384,114 @@ function isKeyOrPlural(str: string): boolean {
   return false
 }
 
+function replaceVariableValue(
+  jsonData: any,
+  variableName: string,
+  currentValue: any,
+  newValue: any,
+  period: string,
+): any {
+  const entities = ["familles", "foyers_fiscaux", "menages", "individus"]
+
+  for (const entity of entities) {
+    if (!jsonData.output[entity]) continue
+
+    for (const entityKey in jsonData.output[entity]) {
+      const entityData = jsonData.output[entity][entityKey]
+
+      if (entityData.hasOwnProperty(variableName)) {
+        const variableData = entityData[variableName]
+        if (typeof variableData === "object") {
+          if (
+            variableData.hasOwnProperty(period) &&
+            Math.trunc(Number(variableData[period])) ===
+              Math.trunc(Number(currentValue))
+          ) {
+            console.info(
+              `Replacing ${variableName} (${period}): ${variableData[period]} → ${newValue}`,
+            )
+            variableData[period] = Number(newValue)
+          } else {
+            console.log(
+              "Warning : rounded value",
+              Math.trunc(Number(currentValue)),
+              "not found for",
+              variableName,
+              "\nThe cause can be multiple instance of the same variable for different entities... or bug",
+            )
+          }
+        }
+      }
+    }
+  }
+
+  return
+}
+
+async function runOpenFiscaTest(
+  openFiscaVenvPath: string,
+  yamlFilePath: string,
+): Promise<any> {
+  return new Promise<void>((resolve, reject) => {
+    const openfiscaExecutable = path.join(openFiscaVenvPath, "bin", "openfisca")
+    const args = ["test", "--country-package", "openfisca_france", yamlFilePath]
+    const env = {
+      ...process.env,
+      VIRTUAL_ENV: openFiscaVenvPath,
+      PATH: `${path.join(openFiscaVenvPath, "bin")}:${process.env.PATH}`,
+      PYTHONPATH: path.join(
+        openFiscaVenvPath,
+        "lib",
+        "python3.12",
+        "site-packages",
+      ),
+    }
+    let testOutput = ""
+
+    const pythonProcess = spawn(openfiscaExecutable, args, {
+      stdio: ["inherit", "pipe", "pipe"],
+      env,
+    })
+
+    pythonProcess.stdout.on("data", (data) => {
+      testOutput += data.toString()
+    })
+
+    pythonProcess.stderr.on("data", (data) => {
+      console.error(`stderr: ${data}`)
+      reject(data)
+    })
+
+    pythonProcess.on("close", (code) => {
+      let returned = undefined as any
+
+      if (code !== 0) {
+        returned = { passed: false }
+
+        const testOutputLines = testOutput.split("\n")
+
+        const regex =
+          /(?<variable>[a-zA-Z0-9_]+)@(?<period>\d{4}(?:-\d{2})?):\s+(?<calculated>-?\d+(?:\.\d+)?)\s+differs\s+from\s+(?<expected>-?\d+(?:\.\d+)?)/
+        testOutputLines.forEach((line) => {
+          const match = line.match(regex)
+          if (match && match.groups) {
+            returned = { ...returned, ...match.groups }
+          }
+        })
+        if (returned.variable !== undefined) {
+          resolve(returned)
+        } else
+          reject(
+            "Erreur de récupération des valeurs lors d'un appel à openfisca test",
+          )
+      } else {
+        returned = { passed: true }
+        resolve(returned)
+      }
+    })
+  })
+}
+
 async function main() {
   console.info("Initializing...")
 
@@ -459,7 +568,7 @@ async function main() {
               !openFiscaVariablesList.hasOwnProperty(variable) &&
               !isKeyOrPlural(variable)
             ) {
-              console.warn(
+              console.info(
                 "variable",
                 variable,
                 "does not exists in openFisca-France.",
@@ -472,7 +581,7 @@ async function main() {
     }
 
     if (testCaseContainsMissingVariable) {
-      console.warn("Ignoring test case", testCase.id)
+      console.info("Ignoring test case", testCase.id)
       continue
     }
 
@@ -567,16 +676,48 @@ async function main() {
       testCase,
     )
 
+    let ok = false
+    while (!ok) {
       const yamlOutput = YAML.dump(jsonForYamlOutput, {
         noCompatMode: true,
         noRefs: true,
       })
-
-    // Remove quotes around year keys
       const cleanedYaml = yamlOutput.replace(/'(\d{4})':/g, "$1:")
+
       await fs.writeFile(path.join(outdir, testCase.id + ".yml"), cleanedYaml)
+
+      const venvPath = "/home/cafe/apps/openfisca-france/.venv"
+
+      try {
+        const openFiscaTestResult = await runOpenFiscaTest(
+          venvPath,
+          path.join(outdir, testCase.id + ".yml"),
+        )
+        if (!openFiscaTestResult.passed) {
+          if (
+            openFiscaTestResult.variable === undefined ||
+            openFiscaTestResult.expected === undefined ||
+            openFiscaTestResult.calculated === undefined ||
+            openFiscaTestResult.period === undefined
+          ) {
+            throw new Error("Error retrieving values to update")
+          }
+          replaceVariableValue(
+            jsonForYamlOutput,
+            openFiscaTestResult.variable,
+            openFiscaTestResult.expected,
+            openFiscaTestResult.calculated,
+            openFiscaTestResult.period,
+          )
+        } else {
+          ok = true
           console.info("YAML file written :", outdir, testCase.id + ".yml")
-    console.info(processedCounter, "files processed over", testCaseCount)
+        }
+      } catch (error) {
+        console.error(error)
+      }
+    }
+    console.info("Processed test-case", processedCounter, "over", testCaseCount)
   }
 }