Skip to content
Snippets Groups Projects
Select Git revision
  • fff632ac2fed11e8f7902c70f29340427fffcb31
  • master default protected
  • ajout_ppa_rsa_budgetaire
  • 365-ouvrir-l-onglet-employeur-ou-taxes-carburant-quand-c-est-le-cas-pour-un-dispositif
  • 381-pb-affichage-labels-des-parametres-sur-plus-de-3-lignes
  • ajoute-duplicate-aide-logement
  • poc_castype_ia
  • parametres-editables-budget
  • ui-parametres
  • 366-signe-a-cote-du-droit-en-vigueur-sur-l-ui-pour-indiquer-que-la-reforme-a-eu-lieu-mais-qu-elle-n
  • 355-les-dispositifs-prestations-sociales-du-graphique-se-cachent-montrent-en-meme-temps-2
  • 358-les-variables-dont-le-montant-est-nul-apparaissent-en-bleu-et-non-cliquables
  • 356-ajuster-la-largeur-sur-les-graphiques-budgetaires
  • incoherence_cas_type_0
  • fix-ui-suppression-tranches-baremes
  • ajout-agregat-cehr-version-plf
  • impact_carbone
  • xlsx
  • header_revamp
  • 270-concevoir-la-page-d-accueil-leximpact
  • 219-conversion-des-montants-min-et-max-de-l-axe-des-x-en-smic
  • 0.0.1123
  • 0.0.1122
  • 0.0.1121
  • 0.0.1120
  • 0.0.1119
  • 0.0.1118
  • 0.0.1117
  • 0.0.1116
  • 0.0.1115
  • 0.0.1114
  • 0.0.1113
  • 0.0.1112
  • 0.0.1111
  • 0.0.1110
  • 0.0.1109
  • 0.0.1108
  • 0.0.1107
  • 0.0.1106
  • 0.0.1105
  • 0.0.1104
41 results

generate_openfisca_tests_yaml.ts

Blame
  • generate_openfisca_tests_yaml.ts 19.53 KiB
    import type { Situation } from "@openfisca/json-model"
    import commandLineArgs from "command-line-args"
    import fs from "fs-extra"
    import path from "path"
    import YAML from "js-yaml"
    import testCasesCoreUnknown from "@leximpact/socio-fiscal-openfisca-json/test_cases.json"
    import {
      summaryCalculatedVariablesName,
      otherCalculatedVariablesName,
      variableSummaryByName,
    } from "$lib/variables"
    import { nonVirtualVariablesName } from "$lib/decompositions"
    import type { EntityByKey, Entity, GroupEntity } from "@openfisca/json-model"
    import { entityByKey } from "$lib/entities"
    import { spawn } from "child_process"
    
    type JsonObject = { [key: string]: any }
    
    const optionsDefinitions = [
      {
        alias: "s",
        help: "don't log anything",
        name: "silent",
        type: Boolean,
      },
      {
        alias: "v",
        help: "verbose logs",
        name: "verbose",
        type: Boolean,
      },
      {
        alias: "y",
        help: "Year to ask as simulation output",
        name: "year",
        type: Number,
      },
      {
        alias: "t",
        help: "Single Test-Case to run",
        name: "testcase",
        type: String,
      },
      {
        defaultOption: true,
        help: "Directory to write OpenFisca test YAML files",
        name: "outdir",
        type: String,
      },
    ]
    const options = commandLineArgs(optionsDefinitions)
    
    async function fetchWithRetries(
      url: string,
      options: RequestInit,
      maxRetries = 5,
      delay = 10000,
    ) {
      for (let attempt = 1; attempt <= maxRetries; attempt++) {
        try {
          const response = await fetch(url, options)
    
          if (response.ok) {
            return response
          }
    
          console.warn(
            `Calling Openfisca API. Attempt ${attempt} : Failed with status ${response.status}`,
          )
        } catch (error) {
          console.error(`Error on attempt ${attempt} :`, error)
        }
    
        if (attempt < maxRetries) {
          console.log(`New attempt in ${delay / 1000}s...`)
          await new Promise((res) => setTimeout(res, delay))
        }
      }
    
      throw new Error(`Failed calling OpenFisca API after ${maxRetries} attempts`)
    }
    
    function removeSpacesFromKeys(
      obj: JsonObject,
      replacedKeys: { [key: string]: string },
    ): JsonObject {
      const newObj: JsonObject = {}
    
      for (const key in obj) {
        if (obj.hasOwnProperty(key)) {
          let newKey: string | number = key
            .normalize("NFD")
            .replace(/[\u0300-\u036f]/g, "")
            .toLowerCase()
            .replace(/ |’|°/g, "_")
    
          if (newKey !== key) {
            replacedKeys[key] = newKey
          }
    
          const value = obj[key]
    
          if (
            typeof value === "object" &&
            value !== null &&
            !Array.isArray(value)
          ) {
            newObj[newKey] = removeSpacesFromKeys(value, replacedKeys)
          } else {
            newObj[newKey] = value
          }
        }
      }
    
      return newObj
    }
    
    function replaceValuesFromReplacedKeys(
      obj: JsonObject,
      replacedKeys: { [key: string]: string },
    ): JsonObject {
      const newObj: JsonObject = {}
    
      for (const key in obj) {
        if (obj.hasOwnProperty(key)) {
          const value = obj[key]
    
          if (
            typeof value === "object" &&
            value !== null &&
            !Array.isArray(value)
          ) {
            newObj[key] = replaceValuesFromReplacedKeys(value, replacedKeys)
          } else if (
            typeof value === "string" &&
            replacedKeys.hasOwnProperty(value)
          ) {
            newObj[key] = replacedKeys[value]
          } else if (Array.isArray(value)) {
            const newArray = []
            for (const arrayValue of value) {
              if (
                typeof arrayValue === "string" &&
                replacedKeys.hasOwnProperty(arrayValue)
              ) {
                newArray.push(replacedKeys[arrayValue])
              } else {
                newArray.push(arrayValue)
              }
            }
            newObj[key] = newArray
          } else {
            newObj[key] = value
          }
        }
      }
    
      return newObj
    }
    
    function removeZeroValues(obj: any): any {
      if (typeof obj !== "object" || obj === null) {
        return obj
      }
    
      if (Array.isArray(obj)) {
        return obj.map(removeZeroValues)
      }
    
      const newObj: any = {}
    
      for (const key in obj) {
        if (obj.hasOwnProperty(key)) {
          const value = obj[key]
    
          if (typeof value === "object" && value !== null) {
            if (Object.values(value).every((v) => v === 0 || v === null)) {
              continue
            }
    
            const cleanedValue = removeZeroValues(value)
            if (Object.keys(cleanedValue).length > 0) {
              newObj[key] = cleanedValue
            }
          } else {
            newObj[key] = value
          }
        }
      }
    
      return newObj
    }
    
    function cleanSimulatedJson(jsonData: any): any {
      const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
    
      for (const section of sections) {
        if (jsonData[section]) {
          jsonData[section] = Object.fromEntries(
            Object.entries(jsonData[section]).map(([key, value]) => [
              key,
              removeZeroValues(value),
            ]),
          )
        }
      }
    
      return jsonData
    }
    
    function splitVariablesByCategory(
      section: string,
      entityKey: string,
      entityData: JsonObject,
      initialTestCase: Situation,
    ) {
      const input: JsonObject = {}
      const output: JsonObject = {}
    
      for (const [variable, value] of Object.entries(entityData)) {
        const replacedKeys: { [key: string]: string } = {}
    
        initialTestCase = removeSpacesFromKeys(initialTestCase, replacedKeys)
        if (
          initialTestCase[section][entityKey] !== undefined &&
          initialTestCase[section][entityKey][variable] === undefined
        ) {
          if (
            variableSummaryByName[variable].value_type === "Enum" &&
            variableSummaryByName[variable].possible_values !== undefined
          ) {
            // Replace Enum values by their index in possible_values
            const newValue: Record<string, number> = {}
            for (const [period, periodValue] of Object.entries(value)) {
              newValue[period] = Object.keys(
                variableSummaryByName[variable].possible_values,
              ).indexOf(periodValue as string)
            }
            output[variable] = newValue
          } else {
            output[variable] = value
          }
        } else {
          input[variable] = value
        }
      }
      return { input, output }
    }
    
    function processSection(
      section: string,
      sectionData: JsonObject,
      initialTestCase: Situation,
    ) {
      const inputSection: JsonObject = {}
      const outputSection: JsonObject = {}
    
      for (const [entityKey, entityData] of Object.entries(sectionData)) {
        const { input, output } = splitVariablesByCategory(
          section,
          entityKey,
          entityData,
          initialTestCase,
        )
    
        if (Object.keys(input).length > 0) {
          inputSection[entityKey] = input
        }
        if (Object.keys(output).length > 0) {
          outputSection[entityKey] = output
        }
      }
    
      return { input: inputSection, output: outputSection }
    }
    
    function buildYamlOutput(
      name: string,
      year: string,
      simulatedTestCase: JsonObject,
      testCase: Situation,
    ) {
      const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
      const jsonForYamlOutput: {
        name: string
        period: string
        max_spiral_loops: number
        input: JsonObject
        output: JsonObject
      } = {
        name: name,
        period: year,
        max_spiral_loops: 4,
        input: {},
        output: {},
      }
    
      for (const section of sections) {
        if (simulatedTestCase[section]) {
          const { input, output } = processSection(
            section,
            simulatedTestCase[section],
            testCase,
          )
    
          if (Object.keys(input).length > 0) {
            jsonForYamlOutput.input[section] = input
          }
          if (Object.keys(output).length > 0) {
            jsonForYamlOutput.output[section] = output
          }
        }
      }
    
      return jsonForYamlOutput
    }
    
    function mensualiseVariable(obj: any, variable: string) {
      if (!obj || typeof obj !== "object") return obj
    
      const annuelValues = obj
      const monthlyValues: any = {}
    
      for (const year in annuelValues) {
        if (typeof annuelValues[year] === "number") {
          const monthlyAmount =
            variableSummaryByName[variable]["set_input"] ===
            "set_input_divide_by_period"
              ? parseFloat((annuelValues[year] / 12).toFixed(2))
              : annuelValues[year]
          for (let month = 1; month <= 12; month++) {
            const monthStr = month < 10 ? `0${month}` : `${month}`
            monthlyValues[`${year}-${monthStr}`] = monthlyAmount
          }
        } else {
          monthlyValues[year] = annuelValues[year]
        }
      }
    
      obj = monthlyValues
      return obj
    }
    
    function getEntityKeyFromKeyPlural(
      keyPlural: string,
      entityByKey: EntityByKey,
    ): string | null {
      for (const entity of Object.values(entityByKey)) {
        if (entity.key_plural === keyPlural) {
          return entity.key
        }
      }
      return null
    }
    
    function isGroupEntity(entity: Entity): entity is GroupEntity {
      return (entity as GroupEntity).roles !== undefined
    }
    
    function isKeyOrPlural(str: string): boolean {
      const keys = Object.keys(entityByKey)
    
      for (const key of keys) {
        const entity = entityByKey[key]
        if (entity.key === str || entity.key_plural === str) {
          return true
        }
        if (isGroupEntity(entity) && entity.roles) {
          for (const role of entity.roles) {
            if (role.key === str || role.key_plural === str) {
              return true
            }
            if (role.subroles) {
              for (const subrole of role.subroles) {
                if (subrole.key === str || subrole.key_plural === str) {
                  return true
                }
              }
            }
          }
        }
      }
    
      return false
    }
    
    function replaceVariableValue(
      jsonData: any,
      variableName: string,
      currentValue: any,
      newValue: any,
      period: string,
    ): any {
      const entities = ["familles", "foyers_fiscaux", "menages", "individus"]
    
      for (const entity of entities) {
        if (!jsonData.output[entity]) continue
    
        for (const entityKey in jsonData.output[entity]) {
          const entityData = jsonData.output[entity][entityKey]
    
          if (entityData.hasOwnProperty(variableName)) {
            const variableData = entityData[variableName]
            if (typeof variableData === "object") {
              if (
                variableData.hasOwnProperty(period) &&
                Math.trunc(Number(variableData[period])) ===
                  Math.trunc(Number(currentValue))
              ) {
                console.info(
                  `Replacing ${variableName} (${period}): ${variableData[period]} → ${newValue}`,
                )
                variableData[period] = Number(newValue)
              } else {
                console.log(
                  "Warning : rounded value",
                  Math.trunc(Number(currentValue)),
                  "not found for",
                  variableName,
                  "\nThe cause can be multiple instance of the same variable for different entities... or bug",
                )
              }
            }
          }
        }
      }
    
      return
    }
    
    async function runOpenFiscaTest(
      openFiscaVenvPath: string,
      yamlFilePath: string,
    ): Promise<any> {
      return new Promise<void>((resolve, reject) => {
        const openfiscaExecutable = path.join(openFiscaVenvPath, "bin", "openfisca")
        const args = ["test", "--country-package", "openfisca_france", yamlFilePath]
        const env = {
          ...process.env,
          VIRTUAL_ENV: openFiscaVenvPath,
          PATH: `${path.join(openFiscaVenvPath, "bin")}:${process.env.PATH}`,
          PYTHONPATH: path.join(
            openFiscaVenvPath,
            "lib",
            "python3.12",
            "site-packages",
          ),
        }
        let testOutput = ""
    
        const pythonProcess = spawn(openfiscaExecutable, args, {
          stdio: ["inherit", "pipe", "pipe"],
          env,
        })
    
        pythonProcess.stdout.on("data", (data) => {
          testOutput += data.toString()
        })
    
        pythonProcess.stderr.on("data", (data) => {
          console.error(`stderr: ${data}`)
          reject(data)
        })
    
        pythonProcess.on("close", (code) => {
          let returned = undefined as any
    
          if (code !== 0) {
            returned = { passed: false }
    
            const testOutputLines = testOutput.split("\n")
    
            const regex =
              /(?<variable>[a-zA-Z0-9_]+)@(?<period>\d{4}(?:-\d{2})?):\s+(?<calculated>-?\d+(?:\.\d+)?)\s+differs\s+from\s+(?<expected>-?\d+(?:\.\d+)?)/
            testOutputLines.forEach((line) => {
              const match = line.match(regex)
              if (match && match.groups) {
                returned = { ...returned, ...match.groups }
              }
            })
            if (returned.variable !== undefined) {
              resolve(returned)
            } else
              reject(
                "Erreur de récupération des valeurs lors d'un appel à openfisca test",
              )
          } else {
            returned = { passed: true }
            resolve(returned)
          }
        })
      })
    }
    
    async function main() {
      console.info("Initializing...")
    
      const personsEntityKey = Object.entries(entityByKey)
        .filter(([, entity]) => entity.is_person)
        .map(([key]) => key)[0]
    
      const year = options.year
      const outdir = options.outdir
      const singleTestCase = options.testcase
    
      const variablesFromDecompositions = [
        ...summaryCalculatedVariablesName,
        ...otherCalculatedVariablesName,
        ...nonVirtualVariablesName,
      ]
    
      const openFiscaVariablesListApiCall = await fetch(
        "https://api.fr.openfisca.org/latest/variables",
        {
          method: "GET",
        },
      )
      if (!openFiscaVariablesListApiCall.ok) {
        console.error(
          `Error ${
            openFiscaVariablesListApiCall.status
          } while calling Openfisca France API`,
        )
        return
      }
    
      const openFiscaVariablesList = await openFiscaVariablesListApiCall.json()
    
      const variablesToCalculate = variablesFromDecompositions.filter(
        (variable: string) => openFiscaVariablesList.hasOwnProperty(variable),
      )
    
      const testCasesCore = testCasesCoreUnknown as unknown as Situation[]
      const testCaseCount = testCasesCore.length
    
      let processedCounter = 0
    
      for (const testCase of testCasesCore) {
        if (singleTestCase !== undefined && testCase.id !== singleTestCase) {
          continue
        }
    
        console.info("Processing ", testCase.id)
        processedCounter++
        let testCaseContainsMissingVariable = false
        let jsonForApiCall: JsonObject = {}
    
        jsonForApiCall.input = {}
    
        const replacedKeys: { [key: string]: string } = {}
    
        const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
    
        // Mensualise monthly input variables
        for (const section of sections) {
          if (testCase[section] !== undefined) {
            for (const entityKey in testCase[section]) {
              for (const variable in testCase[section][entityKey]) {
                if (
                  variableSummaryByName.hasOwnProperty(variable) &&
                  variableSummaryByName[variable].definition_period === "month"
                ) {
                  testCase[section][entityKey][variable] = mensualiseVariable(
                    testCase[section][entityKey][variable],
                    variable,
                  )
                } else if (
                  !openFiscaVariablesList.hasOwnProperty(variable) &&
                  !isKeyOrPlural(variable)
                ) {
                  console.info(
                    "variable",
                    variable,
                    "does not exists in openFisca-France.",
                  )
                  testCaseContainsMissingVariable = true
                }
              }
            }
          }
        }
    
        if (testCaseContainsMissingVariable) {
          console.info("Ignoring test case", testCase.id)
          continue
        }
    
        for (const section of sections) {
          if (testCase[section] !== undefined) {
            jsonForApiCall.input[section] = removeSpacesFromKeys(
              testCase[section],
              replacedKeys,
            )
            for (const entityKey in jsonForApiCall.input[section]) {
              for (const variable of variablesToCalculate) {
                if (variableSummaryByName[variable] !== undefined) {
                  if (
                    variableSummaryByName[variable].entity ===
                      getEntityKeyFromKeyPlural(section, entityByKey) &&
                    jsonForApiCall.input[section][entityKey][variable] === undefined
                  ) {
                    switch (variableSummaryByName[variable].definition_period) {
                      case "year": {
                        jsonForApiCall.input[section][entityKey][variable] = {
                          [year]: null,
                        }
                        break
                      }
                      case "month": {
                        const monthVariableSection = {}
                        for (let month = 1; month <= 12; month++) {
                          const monthString = month < 10 ? "0" + month : month
                          const monthPeriod = year + "-" + monthString
                          Object.assign(monthVariableSection, {
                            [monthPeriod]: null,
                          })
                        }
                        jsonForApiCall.input[section][entityKey][variable] =
                          monthVariableSection
                      }
                    }
                  }
                } else {
                  console.log(
                    "cette variable n'existe pas dans summary : ",
                    variable,
                  )
                }
              }
            }
          }
        }
    
        jsonForApiCall = replaceValuesFromReplacedKeys(jsonForApiCall, replacedKeys)
        // await fs.writeFile(
        //   path.join(".", testCase.id + ".json"),
        //   JSON.stringify(jsonForApiCall.input, null, 2),
        // )
        console.info("Launching simulation on OpenFisca France API...")
        const openFiscaApiUrl = "https://api.fr.openfisca.org/latest/calculate"
        // const openFiscaApiUrl = "http://localhost:5000/calculate"
        const openFiscaApiOptions: RequestInit = {
          body: JSON.stringify(jsonForApiCall.input),
          headers: {
            Accept: "application/json",
            "Content-Type": "application/json; charset=utf-8",
          },
          method: "POST",
        }
        let simulatedTestCaseFromOpenFiscaApi: Response | null = null
    
        try {
          simulatedTestCaseFromOpenFiscaApi = await fetchWithRetries(
            openFiscaApiUrl,
            openFiscaApiOptions,
          )
    
          if (!simulatedTestCaseFromOpenFiscaApi.ok) {
            console.error(
              `Giving up calling openFisca API because of status ${simulatedTestCaseFromOpenFiscaApi.status}`,
            )
            return
          }
        } catch (error) {
          console.error("Error : ", error)
        }
        console.info("Post-processing...")
        const simulatedTestCase = cleanSimulatedJson(
          await simulatedTestCaseFromOpenFiscaApi!.json(),
        )
    
        const jsonForYamlOutput = buildYamlOutput(
          testCase.id!,
          year,
          simulatedTestCase,
          testCase,
        )
    
        let ok = false
        while (!ok) {
          const yamlOutput = YAML.dump(jsonForYamlOutput, {
            noCompatMode: true,
            noRefs: true,
          })
          const cleanedYaml = yamlOutput.replace(/'(\d{4})':/g, "$1:")
    
          await fs.writeFile(path.join(outdir, testCase.id + ".yml"), cleanedYaml)
    
          const venvPath = "/home/cafe/apps/openfisca-france/.venv"
    
          try {
            const openFiscaTestResult = await runOpenFiscaTest(
              venvPath,
              path.join(outdir, testCase.id + ".yml"),
            )
            if (!openFiscaTestResult.passed) {
              if (
                openFiscaTestResult.variable === undefined ||
                openFiscaTestResult.expected === undefined ||
                openFiscaTestResult.calculated === undefined ||
                openFiscaTestResult.period === undefined
              ) {
                throw new Error("Error retrieving values to update")
              }
              replaceVariableValue(
                jsonForYamlOutput,
                openFiscaTestResult.variable,
                openFiscaTestResult.expected,
                openFiscaTestResult.calculated,
                openFiscaTestResult.period,
              )
            } else {
              ok = true
              console.info("YAML file written :", outdir, testCase.id + ".yml")
            }
          } catch (error) {
            console.error(error)
          }
        }
        console.info("Processed test-case", processedCounter, "over", testCaseCount)
      }
    }
    
    main()
      .then(() => {
        process.exit(0)
      })
      .catch((error: unknown) => {
        console.error(error)
        process.exit(1)
      })