Select Git revision
generate_openfisca_tests_yaml.ts
-
David Smadja authoredDavid Smadja authored
generate_openfisca_tests_yaml.ts 19.53 KiB
import type { Situation } from "@openfisca/json-model"
import commandLineArgs from "command-line-args"
import fs from "fs-extra"
import path from "path"
import YAML from "js-yaml"
import testCasesCoreUnknown from "@leximpact/socio-fiscal-openfisca-json/test_cases.json"
import {
summaryCalculatedVariablesName,
otherCalculatedVariablesName,
variableSummaryByName,
} from "$lib/variables"
import { nonVirtualVariablesName } from "$lib/decompositions"
import type { EntityByKey, Entity, GroupEntity } from "@openfisca/json-model"
import { entityByKey } from "$lib/entities"
import { spawn } from "child_process"
type JsonObject = { [key: string]: any }
const optionsDefinitions = [
{
alias: "s",
help: "don't log anything",
name: "silent",
type: Boolean,
},
{
alias: "v",
help: "verbose logs",
name: "verbose",
type: Boolean,
},
{
alias: "y",
help: "Year to ask as simulation output",
name: "year",
type: Number,
},
{
alias: "t",
help: "Single Test-Case to run",
name: "testcase",
type: String,
},
{
defaultOption: true,
help: "Directory to write OpenFisca test YAML files",
name: "outdir",
type: String,
},
]
const options = commandLineArgs(optionsDefinitions)
async function fetchWithRetries(
url: string,
options: RequestInit,
maxRetries = 5,
delay = 10000,
) {
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
const response = await fetch(url, options)
if (response.ok) {
return response
}
console.warn(
`Calling Openfisca API. Attempt ${attempt} : Failed with status ${response.status}`,
)
} catch (error) {
console.error(`Error on attempt ${attempt} :`, error)
}
if (attempt < maxRetries) {
console.log(`New attempt in ${delay / 1000}s...`)
await new Promise((res) => setTimeout(res, delay))
}
}
throw new Error(`Failed calling OpenFisca API after ${maxRetries} attempts`)
}
function removeSpacesFromKeys(
obj: JsonObject,
replacedKeys: { [key: string]: string },
): JsonObject {
const newObj: JsonObject = {}
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
let newKey: string | number = key
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/ |’|°/g, "_")
if (newKey !== key) {
replacedKeys[key] = newKey
}
const value = obj[key]
if (
typeof value === "object" &&
value !== null &&
!Array.isArray(value)
) {
newObj[newKey] = removeSpacesFromKeys(value, replacedKeys)
} else {
newObj[newKey] = value
}
}
}
return newObj
}
function replaceValuesFromReplacedKeys(
obj: JsonObject,
replacedKeys: { [key: string]: string },
): JsonObject {
const newObj: JsonObject = {}
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
const value = obj[key]
if (
typeof value === "object" &&
value !== null &&
!Array.isArray(value)
) {
newObj[key] = replaceValuesFromReplacedKeys(value, replacedKeys)
} else if (
typeof value === "string" &&
replacedKeys.hasOwnProperty(value)
) {
newObj[key] = replacedKeys[value]
} else if (Array.isArray(value)) {
const newArray = []
for (const arrayValue of value) {
if (
typeof arrayValue === "string" &&
replacedKeys.hasOwnProperty(arrayValue)
) {
newArray.push(replacedKeys[arrayValue])
} else {
newArray.push(arrayValue)
}
}
newObj[key] = newArray
} else {
newObj[key] = value
}
}
}
return newObj
}
function removeZeroValues(obj: any): any {
if (typeof obj !== "object" || obj === null) {
return obj
}
if (Array.isArray(obj)) {
return obj.map(removeZeroValues)
}
const newObj: any = {}
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
const value = obj[key]
if (typeof value === "object" && value !== null) {
if (Object.values(value).every((v) => v === 0 || v === null)) {
continue
}
const cleanedValue = removeZeroValues(value)
if (Object.keys(cleanedValue).length > 0) {
newObj[key] = cleanedValue
}
} else {
newObj[key] = value
}
}
}
return newObj
}
function cleanSimulatedJson(jsonData: any): any {
const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
for (const section of sections) {
if (jsonData[section]) {
jsonData[section] = Object.fromEntries(
Object.entries(jsonData[section]).map(([key, value]) => [
key,
removeZeroValues(value),
]),
)
}
}
return jsonData
}
function splitVariablesByCategory(
section: string,
entityKey: string,
entityData: JsonObject,
initialTestCase: Situation,
) {
const input: JsonObject = {}
const output: JsonObject = {}
for (const [variable, value] of Object.entries(entityData)) {
const replacedKeys: { [key: string]: string } = {}
initialTestCase = removeSpacesFromKeys(initialTestCase, replacedKeys)
if (
initialTestCase[section][entityKey] !== undefined &&
initialTestCase[section][entityKey][variable] === undefined
) {
if (
variableSummaryByName[variable].value_type === "Enum" &&
variableSummaryByName[variable].possible_values !== undefined
) {
// Replace Enum values by their index in possible_values
const newValue: Record<string, number> = {}
for (const [period, periodValue] of Object.entries(value)) {
newValue[period] = Object.keys(
variableSummaryByName[variable].possible_values,
).indexOf(periodValue as string)
}
output[variable] = newValue
} else {
output[variable] = value
}
} else {
input[variable] = value
}
}
return { input, output }
}
function processSection(
section: string,
sectionData: JsonObject,
initialTestCase: Situation,
) {
const inputSection: JsonObject = {}
const outputSection: JsonObject = {}
for (const [entityKey, entityData] of Object.entries(sectionData)) {
const { input, output } = splitVariablesByCategory(
section,
entityKey,
entityData,
initialTestCase,
)
if (Object.keys(input).length > 0) {
inputSection[entityKey] = input
}
if (Object.keys(output).length > 0) {
outputSection[entityKey] = output
}
}
return { input: inputSection, output: outputSection }
}
function buildYamlOutput(
name: string,
year: string,
simulatedTestCase: JsonObject,
testCase: Situation,
) {
const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
const jsonForYamlOutput: {
name: string
period: string
max_spiral_loops: number
input: JsonObject
output: JsonObject
} = {
name: name,
period: year,
max_spiral_loops: 4,
input: {},
output: {},
}
for (const section of sections) {
if (simulatedTestCase[section]) {
const { input, output } = processSection(
section,
simulatedTestCase[section],
testCase,
)
if (Object.keys(input).length > 0) {
jsonForYamlOutput.input[section] = input
}
if (Object.keys(output).length > 0) {
jsonForYamlOutput.output[section] = output
}
}
}
return jsonForYamlOutput
}
function mensualiseVariable(obj: any, variable: string) {
if (!obj || typeof obj !== "object") return obj
const annuelValues = obj
const monthlyValues: any = {}
for (const year in annuelValues) {
if (typeof annuelValues[year] === "number") {
const monthlyAmount =
variableSummaryByName[variable]["set_input"] ===
"set_input_divide_by_period"
? parseFloat((annuelValues[year] / 12).toFixed(2))
: annuelValues[year]
for (let month = 1; month <= 12; month++) {
const monthStr = month < 10 ? `0${month}` : `${month}`
monthlyValues[`${year}-${monthStr}`] = monthlyAmount
}
} else {
monthlyValues[year] = annuelValues[year]
}
}
obj = monthlyValues
return obj
}
function getEntityKeyFromKeyPlural(
keyPlural: string,
entityByKey: EntityByKey,
): string | null {
for (const entity of Object.values(entityByKey)) {
if (entity.key_plural === keyPlural) {
return entity.key
}
}
return null
}
function isGroupEntity(entity: Entity): entity is GroupEntity {
return (entity as GroupEntity).roles !== undefined
}
function isKeyOrPlural(str: string): boolean {
const keys = Object.keys(entityByKey)
for (const key of keys) {
const entity = entityByKey[key]
if (entity.key === str || entity.key_plural === str) {
return true
}
if (isGroupEntity(entity) && entity.roles) {
for (const role of entity.roles) {
if (role.key === str || role.key_plural === str) {
return true
}
if (role.subroles) {
for (const subrole of role.subroles) {
if (subrole.key === str || subrole.key_plural === str) {
return true
}
}
}
}
}
}
return false
}
function replaceVariableValue(
jsonData: any,
variableName: string,
currentValue: any,
newValue: any,
period: string,
): any {
const entities = ["familles", "foyers_fiscaux", "menages", "individus"]
for (const entity of entities) {
if (!jsonData.output[entity]) continue
for (const entityKey in jsonData.output[entity]) {
const entityData = jsonData.output[entity][entityKey]
if (entityData.hasOwnProperty(variableName)) {
const variableData = entityData[variableName]
if (typeof variableData === "object") {
if (
variableData.hasOwnProperty(period) &&
Math.trunc(Number(variableData[period])) ===
Math.trunc(Number(currentValue))
) {
console.info(
`Replacing ${variableName} (${period}): ${variableData[period]} → ${newValue}`,
)
variableData[period] = Number(newValue)
} else {
console.log(
"Warning : rounded value",
Math.trunc(Number(currentValue)),
"not found for",
variableName,
"\nThe cause can be multiple instance of the same variable for different entities... or bug",
)
}
}
}
}
}
return
}
async function runOpenFiscaTest(
openFiscaVenvPath: string,
yamlFilePath: string,
): Promise<any> {
return new Promise<void>((resolve, reject) => {
const openfiscaExecutable = path.join(openFiscaVenvPath, "bin", "openfisca")
const args = ["test", "--country-package", "openfisca_france", yamlFilePath]
const env = {
...process.env,
VIRTUAL_ENV: openFiscaVenvPath,
PATH: `${path.join(openFiscaVenvPath, "bin")}:${process.env.PATH}`,
PYTHONPATH: path.join(
openFiscaVenvPath,
"lib",
"python3.12",
"site-packages",
),
}
let testOutput = ""
const pythonProcess = spawn(openfiscaExecutable, args, {
stdio: ["inherit", "pipe", "pipe"],
env,
})
pythonProcess.stdout.on("data", (data) => {
testOutput += data.toString()
})
pythonProcess.stderr.on("data", (data) => {
console.error(`stderr: ${data}`)
reject(data)
})
pythonProcess.on("close", (code) => {
let returned = undefined as any
if (code !== 0) {
returned = { passed: false }
const testOutputLines = testOutput.split("\n")
const regex =
/(?<variable>[a-zA-Z0-9_]+)@(?<period>\d{4}(?:-\d{2})?):\s+(?<calculated>-?\d+(?:\.\d+)?)\s+differs\s+from\s+(?<expected>-?\d+(?:\.\d+)?)/
testOutputLines.forEach((line) => {
const match = line.match(regex)
if (match && match.groups) {
returned = { ...returned, ...match.groups }
}
})
if (returned.variable !== undefined) {
resolve(returned)
} else
reject(
"Erreur de récupération des valeurs lors d'un appel à openfisca test",
)
} else {
returned = { passed: true }
resolve(returned)
}
})
})
}
async function main() {
console.info("Initializing...")
const personsEntityKey = Object.entries(entityByKey)
.filter(([, entity]) => entity.is_person)
.map(([key]) => key)[0]
const year = options.year
const outdir = options.outdir
const singleTestCase = options.testcase
const variablesFromDecompositions = [
...summaryCalculatedVariablesName,
...otherCalculatedVariablesName,
...nonVirtualVariablesName,
]
const openFiscaVariablesListApiCall = await fetch(
"https://api.fr.openfisca.org/latest/variables",
{
method: "GET",
},
)
if (!openFiscaVariablesListApiCall.ok) {
console.error(
`Error ${
openFiscaVariablesListApiCall.status
} while calling Openfisca France API`,
)
return
}
const openFiscaVariablesList = await openFiscaVariablesListApiCall.json()
const variablesToCalculate = variablesFromDecompositions.filter(
(variable: string) => openFiscaVariablesList.hasOwnProperty(variable),
)
const testCasesCore = testCasesCoreUnknown as unknown as Situation[]
const testCaseCount = testCasesCore.length
let processedCounter = 0
for (const testCase of testCasesCore) {
if (singleTestCase !== undefined && testCase.id !== singleTestCase) {
continue
}
console.info("Processing ", testCase.id)
processedCounter++
let testCaseContainsMissingVariable = false
let jsonForApiCall: JsonObject = {}
jsonForApiCall.input = {}
const replacedKeys: { [key: string]: string } = {}
const sections = ["familles", "foyers_fiscaux", "menages", "individus"]
// Mensualise monthly input variables
for (const section of sections) {
if (testCase[section] !== undefined) {
for (const entityKey in testCase[section]) {
for (const variable in testCase[section][entityKey]) {
if (
variableSummaryByName.hasOwnProperty(variable) &&
variableSummaryByName[variable].definition_period === "month"
) {
testCase[section][entityKey][variable] = mensualiseVariable(
testCase[section][entityKey][variable],
variable,
)
} else if (
!openFiscaVariablesList.hasOwnProperty(variable) &&
!isKeyOrPlural(variable)
) {
console.info(
"variable",
variable,
"does not exists in openFisca-France.",
)
testCaseContainsMissingVariable = true
}
}
}
}
}
if (testCaseContainsMissingVariable) {
console.info("Ignoring test case", testCase.id)
continue
}
for (const section of sections) {
if (testCase[section] !== undefined) {
jsonForApiCall.input[section] = removeSpacesFromKeys(
testCase[section],
replacedKeys,
)
for (const entityKey in jsonForApiCall.input[section]) {
for (const variable of variablesToCalculate) {
if (variableSummaryByName[variable] !== undefined) {
if (
variableSummaryByName[variable].entity ===
getEntityKeyFromKeyPlural(section, entityByKey) &&
jsonForApiCall.input[section][entityKey][variable] === undefined
) {
switch (variableSummaryByName[variable].definition_period) {
case "year": {
jsonForApiCall.input[section][entityKey][variable] = {
[year]: null,
}
break
}
case "month": {
const monthVariableSection = {}
for (let month = 1; month <= 12; month++) {
const monthString = month < 10 ? "0" + month : month
const monthPeriod = year + "-" + monthString
Object.assign(monthVariableSection, {
[monthPeriod]: null,
})
}
jsonForApiCall.input[section][entityKey][variable] =
monthVariableSection
}
}
}
} else {
console.log(
"cette variable n'existe pas dans summary : ",
variable,
)
}
}
}
}
}
jsonForApiCall = replaceValuesFromReplacedKeys(jsonForApiCall, replacedKeys)
// await fs.writeFile(
// path.join(".", testCase.id + ".json"),
// JSON.stringify(jsonForApiCall.input, null, 2),
// )
console.info("Launching simulation on OpenFisca France API...")
const openFiscaApiUrl = "https://api.fr.openfisca.org/latest/calculate"
// const openFiscaApiUrl = "http://localhost:5000/calculate"
const openFiscaApiOptions: RequestInit = {
body: JSON.stringify(jsonForApiCall.input),
headers: {
Accept: "application/json",
"Content-Type": "application/json; charset=utf-8",
},
method: "POST",
}
let simulatedTestCaseFromOpenFiscaApi: Response | null = null
try {
simulatedTestCaseFromOpenFiscaApi = await fetchWithRetries(
openFiscaApiUrl,
openFiscaApiOptions,
)
if (!simulatedTestCaseFromOpenFiscaApi.ok) {
console.error(
`Giving up calling openFisca API because of status ${simulatedTestCaseFromOpenFiscaApi.status}`,
)
return
}
} catch (error) {
console.error("Error : ", error)
}
console.info("Post-processing...")
const simulatedTestCase = cleanSimulatedJson(
await simulatedTestCaseFromOpenFiscaApi!.json(),
)
const jsonForYamlOutput = buildYamlOutput(
testCase.id!,
year,
simulatedTestCase,
testCase,
)
let ok = false
while (!ok) {
const yamlOutput = YAML.dump(jsonForYamlOutput, {
noCompatMode: true,
noRefs: true,
})
const cleanedYaml = yamlOutput.replace(/'(\d{4})':/g, "$1:")
await fs.writeFile(path.join(outdir, testCase.id + ".yml"), cleanedYaml)
const venvPath = "/home/cafe/apps/openfisca-france/.venv"
try {
const openFiscaTestResult = await runOpenFiscaTest(
venvPath,
path.join(outdir, testCase.id + ".yml"),
)
if (!openFiscaTestResult.passed) {
if (
openFiscaTestResult.variable === undefined ||
openFiscaTestResult.expected === undefined ||
openFiscaTestResult.calculated === undefined ||
openFiscaTestResult.period === undefined
) {
throw new Error("Error retrieving values to update")
}
replaceVariableValue(
jsonForYamlOutput,
openFiscaTestResult.variable,
openFiscaTestResult.expected,
openFiscaTestResult.calculated,
openFiscaTestResult.period,
)
} else {
ok = true
console.info("YAML file written :", outdir, testCase.id + ".yml")
}
} catch (error) {
console.error(error)
}
}
console.info("Processed test-case", processedCounter, "over", testCaseCount)
}
}
main()
.then(() => {
process.exit(0)
})
.catch((error: unknown) => {
console.error(error)
process.exit(1)
})