Use API to fill XLSX file

src/routes/variables/[variable]/ast/+server.ts

@@ -29,8 +29,8 @@ const variablesDir = path.join(
 async function addVariableAndInputs(
   name,
   encounteredVariablesName = new Set<string>(),
-  variableByName: { [name: string]: Variable } = {},
-): Promise<{ [name: string]: Variable }> {
+  variables: Variable[] = [],
+): Promise<Variable[]> {
   encounteredVariablesName.add(name)
   const variableFilePath = path.join(variablesDir, `${name}.json`)
   const variable = (await fs.readJson(variableFilePath)) as Variable
@@ -44,14 +44,14 @@ async function addVariableAndInputs(
           await addVariableAndInputs(
             inputVariableName,
             encounteredVariablesName,
-            variableByName,
+            variables,
           )
         }
       }
     }
   }
-  variableByName[name] = variable
-  return variableByName
+  variables.push(variable)
+  return variables
 }
 
 function auditParams(audit: Audit, dataUnknown: unknown): [unknown, unknown] {
@@ -100,6 +100,6 @@ export const GET: RequestHandler = async ({ params: requestParams, url }) => {
     throw error(404, `Variable ${name} not found`)
   }
 
-  const variableByName = await addVariableAndInputs(name)
-  return json({ variables: variableByName })
+  const variables = await addVariableAndInputs(name)
+  return json({ variables })
 }

src/routes/variables/[variable]/xlsx/+page.svelte

 <script lang="ts">
   import {
+    getRolePersonsIdKey,
     getVariableLatestFormulaDate,
     openfiscaAstFromFormulaPythonAst,
     xlsxFormulaFromOpenfiscaAst,
+    type GroupEntity,
+    type PopulationWithoutId,
     type Variable,
   } from "@openfisca/json-model"
-  import { Workbook } from "exceljs"
+  import { Workbook, type Worksheet } from "exceljs"
   import Range from "exceljs/lib/doc/range"
+  import Sockette from "sockette"
+  import { getContext } from "svelte"
+  import type { Writable } from "svelte/store"
+  import { v4 as uuidV4 } from "uuid"
 
   import { saveAs } from "file-saver"
 
   import { browser } from "$app/environment"
+  import { calculationNames, type CalculationByName } from "$lib/calculations"
   import { entityByKey } from "$lib/entities"
-  import { variableSummaryByName } from "$lib/variables"
+  import {
+    getPopulationReservedKeys,
+    type Axis,
+    type Situation,
+    type SituationWithAxes,
+  } from "$lib/situations"
+  import {
+    variableSummaryByName,
+    type ValuesByCalculationNameByVariableName,
+    type VariableValue,
+  } from "$lib/variables"
+  import type { WebSocketByName, WebSocketOpenByName } from "$lib/websockets"
 
   import type { PageData } from "./$types"
 
   export let data: PageData
 
-  $: ({ variables: variableByName } = data)
+  const axes = getContext("axes") as Writable<Axis[][]>
+  let calculationByName: CalculationByName | undefined = undefined
+  const inputInstantsByVariableNameArray = getContext(
+    "inputInstantsByVariableNameArray",
+  ) as Writable<
+    Array<{
+      [name: string]: Set<string>
+    }>
+  >
+  const testCases = getContext("testCases") as Writable<Situation[]>
+  let valuesByCalculationNameByVariableName: ValuesByCalculationNameByVariableName =
+    {}
+  let webSocketByName: WebSocketByName | undefined = undefined
+  const webSocketOpenByName: WebSocketOpenByName = {}
+  const year = getContext("year") as Writable<number>
+
+  $: ({ variables } = data)
 
   $: if (browser) {
-    generateWorkbook(variableByName)
+    openWebSocket()
+  }
+
+  $: if (
+    calculationByName !== undefined &&
+    !Object.values(calculationByName).some(({ running }) => running)
+  ) {
+    generateWorkbook(variables, valuesByCalculationNameByVariableName)
+  }
+
+  function calculate(variables: Variable[]) {
+    console.log("CALCULATE")
+    const aggregatedSituation: SituationWithAxes = {}
+    const entities = Object.values(entityByKey)
+    // Aggregate every situations into a single one without calculated variables.
+    for (const [situationIndex, situation] of $testCases.entries()) {
+      const inputInstantsByVariableName =
+        $inputInstantsByVariableNameArray[situationIndex]
+      const situationPrefix = `Cas type n°${situationIndex + 1} | `
+      for (const entity of entities) {
+        let entitySituation = situation[entity.key_plural]
+        if (entitySituation === undefined) {
+          continue
+        }
+        let aggregatedEntitySituation = aggregatedSituation[entity.key_plural]
+        if (aggregatedEntitySituation === undefined) {
+          aggregatedEntitySituation = aggregatedSituation[entity.key_plural] =
+            {}
+        }
+        const reservedKeys = getPopulationReservedKeys(entity)
+        for (const [populationId, population] of Object.entries(
+          entitySituation,
+        ).sort(([populationId1], [populationId2]) =>
+          populationId1.localeCompare(populationId2),
+        )) {
+          const transformedPopulation: PopulationWithoutId = {}
+          if (!entity.is_person) {
+            for (const role of (entity as GroupEntity).roles) {
+              const personsIdKey = getRolePersonsIdKey(role)
+              const personsId = population[personsIdKey] as string[] | undefined
+              if (personsId === undefined) {
+                continue
+              }
+              transformedPopulation[personsIdKey] = personsId.map(
+                (personId) => situationPrefix + personId,
+              )
+            }
+          }
+          for (const [variableName, variableValueByInstant] of Object.entries(
+            population,
+          )) {
+            if (reservedKeys.has(variableName)) {
+              continue
+            }
+            const inputVariableValueByInstant: {
+              [instant: string]: VariableValue | null
+            } = {}
+            const inputInstants =
+              inputInstantsByVariableName[variableName] ?? new Set<string>()
+            for (const [instant, variableValue] of Object.entries(
+              variableValueByInstant,
+            )) {
+              if (!inputInstants.has(instant)) {
+                // Ignore calculated value.
+                continue
+              }
+              inputVariableValueByInstant[instant] = variableValue
+            }
+            if (Object.keys(inputVariableValueByInstant).length > 0) {
+              transformedPopulation[variableName] = inputVariableValueByInstant
+            }
+          }
+          aggregatedEntitySituation[situationPrefix + populationId] =
+            transformedPopulation
+        }
+      }
+    }
+
+    if ($axes.length > 0) {
+      aggregatedSituation.axes = $axes
+    }
+
+    const message = {
+      calculate: true,
+      period: $year.toString(),
+    }
+    calculationByName = {}
+
+    console.log(
+      "webSocketOpenByName",
+      webSocketOpenByName,
+      variables.map(({ name }) => name),
+    )
+    if (webSocketOpenByName.law) {
+      // Note: crypto.randomUUID() is not supported by Safari before version 15.4.
+      // const token = crypto.randomUUID()
+      const token = uuidV4()
+      calculationByName.law = { running: true, token }
+      webSocketByName.law.send(
+        JSON.stringify({
+          ...message,
+          situation: aggregatedSituation,
+          title: "law",
+          token,
+          variables: variables.map(({ name }) => name),
+        }),
+      )
     }
 
-  async function generateWorkbook(variableByName: {
-    [name: string]: Variable
-  }) {
+    // if (
+    //   $billName !== undefined &&
+    //   webSocketOpenByName.bill
+    // ) {
+    //   // Note: crypto.randomUUID() is not supported by Safari before version 15.4.
+    //   // const token = crypto.randomUUID()
+    //   const token = uuidV4()
+    //   calculationByName.bill = { running: true, token }
+    //   $webSocketByName.bill.send(
+    //     JSON.stringify({
+    //       ...message,
+    //       reform_name: $billName,
+    //       situation: aggregatedSituation,
+    //       title: "bill",
+    //       token,
+    //       variables: [
+    //         ...(nonVirtualDecompositionsNameByReformName[$billName] ??
+    //           nonVirtualDecompositionsName),
+    //       ],
+    //     }),
+    //   )
+    // }
+
+    // if (
+    //   Object.keys($parametricReform).length > 0 &&
+    //   webSocketOpenByName.amendment
+    // ) {
+    //   // Note: crypto.randomUUID() is not supported by Safari before version 15.4.
+    //   // const token = crypto.randomUUID()
+    //   const token = uuidV4()
+    //   calculationByName.amendment = { running: true, token }
+    //   webSocketByName.amendment.send(
+    //     JSON.stringify({
+    //       ...message,
+    //       parametric_reform: $parametricReform,
+    //       reform_name: $billName, // Will be removed when $billName is undefined.
+    //       situation: aggregatedSituation,
+    //       title: "amendment",
+    //       token,
+    //       variables: [
+    //         ...(nonVirtualDecompositionsNameByReformName[$billName] ??
+    //           nonVirtualDecompositionsName),
+    //       ],
+    //     }),
+    //   )
+    // }
+  }
+
+  async function generateWorkbook(
+    variables: Variable[],
+    valuesByCalculationNameByVariableName: ValuesByCalculationNameByVariableName,
+  ) {
     const workbook = new Workbook()
     workbook.creator = "Leximpact"
     workbook.lastModifiedBy = "Leximpact"
@@ -41,28 +231,37 @@
     workbook.calcProperties.fullCalcOnLoad = true
 
     // The Workbook views controls how many separate windows Excel will open when viewing the workbook.
-    workbook.views = [
-      {
-        x: 0,
-        y: 0,
-        width: 10000,
-        height: 20000,
-        firstSheet: 0,
-        activeTab: 1,
-        visibility: "visible",
-      },
-    ]
+    // workbook.views = [
+    //   {
+    //     x: 0,
+    //     y: 0,
+    //     width: 10000,
+    //     height: 20000,
+    //     firstSheet: 0,
+    //     activeTab: 1,
+    //     visibility: "visible",
+    //   },
+    // ]
 
-    const worksheet = workbook.addWorksheet("Individus")
+    const indexByEntityKey: { [key: string]: number } = {}
+    const worksheetByEntityKey: { [key: string]: Worksheet } = {}
+    for (const variable of variables) {
+      const values = valuesByCalculationNameByVariableName[variable.name].law
 
-    for (const [index, variable] of Object.values(variableByName).entries()) {
-      const column = worksheet.getColumn(index + 1)
-      column.values = [variable.name]
+      let worksheet = worksheetByEntityKey[variable.entity]
+      if (worksheet === undefined) {
+        worksheet = worksheetByEntityKey[variable.entity] =
+          workbook.addWorksheet(variable.entity)
+      }
+      let index = (indexByEntityKey[variable.entity] =
+        (indexByEntityKey[variable.entity] ?? 0) + 1)
+      const column = worksheet.getColumn(index)
+      column.values = [variable.name, ...values]
       const range = new Range({
         top: 2,
-        left: index + 1,
-        bottom: 10,
-        right: index + 1,
+        left: index,
+        bottom: 2 + values.length - 1,
+        right: index,
       }).toString()
       workbook.definedNames.add(range, variable.name)
 
@@ -85,17 +284,7 @@
               worksheet.fillFormula(
                 range,
                 latestXlsxFormula.output,
-                [
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                  "test",
-                ],
+                values.map((value) => ""),
                 "array",
               )
               // } else {
@@ -147,6 +336,93 @@
 
     saveAs(blob, "leximpact.xlsx")
   }
+
+  function openWebSocket() {
+    let remainingApiWebSocketBaseUrls: string[] = []
+    webSocketByName = Object.fromEntries(
+      calculationNames
+        .filter((_calculationName, index) => index === 0)
+        .map((calculationName) => {
+          if (remainingApiWebSocketBaseUrls.length === 0) {
+            remainingApiWebSocketBaseUrls = [...data.apiWebSocketBaseUrls]
+          }
+          const apiWebSocketBaseUrlIndex = Math.floor(
+            Math.random() * remainingApiWebSocketBaseUrls.length,
+          )
+          const apiWebSocketBaseUrl =
+            remainingApiWebSocketBaseUrls[apiWebSocketBaseUrlIndex]
+          remainingApiWebSocketBaseUrls.splice(apiWebSocketBaseUrlIndex, 1)
+          const webSocket = new Sockette(
+            new URL("simulations/calculate", apiWebSocketBaseUrl).toString(),
+            {
+              // maxAttempts: 10,
+              onmessage: (event) => {
+                let calculation = calculationByName[calculationName]
+                const result = JSON.parse(event.data)
+                if (result.errors !== undefined) {
+                  console.error("API Error:", result)
+                } else if (calculation === undefined) {
+                  // console.log(
+                  //   `Ignoring API ${calculationName} response, because this calculation is not running.`,
+                  // )
+                } else if (result.token !== calculation.token) {
+                  // console.log(
+                  //   `Ignoring API ${calculationName} response with invalid token ${result.token} instead of ${calculation.token}.`,
+                  // )
+                } else if (result.done) {
+                  calculation = { ...calculation }
+                  delete calculation.running
+                  calculationByName = {
+                    ...calculationByName,
+                    [calculationName]: calculation,
+                  }
+                } else {
+                  // Round returned values if unit of variable is a currency.
+                  const variable = variableSummaryByName[result.name]
+                  const resultValue =
+                    variable.unit === "currency" ||
+                    variable.unit?.startsWith("currency-")
+                      ? result.value.map((valueAtIndex) =>
+                          Math.round(valueAtIndex),
+                        )
+                      : result.value
+                  const valuesByCalculationName =
+                    valuesByCalculationNameByVariableName[result.name] ?? {}
+                  valuesByCalculationNameByVariableName = {
+                    ...valuesByCalculationNameByVariableName,
+                    [result.name]: {
+                      ...valuesByCalculationName,
+                      [calculationName]: resultValue,
+                    },
+                  }
+                }
+              },
+              // onopen: (event) => console.log("[WebSocket] Connected!", event),
+              onopen(this) {
+                if (!webSocketOpenByName[calculationName]) {
+                  webSocketOpenByName[calculationName] = true
+                  if (
+                    Object.values(webSocketOpenByName).filter(Boolean)
+                      .length === Object.keys(webSocketByName).length
+                  ) {
+                    // All web sockets are open. Launch simulation.
+                    calculate(variables)
+                  }
+                }
+              },
+              // onreconnect: (event) =>
+              //   console.log("[WebSocket] Reconnecting...", event),
+              // onmaximum: (event) =>
+              //   console.log("[WebSocket] Stop Attempting!", event),
+              // onclose: (event) => console.log("[WebSocket] Closed!", event),
+              // onerror: (event) => console.log("[WebSocket] Error:", event),
+              // timeout: 5e3,
+            },
+          )
+          return [calculationName, webSocket]
+        }),
+    ) as unknown as WebSocketByName
+  }
 </script>
 
-<p>{Object.keys(variableByName).length}</p>
+<p>{variables.length}</p>

src/routes/variables/[variable]/xlsx/+page.ts

@@ -11,11 +11,9 @@ export const load: PageLoad = async function ({ fetch, params }) {
   if (!response.ok) {
     throw error(response.status, `Could not load AST`)
   }
-  const { variables: variableByName } = (await response.json()) as {
-    variables: { [name: string]: Variable }
-  }
+  const { variables } = (await response.json()) as { variables: Variable[] }
 
   return {
-    variables: variableByName,
+    variables,
   }
 }