Skip to main content
Explore
Sign in
Explore
Snippets Groups Projects
Commit 7571649e authored by Emmanuel Raviart's avatar Emmanuel Raviart
Browse files

Rewrite CI in TypeScript.

parent 19f66193
Branches
Tags
1 merge request!5Rewrite CI in TypeScript
Pipeline #1071 failed
Stage: build
Show whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 6
− 85
View file @ 7571649e
...@@ -6,103 +6,24 @@ default: ...@@ -6,103 +6,24 @@ default:
- /root/.cache/pypoetry/virtualenvs - /root/.cache/pypoetry/virtualenvs
- node_modules - node_modules
stages: ci:
- build
- sync_with_openfisca_france
- trigger
build:
stage: build
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
before_script:
## Without apt update, apt install fails.
- apt update --yes
- apt install --yes python-is-python3 python3-virtualenv
- curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
## Add Poetry to path:
- export PATH="/root/.local/bin:$PATH"
- poetry install
script:
- poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test
build_with_latest_dependencies:
stage: build stage: build
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
before_script: before_script:
## Without apt update, apt install fails. ## Without apt update, apt install fails.
- apt update --yes - apt update --yes
- apt install --yes python-is-python3 python3-virtualenv
- curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
## Add Poetry to path:
- export PATH="/root/.local/bin:$PATH"
- poetry update
- poetry install
script:
- poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test
sync_with_openfisca_france:
stage: sync_with_openfisca_france
rules:
# This step is called
# - When a pipeline has been triggered (for example when OpenFisca-France has been updated).
# In this case a new version of OpenFisca-France-Reforms must be created if a dependency has changed.
- if: '($CI_PIPELINE_SOURCE == "api" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "web") && $CI_PROJECT_URL == "https://git.leximpact.dev/openfisca/openfisca-france-reforms" && $CI_COMMIT_BRANCH == "master"'
# - When a merge request has been merged into master (ie content of OpenFisca-France-Reforms has been changed).
# In this case a new version of OpenFisca-France-Reforms must always be created.
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_PROJECT_URL == "https://git.leximpact.dev/openfisca/openfisca-france-reforms" && $CI_COMMIT_BRANCH == "master"'
before_script:
## Without apt update, apt install fails.
- apt update --yes
- apt install --yes python-is-python3 python3-virtualenv
- curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
## Add Poetry to path:
- export PATH="/root/.local/bin:$PATH"
## Install ssh-agent if not already installed, it is required by Docker. ## Install ssh-agent if not already installed, it is required by Docker.
- "which ssh-agent || apt install -y openssh-client" - "which ssh-agent || apt install -y openssh-client"
## Run ssh-agent (inside the build environment) ## Run ssh-agent (inside the build environment)
- eval $(ssh-agent -s) - eval $(ssh-agent -s)
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
## Create the SSH directory and give it the right permissions
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
## Accept the SSH host keys of GitLab server.
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
## Set the Git user name and email.
- git config --global user.email "admin+openfisca-france-reforms-ci@tax-benefit.org"
- git config --global user.name "OpenFisca-France Reforms CI"
# Reset current repo, because it may have been tranformed by a previous CI that failed. - apt install --yes python-is-python3 python3-virtualenv
- git switch master - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python -
- git reset --hard origin/master ## Add Poetry to path:
- git status - export PATH="/root/.local/bin:$PATH"
- git remote set-url origin git@git.leximpact.dev:openfisca/openfisca-france-reforms.git
- npm install - npm install
## Needed when Node version changes: ## Needed when Node version changes:
- npm rebuild - npm rebuild
script: script:
- | - npx babel-node --extensions ".ts" -- src/scripts/gitlab-ci.ts
if [ "$CI_PIPELINE_SOURCE" == "push" ]; then
npx babel-node --extensions ".ts" -- src/scripts/sync_with_openfisca_france.ts --force
else
npx babel-node --extensions ".ts" -- src/scripts/sync_with_openfisca_france.ts
fi
trigger_openfisca_json_model:
stage: trigger
only:
# OpenFisca-JSON-Model pipeline should only be called
# when a new version branch and version tag has been created.
# Here we assume that tags are only created for versions.
- tags
trigger: openfisca/json-model
src/scripts/sync_with_openfisca_france.ts → src/scripts/gitlab-ci.ts
+ 234
− 0
View file @ 7571649e
import toml from "@ltd/j-toml" import toml from "@ltd/j-toml"
import { $, argv, fs } from "zx" import assert from "assert"
import { $, fs } from "zx"
interface Package { interface Package {
name: string name: string
version: string version: string
} }
async function latestVersionObjectFromTags() { export interface VersionObject {
return (await $`git tag --list`).stdout major: number
.split("\n") minor: number
.filter((line) => line.match(/^\d+\.\d+\.\d+$/)) patch: number
.map((version) => {
const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/) as string[]
return {
major: parseInt(match[1]),
minor: parseInt(match[2]),
patch: parseInt(match[3]),
}
})
.sort((groups1, groups2) =>
groups1.major !== groups2.major
? groups2.major - groups1.major
: groups1.minor !== groups2.minor
? groups2.minor - groups1.minor
: groups2.patch - groups1.patch,
)[0]
} }
/// Upgrade, test & push openfisca-france-reforms when openfisca-france or const {
/// openfisca-france-reforms change. CI_COMMIT_BRANCH,
async function main() { CI_DEFAULT_BRANCH,
// Retrieve all tags (used by latestVersionObjectFromTags). CI_MERGE_REQUEST_SOURCE_BRANCH_NAME,
await `git fetch --all --tags` CI_MERGE_REQUEST_TARGET_BRANCH_NAME,
CI_PIPELINE_SOURCE,
SSH_PRIVATE_KEY,
} = process.env
await $`poetry update` async function configure() {
await $`poetry install` console.log("Configuring CI…")
if (!argv.force) { await configureGit()
await $`git add .` await configureSsh()
if ((await $`git diff --quiet --staged`.exitCode) === 0) {
// Repository content has not changed.
// => Ensure that job is stopped and other pipelines are not triggered.
process.exit(1) console.log("CI configuration finished.")
} }
async function configureGit() {
// Set the Git user name and email.
await $`git config --global user.email "admin+openfisca-france-reforms-ci@tax-benefit.org"`
await $`git config --global user.name "OpenFisca-France Reforms CI"`
} }
// Dependencies of openfisca-france-reforms have changed. async function configureSsh() {
// Note: `eval $(ssh-agent -s)` must be done before calling this script because it
// creates 2 environment variables (then used by ssh clients).
// // Install ssh-agent if not already installed, to be able to use git with ssh.
// if ((await $`which ssh-agent`.exitCode) !== 0) {
// await $`apt install -y openssh-client`
// }
// // Run ssh-agent (inside the build environment)
// await $`eval $(ssh-agent -s)`
// Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
// Use `tr` to fix line endings which makes ed25519 keys work without
// extra base64 encoding.
// https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
if (SSH_PRIVATE_KEY !== undefined) {
await $`echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -`
}
// Create the SSH directory and give it the right permissions
await $`mkdir -p ~/.ssh`
await $`chmod 700 ~/.ssh`
// Accept the SSH host keys of GitLab server.
await $`echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts`
await $`chmod 644 ~/.ssh/known_hosts`
}
async function createNewBranchWithUpdatedVersions() {
// Ensure that everything works. // Ensure that everything works.
await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test` await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test`
...@@ -60,7 +76,9 @@ async function main() { ...@@ -60,7 +76,9 @@ async function main() {
const openFiscaFranceVersion = openFiscaFrancePackage!.version const openFiscaFranceVersion = openFiscaFrancePackage!.version
// Retrieve next version of OpenFisca-France-Reforms. // Retrieve next version of OpenFisca-France-Reforms.
const { major, minor, patch } = await latestVersionObjectFromTags() const versionObject = await latestVersionObjectFromTags()
assert.notStrictEqual(versionObject, undefined)
const { major, minor, patch } = versionObject!
const nextVersion = `${major}.${minor}.${patch + 1}` const nextVersion = `${major}.${minor}.${patch + 1}`
// Update pyproject.toml with latest version of OpenFisca-France // Update pyproject.toml with latest version of OpenFisca-France
...@@ -87,6 +105,125 @@ async function main() { ...@@ -87,6 +105,125 @@ async function main() {
await $`git switch master` await $`git switch master`
} }
async function latestVersionObjectFromTags(): Promise<
VersionObject | undefined
> {
// Retrieve all tags.
await `git fetch --all --tags`
return (await $`git tag --list`).stdout
.split("\n")
.filter((line) => line.match(/^\d+\.\d+\.\d+$/))
.map((version): VersionObject => {
const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/) as string[]
return {
major: parseInt(match[1]),
minor: parseInt(match[2]),
patch: parseInt(match[3]),
}
})
.sort((groups1, groups2) =>
groups1.major !== groups2.major
? groups2.major - groups1.major
: groups1.minor !== groups2.minor
? groups2.minor - groups1.minor
: groups2.patch - groups1.patch,
)[0]
}
async function main() {
console.log("Starting gitlab-ci.ts…")
await configure()
console.log(`Handling "${CI_PIPELINE_SOURCE}" event…`)
switch (CI_PIPELINE_SOURCE) {
case "api":
case "schedule":
case "trigger":
case "web": {
if (CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH) {
// A pipeline has been triggered (for example when OpenFisca-France has been updated or a user as
// launched a new pipeline manually or…).
// => A new version of OpenFisca-France-Reforms must be created if and only if a dependency has
// changed.
console.log(`Handling trigger…`)
// // Reset current repo, because it may have been tranformed by a previous CI that failed.
// await $`git switch master`
// await $`git reset --hard origin/master`
await $`git status`
await $`git remote set-url origin git@git.leximpact.dev:openfisca/openfisca-france-reforms.git`
await $`poetry update`
await $`poetry install`
await $`git add .`
if ((await $`git diff --quiet --staged`.exitCode) !== 0) {
// Dependencies of openfisca-france-reforms have changed.
await createNewBranchWithUpdatedVersions()
}
} else {
console.log(
`Unhandled event "${CI_PIPELINE_SOURCE}" in branch "${CI_COMMIT_BRANCH}".`,
)
}
break
}
case "merge_request_event": {
console.log(
`Merge request from branch ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} to branch ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}`,
)
// Test OpenFisca-France-Reforms with its current dependencies.
await $`poetry install`
await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test`
// Test OpenFisca-France-Reforms with the latest dependencies (especially OpenFisca-France).
await $`poetry update`
await $`poetry install`
await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test`
break
}
case "push": {
console.log(`Push to branch ${CI_COMMIT_BRANCH}`)
// Test OpenFisca-France-Reforms with its current dependencies.
await $`poetry install`
await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test`
// Test OpenFisca-France-Reforms with the latest dependencies (especially OpenFisca-France).
await $`poetry update`
await $`poetry install`
await $`poetry run python -m openfisca_france_reforms.plf_2022.scripts.run_test`
if (CI_COMMIT_BRANCH === CI_DEFAULT_BRANCH) {
// A merge request has been merged into master (ie content of OpenFisca-France-Reforms has been changed).
// => Create a new version of OpenFisca-France-Reforms.
// // Reset current repo, because it may have been tranformed by a previous CI that failed.
// await $`git switch master`
// await $`git reset --hard origin/master`
await $`git status`
await $`git remote set-url origin git@git.leximpact.dev:openfisca/openfisca-france-reforms.git`
await createNewBranchWithUpdatedVersions()
} else {
console.log(
`Unhandled event "${CI_PIPELINE_SOURCE}" in branch "${CI_COMMIT_BRANCH}".`,
)
}
break
}
default:
console.log(
`Unhandled event "${CI_PIPELINE_SOURCE}" in branch "${CI_COMMIT_BRANCH}".`,
)
}
}
main() main()
.then(() => { .then(() => {
process.exit(0) process.exit(0)
... ...
......
src/scripts/test.ts deleted 100644 → 0
+ 0
− 38
View file @ 19f66193
import { $ } from "zx"
async function latestVersionObjectFromTags() {
return (await $`git tag --list`).stdout
.split("\n")
.filter((line) => line.match(/^\d+\.\d+\.\d+$/))
.map((version) => {
const match = version.match(/^(\d+)\.(\d+)\.(\d+)$/) as string[]
return {
major: parseInt(match[1]),
minor: parseInt(match[2]),
patch: parseInt(match[3]),
}
})
.sort((groups1, groups2) =>
groups1.major !== groups2.major
? groups2.major - groups1.major
: groups1.minor !== groups2.minor
? groups2.minor - groups1.minor
: groups2.patch - groups1.patch,
)[0]
}
async function main() {
const { major, minor, patch } = await latestVersionObjectFromTags()
console.log("a", major, minor, patch)
const nextVersion = `${major}.${minor}.${patch + 1}`
console.log("b", nextVersion)
}
main()
.then(() => {
process.exit(0)
})
.catch((error: unknown) => {
console.error(error)
process.exit(1)
})
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment