Skip to content
Commits on Source (22)
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 1efda69d
# variables in the GitLab CI/CD variables:
# GITLAB_TOKEN to support the semantic-release
# DOCKER_AUTH_CONFIG to support the usage of private docker images as job docker image
# TMPL_RELEASE_ENABLED to enable the semantic-release job
# TBC_NAMESPACE: smartdatalab/public/ci-cd-components
include:
- project: "to-be-continuous/tools/gitlab-ci"
ref: "master"
file: "/templates/extract.yml"
- project: "to-be-continuous/tools/gitlab-ci"
ref: "master"
file: "/templates/validation.yml"
- project: "to-be-continuous/kicker"
ref: "master"
file: "/templates/validation.yml"
- project: "to-be-continuous/bash"
ref: "3.5"
file: "/templates/gitlab-ci-bash.yml"
- component: $CI_SERVER_FQDN/to-be-continuous/bash/gitlab-ci-bash@3.5
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/gitlab-ci/extract@master
inputs:
extract-script-job-tags: ["docker"]
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/gitlab-ci/validation@master
inputs:
check-links-job-tags: ["docker"]
tbc-check-job-tags: ["docker"]
tbc-check-image: cicd-docker-dev.artifact.tecnalia.dev/tbc-check:master
gitlab-ci-lint-job-tags: ["docker"]
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/kicker/validation@master
inputs:
kicker-validation-job-tags: ["docker"]
schema-base-url: "https://git.code.tecnalia.dev/api/v4/projects/smartdatalab%2Fpublic%2Fci-cd-components%2Fkicker/repository/files"
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/bash/gitlab-ci-bash@master
inputs:
bash-shellcheck-job-tags: ["docker"]
shellcheck-files: "*.sh"
- component: $CI_SERVER_FQDN/to-be-continuous/gitleaks/gitlab-ci-gitleaks@2.6
- component: $CI_SERVER_FQDN/to-be-continuous/semantic-release/gitlab-ci-semrel@3.11
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/semantic-release/gitlab-ci-semrel@master
inputs:
semantic-release-job-tags: ["docker"]
- component: $CI_SERVER_FQDN/$TBC_NAMESPACE/gitleaks/gitlab-ci-gitleaks@master
inputs:
gitleaks-job-tags: ["docker"]
variables:
GITLAB_CI_FILES: "templates/gitlab-ci-python.yml"
GIT_STRATEGY: clone
semantic-release:
rules:
......
CHANGELOG.md
View file @ 1efda69d
## [7.7.1](https://git.code.tecnalia.dev/smartdatalab/public/ci-cd-components/python/compare/7.7.0...7.7.1) (2025-01-20)
### Bug Fixes
* move back 'reports' dir creation at job level to fix variants missing reports dir ([bf15efe](https://git.code.tecnalia.dev/smartdatalab/public/ci-cd-components/python/commit/bf15efe4b008a5f292e782d0363a52000bf43f37))
## [7.7.1](https://gitlab.com/to-be-continuous/python/compare/7.7.0...7.7.1) (2025-01-12)
### Bug Fixes
* move back 'reports' dir creation at job level to fix variants missing reports dir ([bf15efe](https://gitlab.com/to-be-continuous/python/commit/bf15efe4b008a5f292e782d0363a52000bf43f37))
# [7.7.0](https://gitlab.com/to-be-continuous/python/compare/7.6.0...7.7.0) (2025-01-12)
......
README.md
View file @ 1efda69d
......@@ -14,7 +14,7 @@ Add the following to your `.gitlab-ci.yml`:
```yaml
include:
# 1: include the component
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.12-slim
......@@ -29,7 +29,7 @@ Add the following to your `.gitlab-ci.yml`:
include:
# 1: include the template
- project: 'to-be-continuous/python'
ref: '7.7.0'
ref: '7.7.1'
file: '/templates/gitlab-ci-python.yml'
variables:
......@@ -53,6 +53,7 @@ The Python template uses some global configuration used throughout all jobs.
| `extra-deps` / `PYTHON_EXTRA_DEPS` | Python extra sets of dependencies to install<br/>For [Setuptools](https://setuptools.pypa.io/en/latest/userguide/dependency_management.html?highlight=extras#optional-dependencies) or [Poetry](https://python-poetry.org/docs/pyproject/#extras) only | _none_ |
| `reqs-file` / `PYTHON_REQS_FILE` | Main requirements file _(relative to `$PYTHON_PROJECT_DIR`)_<br/>For [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) build-system only | `requirements.txt` |
| `extra-reqs-files` / `PYTHON_EXTRA_REQS_FILES` | Extra dev requirements file(s) to install _(relative to `$PYTHON_PROJECT_DIR`)_ | `requirements-dev.txt` |
| `py-publish-job-tags` / `PY_PUBLISH_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
The cache policy also makes the necessary to manage pip cache (not to download Python dependencies over and over again).
......@@ -83,6 +84,10 @@ This job allows building your Python project [distribution packages](https://pac
It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`.
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `py-package-job-tags` / `PY_PACKAGE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### Lint jobs
#### `py-lint` job
......@@ -97,6 +102,7 @@ It is bound to the `build` stage, and uses the following variables:
| `pylint-enabled` / `PYLINT_ENABLED` | Set to `true` to enable the `pylint` job | _none_ (disabled) |
| `pylint-args` / `PYLINT_ARGS` | Additional [pylint CLI options](http://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options) | _none_ |
| `pylint-files` / `PYLINT_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
| `py-lint-job-tags` / `PY_LINT_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
In addition to a textual report in the console, this job produces the following reports, kept for one day:
......@@ -127,6 +133,7 @@ It is bound to the `build` stage, and uses the following variables:
| ------------------------ | -------------------------------------------------------------------- | ----------------------- |
| `unittest-enabled` / `UNITTEST_ENABLED` | Set to `true` to enable the `unittest` job | _none_ (disabled) |
| `unittest-args` / `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options | _none_ |
| `py-unittest-job-tags` / `PY_UNITTEST_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
......@@ -160,6 +167,7 @@ It is bound to the `build` stage, and uses the following variables:
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
| `pytest-enabled` / `PYTEST_ENABLED` | Set to `true` to enable the `pytest` job | _none_ (disabled) |
| `pytest-args` / `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_ |
| `py-pytest-job-tags` / `PY_PYTEST_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
......@@ -193,6 +201,7 @@ It is bound to the `build` stage, and uses the following variables:
| ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- |
| `nosetests-enabled` / `NOSETESTS_ENABLED` | Set to `true` to enable the `nose` job | _none_ (disabled) |
| `nosetests-args` / `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_ |
| `py-nosetests-job-tags` / `PY_NOSETESTS_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
By default coverage will be run on all the project directories. You can restrict it to your packages by setting the `$NOSE_COVER_PACKAGE` variable.
More [info](https://nose.readthedocs.io/en/latest/plugins/cover.html)
......@@ -216,6 +225,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| --------------------- | ----------------------------------------------------------------------------- | ------------- |
| `compile-args` / `PYTHON_COMPILE_ARGS` | [`compileall` CLI options](https://docs.python.org/3/library/compileall.html) | `*` |
| `py-compile-job-tags` / `PY_COMPILE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### `py-bandit` job (SAST)
......@@ -227,6 +237,7 @@ It is bound to the `test` stage, and uses the following variables:
| ---------------- | ---------------------------------------------------------------------- | ----------------- |
| `bandit-enabled` / `BANDIT_ENABLED` | Set to `true` to enable Bandit analysis | _none_ (disabled) |
| `bandit-args` / `BANDIT_ARGS` | Additional [Bandit CLI options](https://github.com/PyCQA/bandit#usage) | `--recursive .` |
| `py-bandit-job-tags` / `PY_BANDIT_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
In addition to a textual report in the console, this job produces the following reports, kept for one day and only available for download by users with the Developer role or higher:
......@@ -247,7 +258,8 @@ It is bound to the `test` stage, and uses the following variables:
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `trivy-disabled` / `PYTHON_TRIVY_DISABLED` | Set to `true` to disable Trivy job | _none_ (enabled) |
| `trivy-dist-url` / `PYTHON_TRIVY_DIST_URL` | Url to the `tar.gz` package for `linux_amd64` of Trivy to use (ex: `https://github.com/aquasecurity/trivy/releases/download/v0.51.1/trivy_0.51.1_Linux-64bit.tar.gz`)<br/>_When unset, the latest version will be used_ | _none_ |
| `trivy-args` / `PYTHON_TRIVY_ARGS` | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/#options) | `--ignore-unfixed --pkg-types library --detection-priority comprehensive` |
| `trivy-args` / `PYTHON_TRIVY_ARGS` | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/) | `--vuln-type library` |
| `py-trivy-job-tags` / `PY_TRIVY_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
Other Trivy parameters shall be configured using [Trivy environment variables](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/#options).
Examples:
......@@ -277,6 +289,7 @@ It is bound to the `test` stage, and uses the following variables:
| `sbom-syft-url` / `PYTHON_SBOM_SYFT_URL` | Url to the `tar.gz` package for `linux_amd64` of Syft to use (ex: `https://github.com/anchore/syft/releases/download/v0.62.3/syft_0.62.3_linux_amd64.tar.gz`)<br/>_When unset, the latest version will be used_ | _none_ |
| `sbom-name` / `PYTHON_SBOM_NAME` | Component name of the emitted SBOM | `$CI_PROJECT_PATH/$PYTHON_PROJECT_DIR` |
| `sbom-opts` / `PYTHON_SBOM_OPTS` | Options for syft used for SBOM analysis | `--override-default-catalogers python-package-cataloger` |
| `py-sbom-job-tags` / `PY_SBOM_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
In addition to logs in the console, this job produces the following reports, kept for one week:
......@@ -291,6 +304,7 @@ This job **disabled by default** and runs [black](https://black.readthedocs.io)
| Input / Variable | Description | Default value |
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `black-enabled` / `PYTHON_BLACK_ENABLED` | Set to `true` to enable black job | _none_ (disabled) |
| `py-black-job-tags` / `PY_BLACK_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### `py-isort` job
......@@ -299,6 +313,7 @@ This job **disabled by default** and runs [isort](https://pycqa.github.io/isort/
| Input / Variable | Description | Default value |
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `isort-enabled` / `PYTHON_ISORT_ENABLED` | Set to `true` to enable isort job | _none_ (disabled) |
| `py-isort-job-tags` / `PY_ISORT_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### `py-ruff` job
......@@ -308,6 +323,7 @@ This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) o
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `ruff-enabled` / `RUFF_ENABLED` | Set to `true` to enable ruff job | _none_ (disabled) |
| `ruff-args` / `RUFF_ARGS` | Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface) | _none_ |
| `py-ruff-job-tags` / `PY_RUFF_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:warning: Ruff can replace isort, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
......@@ -340,6 +356,7 @@ It is bound to the `build` stage, and uses the following variables:
| `mypy-enabled` / `MYPY_ENABLED` | Set to `true` to enable the `mypy` job | _none_ (disabled) |
| `mypy-args` / `MYPY_ARGS` | Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html) | _none_ |
| `mypy-files` / `MYPY_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
| `py-mypy-job-tags` / `PY_MYPY_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
In addition to a textual report in the console, this job produces the following reports, kept for one day:
......@@ -431,6 +448,7 @@ The publish job is bound to the `publish` stage, is executed on a Git tag matchi
| `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
| `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` |
| :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` |
| `py-release-job-tags` / `PY_RELEASE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
#### Setuptools tip
......@@ -562,9 +580,9 @@ With:
```yaml
include:
# main component
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# Vault variant
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-vault@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-vault@7.7.1
inputs:
vault-base-url: "https://vault.acme.host/v1"
# audience claim for JWT
......@@ -603,13 +621,13 @@ The variant requires the additional configuration parameters:
```yaml
include:
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.12-slim
pytest-enabled: true
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-gcp@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-gcp@7.7.1
inputs:
# common OIDC config for non-prod envs
gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
......@@ -669,13 +687,13 @@ then set the required configuration.
```yaml
include:
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.12-slim
pytest-enabled: true
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-aws-codeartifact@7.7.0
- component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-aws-codeartifact@7.7.1
inputs:
aws-region: "us-east-1"
aws-codeartifact-domain: "acme"
......
kicker.json
View file @ 1efda69d
......@@ -43,6 +43,20 @@
"default": "*",
"advanced": true
},
{
"name": "PY_COMPILE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
},
{
"name": "PY_PUBLISH_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
},
{
"name": "PIP_OPTS",
"description": "pip extra [options](https://pip.pypa.io/en/stable/cli/pip/#general-options)",
......@@ -58,7 +72,16 @@
"id":"package",
"name":"package",
"description":"This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package).",
"enable_with": "PYTHON_PACKAGE_ENABLED"
"enable_with": "PYTHON_PACKAGE_ENABLED",
"variables": [
{
"name": "PY_PACKAGE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id":"publish",
......@@ -81,6 +104,13 @@
"name": "PYLINT_FILES",
"description": "Files or directories to analyse",
"advanced": true
},
{
"name": "PY_LINT_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -94,6 +124,13 @@
"name": "UNITTEST_ARGS",
"description": "Additional xmlrunner/unittest CLI options",
"advanced": true
},
{
"name": "PY_UNITTEST_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -107,6 +144,13 @@
"name": "PYTEST_ARGS",
"description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options",
"advanced": true
},
{
"name": "PY_PYTEST_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -120,6 +164,13 @@
"name": "NOSETESTS_ARGS",
"description": "Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options)",
"advanced": true
},
{
"name": "PY_NOSETESTS_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -134,6 +185,13 @@
"description": "Additional [Bandit CLI options](https://github.com/PyCQA/bandit#usage)",
"default": "--recursive .",
"advanced": true
},
{
"name": "PY_BANDIT_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -153,6 +211,13 @@
"description": "Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/)",
"default": "--ignore-unfixed --pkg-types library --detection-priority comprehensive",
"advanced": true
},
{
"name": "PY_TRIVY_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -178,6 +243,13 @@
"description": "Options for syft used for SBOM analysis",
"default": "--override-default-catalogers python-package-cataloger",
"advanced": true
},
{
"name": "PY_SBOM_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -250,6 +322,13 @@
"description": "Target PyPI repository password credential",
"secret": true,
"default": "$CI_JOB_TOKEN"
},
{
"name": "PY_RELEASE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -257,13 +336,31 @@
"id": "black",
"name": "black",
"description": "Code formatting based on [black](https://black.readthedocs.io)",
"enable_with": "PYTHON_BLACK_ENABLED"
"enable_with": "PYTHON_BLACK_ENABLED",
"variables": [
{
"name": "PY_BLACK_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "isort",
"name": "isort",
"description": "Check imports order with [isort](https://pycqa.github.io/isort)",
"enable_with": "PYTHON_ISORT_ENABLED"
"enable_with": "PYTHON_ISORT_ENABLED",
"variables": [
{
"name": "PY_ISORT_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "ruff",
......@@ -275,6 +372,13 @@
"name": "RUFF_ARGS",
"description": "Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)",
"advanced": true
},
{
"name": "PY_RUFF_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -301,6 +405,13 @@
"name": "MYPY_FILES",
"description": "Files or directories to analyse",
"advanced": true
},
{
"name": "PY_MYPY_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
}
......
templates/gitlab-ci-python-gcp.yml
View file @ 1efda69d
......@@ -44,7 +44,7 @@ variables:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "7.7.0"]
command: ["--service", "python", "7.7.1"]
variables:
GCP_JWT: $GCP_JWT
before_script:
......
templates/gitlab-ci-python-vault.yml
View file @ 1efda69d
......@@ -22,7 +22,7 @@ variables:
.python-base:
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "7.7.0"]
command: ["--service", "python", "7.7.1"]
- name: "$TBC_VAULT_IMAGE"
alias: "vault-secrets-provider"
variables:
......
templates/gitlab-ci-python.yml
View file @ 1efda69d
......@@ -188,6 +188,67 @@ spec:
mypy-files:
description: Files or directories to analyse
default: ''
py-package-job-tags:
description: tags to filter applicable runners for py-package job
type: array
default: []
py-pytest-job-tags:
description: tags to filter applicable runners for py-pytest job
type: array
default: []
py-ruff-job-tags:
description: tags to filter applicable runners for py-ruff job
type: array
default: []
py-release-job-tags:
description: tags to filter applicable runners for py-release job
type: array
default: []
py-compile-job-tags:
description: tags to filter applicable runners for py-compile job
type: array
default: []
py-publish-job-tags:
description: tags to filter applicable runners for py-publish job
type: array
default: []
py-lint-job-tags:
description: tags to filter applicable runners for py-pylint job
type: array
default: []
py-black-job-tags:
description: tags to filter applicable runners for py-black job
type: array
default: []
py-isort-job-tags:
description: tags to filter applicable runners for py-isort job
type: array
default: []
py-mypy-job-tags:
description: tags to filter applicable runners for py-mypy job
type: array
default: []
py-unittest-job-tags:
description: tags to filter applicable runners for py-unittest job
type: array
default: []
py-nosetests-job-tags:
description: tags to filter applicable runners for py-nosetests job
type: array
default: []
py-bandit-job-tags:
description: tags to filter applicable runners for py-bandit job
type: array
default: []
py-trivy-job-tags:
description: tags to filter applicable runners for py-trivy job
type: array
default: []
py-sbom-job-tags:
description: tags to filter applicable runners for py-sbom job
type: array
default: []
---
# default workflow rules: Merge Request pipelines
workflow:
......@@ -971,7 +1032,7 @@ stages:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "7.7.0"]
command: ["--service", "python", "7.7.1"]
variables:
# set local cache dir; most Python tools honour XDG specs
XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"
......@@ -990,7 +1051,6 @@ stages:
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
- cd ${PYTHON_PROJECT_DIR}
- guess_build_system
- mkdir -p -m 777 reports
.python-test:
extends: .python-base
......@@ -1025,11 +1085,13 @@ py-package:
expire_in: 1 day
rules:
- if: '$PYTHON_PACKAGE_ENABLED == "true"'
tags: $[[ inputs.py-package-job-tags ]]
py-lint:
extends: .python-base
stage: build
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install pylint_gitlab # codeclimate reports
# run pylint and generate reports all at once
......@@ -1047,6 +1109,7 @@ py-lint:
- if: '$PYLINT_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-lint-job-tags ]]
py-compile:
extends: .python-base
......@@ -1059,6 +1122,7 @@ py-compile:
- if: '$UNITTEST_ENABLED == "true" || $PYTEST_ENABLED == "true" || $NOSETESTS_ENABLED == "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-compile-job-tags ]]
py-black:
extends: .python-base
......@@ -1072,6 +1136,7 @@ py-black:
- if: '$PYTHON_BLACK_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-black-job-tags ]]
py-isort:
extends: .python-base
......@@ -1085,11 +1150,13 @@ py-isort:
- if: '$PYTHON_ISORT_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-isort-job-tags ]]
py-ruff:
extends: .python-base
stage: build
script:
- mkdir -p -m 777 reports
- |
if [[ ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" ]]; then
log_warn "Ruff can replace isort, Bandit, Pylint"
......@@ -1117,6 +1184,7 @@ py-ruff:
- if: '$RUFF_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-ruff-job-tags ]]
py-ruff-format:
extends: .python-base
......@@ -1141,6 +1209,7 @@ py-mypy:
variables:
MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy"
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install mypy mypy-to-codeclimate
- _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*" -not -path "./.venv/*")} | tee reports/py-mypy.console.txt || true
......@@ -1159,6 +1228,7 @@ py-mypy:
- if: '$MYPY_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-mypy-job-tags ]]
###############################################################################################
# test stage #
......@@ -1166,6 +1236,7 @@ py-mypy:
py-unittest:
extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
# code coverage
- _pip install coverage
......@@ -1179,10 +1250,12 @@ py-unittest:
- if: '$UNITTEST_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-unittest-job-tags ]]
py-pytest:
extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install pytest pytest-cov coverage
- _python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term --cov-report xml:reports/py-coverage.cobertura.xml ${PYTEST_ARGS}
......@@ -1191,10 +1264,12 @@ py-pytest:
- if: '$PYTEST_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-pytest-job-tags ]]
py-nosetests:
extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
- _run nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/py-coverage.cobertura.xml ${NOSETESTS_ARGS}
rules:
......@@ -1202,6 +1277,7 @@ py-nosetests:
- if: '$NOSETESTS_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-nosetests-job-tags ]]
# Bandit (SAST)
py-bandit:
......@@ -1210,6 +1286,7 @@ py-bandit:
# force no dependencies
dependencies: []
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install bandit
# CSV (for SonarQube)
......@@ -1237,6 +1314,7 @@ py-bandit:
- if: '$BANDIT_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-bandit-job-tags ]]
# Trivy (dependency check)
py-trivy:
......@@ -1245,6 +1323,7 @@ py-trivy:
# force no dependencies
dependencies: []
script:
- mkdir -p -m 777 reports
- |
if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]
then
......@@ -1315,6 +1394,7 @@ py-trivy:
- if: '$PYTHON_TRIVY_DISABLED == "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-trivy-job-tags ]]
py-sbom:
extends: .python-base
......@@ -1323,6 +1403,7 @@ py-sbom:
dependencies: []
needs: []
script:
- mkdir -p -m 777 reports
- |
case "$PYTHON_BUILD_SYSTEM" in
poetry*|pipenv*)
......@@ -1370,6 +1451,7 @@ py-sbom:
- if: '$PYTHON_SBOM_DISABLED == "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-sbom-job-tags ]]
# (manual from master branch): triggers a release (tag creation)
py-release:
......@@ -1398,6 +1480,7 @@ py-release:
- if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
when: manual
allow_failure: true
tags: $[[ inputs.py-release-job-tags ]]
# (auto from release tag): publishes the Python package(s) to a PyPi registry
py-publish:
......@@ -1418,3 +1501,4 @@ py-publish:
when: never
# on tag with release pattern: auto
- if: '$CI_COMMIT_TAG =~ $RELEASE_REF'
tags: $[[ inputs.py-publish-job-tags ]]