From a53b81ce4e649dd637a217686745a6f6c6c81ca5 Mon Sep 17 00:00:00 2001 From: will Date: Wed, 4 Feb 2026 10:45:21 +0000 Subject: [PATCH 1/5] lint: switch to uv for python management in linter https://docs.astral.sh/uv/ Install python in the linter using uv and a venv. This is faster and more simple than building pyenv. --- ci/lint/01_install.sh | 21 +++++---------------- ci/lint/06_script.sh | 2 +- ci/lint_imagefile | 4 ++++ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 573dbca5fe5..50615030336 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -22,25 +22,14 @@ ${CI_RETRY_EXE} apt-get update # - moreutils (used by scripted-diff) ${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg moreutils -PYTHON_PATH="/python_build" -if [ ! -d "${PYTHON_PATH}/bin" ]; then - ( - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/pyenv/pyenv.git - cd pyenv/plugins/python-build || exit 1 - ./install.sh - ) - # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment - ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ - clang - env CC=clang python-build "$(cat "/.python-version")" "${PYTHON_PATH}" -fi -export PATH="${PYTHON_PATH}/bin:${PATH}" +# Install Python and create venv using uv (reads version from .python-version) +uv venv /python_env + +export PATH="/python_env/bin:${PATH}" command -v python3 python3 --version -${CI_RETRY_EXE} pip3 install \ +uv pip install --python /python_env \ lief==0.17.5 \ mypy==1.19.1 \ pyzmq==27.1.0 \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 1b36fada9fc..a0f2dc88bc4 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -12,7 +12,7 @@ set -o errexit -o pipefail -o xtrace # of the mounted bitcoin src dir. git config --global --add safe.directory /bitcoin -export PATH="/python_build/bin:${PATH}" +export PATH="/python_env/bin:${PATH}" if [ -n "${LINT_CI_IS_PR}" ]; then export COMMIT_RANGE="HEAD~..HEAD" diff --git a/ci/lint_imagefile b/ci/lint_imagefile index 77e9688c650..d050153f8ad 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -6,6 +6,10 @@ FROM mirror.gcr.io/ubuntu:24.04 +# Pin uv to minor version to avoid breaking changes: +# https://docs.astral.sh/uv/reference/policies/versioning/ +COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /uvx /bin/ + COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version COPY ./ci/lint/01_install.sh /install.sh From 5f4d3383daa064d188ef8c6b1d9adbad3a67bcb6 Mon Sep 17 00:00:00 2001 From: will Date: Wed, 4 Feb 2026 10:45:24 +0000 Subject: [PATCH 2/5] lint: switch to ruff for formatting and linting - use dedicated ruff.toml for configuration - download via docker image layer at build time --- ci/lint/01_install.sh | 3 +- ci/lint_imagefile | 4 ++- ruff.toml | 42 +++++++++++++++++++++++++ test/lint/test_runner/src/lint_py.rs | 47 +--------------------------- 4 files changed, 47 insertions(+), 49 deletions(-) create mode 100644 ruff.toml diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 50615030336..0d9ccfbbac5 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -32,8 +32,7 @@ python3 --version uv pip install --python /python_env \ lief==0.17.5 \ mypy==1.19.1 \ - pyzmq==27.1.0 \ - ruff==0.15.5 + pyzmq==27.1.0 SHELLCHECK_VERSION=v0.11.0 curl --fail -L "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.$(uname --machine).tar.xz" | \ diff --git a/ci/lint_imagefile b/ci/lint_imagefile index d050153f8ad..7dc21203f53 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -6,9 +6,11 @@ FROM mirror.gcr.io/ubuntu:24.04 -# Pin uv to minor version to avoid breaking changes: +# Pin uv and ruff to minor version to avoid breaking changes # https://docs.astral.sh/uv/reference/policies/versioning/ +# https://docs.astral.sh/ruff/versioning/ COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/ruff:0.15 /ruff /bin/ COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000000..10df372c7a8 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,42 @@ +[lint] +select = [ + "B006", # mutable-argument-default + "B008", # function-call-in-default-argument + "E101", # indentation contains mixed spaces and tabs + "E401", # multiple imports on one line + "E402", # module level import not at top of file + "E701", # multiple statements on one line (colon) + "E702", # multiple statements on one line (semicolon) + "E703", # statement ends with a semicolon + "E711", # comparison to None should be 'if cond is None:' + "E713", # test for membership should be "not in" + "E714", # test for object identity should be "is not" + "E721", # do not compare types, use "isinstance()" + "E722", # do not use bare 'except' + "E742", # do not define classes named "l", "O", or "I" + "E743", # do not define functions named "l", "O", or "I" + "F401", # module imported but unused + "F402", # import module from line N shadowed by loop variable + "F403", # 'from foo_module import *' used; unable to detect undefined names + "F404", # future import(s) name after other statements + "F405", # foo_function may be undefined, or defined from star imports: bar_module + "F406", # "from module import *" only allowed at module level + "F407", # an undefined __future__ feature name was imported + "F541", # f-string without any placeholders + "F601", # dictionary key name repeated with different values + "F602", # dictionary key variable name repeated with different values + "F621", # too many expressions in an assignment with star-unpacking + "F631", # assertion test is a tuple, which are always True + "F632", # use ==/!= to compare str, bytes, and int literals + "F811", # redefinition of unused name from line N + "F821", # undefined name 'Foo' + "F822", # undefined name name in __all__ + "F823", # local variable name referenced before assignment + "F841", # local variable 'foo' is assigned to but never used + "PLE", # Pylint errors + "W191", # indentation contains tabs + "W291", # trailing whitespace + "W292", # no newline at end of file + "W293", # blank line contains whitespace + "W605", # invalid escape sequence "x" +] diff --git a/test/lint/test_runner/src/lint_py.rs b/test/lint/test_runner/src/lint_py.rs index d9177c86907..1d0ebed9b4f 100644 --- a/test/lint/test_runner/src/lint_py.rs +++ b/test/lint/test_runner/src/lint_py.rs @@ -9,51 +9,6 @@ use crate::util::{check_output, get_pathspecs_default_excludes, git, LintResult} pub fn lint_py_lint() -> LintResult { let bin_name = "ruff"; - let checks = format!( - "--select={}", - [ - "B006", // mutable-argument-default - "B008", // function-call-in-default-argument - "E101", // indentation contains mixed spaces and tabs - "E401", // multiple imports on one line - "E402", // module level import not at top of file - "E701", // multiple statements on one line (colon) - "E702", // multiple statements on one line (semicolon) - "E703", // statement ends with a semicolon - "E711", // comparison to None should be 'if cond is None:' - "E713", // test for membership should be "not in" - "E714", // test for object identity should be "is not" - "E721", // do not compare types, use "isinstance()" - "E722", // do not use bare 'except' - "E742", // do not define classes named "l", "O", or "I" - "E743", // do not define functions named "l", "O", or "I" - "F401", // module imported but unused - "F402", // import module from line N shadowed by loop variable - "F403", // 'from foo_module import *' used; unable to detect undefined names - "F404", // future import(s) name after other statements - "F405", // foo_function may be undefined, or defined from star imports: bar_module - "F406", // "from module import *" only allowed at module level - "F407", // an undefined __future__ feature name was imported - "F541", // f-string without any placeholders - "F601", // dictionary key name repeated with different values - "F602", // dictionary key variable name repeated with different values - "F621", // too many expressions in an assignment with star-unpacking - "F631", // assertion test is a tuple, which are always True - "F632", // use ==/!= to compare str, bytes, and int literals - "F811", // redefinition of unused name from line N - "F821", // undefined name 'Foo' - "F822", // undefined name name in __all__ - "F823", // local variable name … referenced before assignment - "F841", // local variable 'foo' is assigned to but never used - "PLE", // Pylint errors - "W191", // indentation contains tabs - "W291", // trailing whitespace - "W292", // no newline at end of file - "W293", // blank line contains whitespace - "W605", // invalid escape sequence "x" - ] - .join(",") - ); let files = check_output( git() .args(["ls-files", "--", "*.py"]) @@ -61,7 +16,7 @@ pub fn lint_py_lint() -> LintResult { )?; let mut cmd = Command::new(bin_name); - cmd.args(["check", &checks]).args(files.lines()); + cmd.arg("check").args(files.lines()); match cmd.status() { Ok(status) if status.success() => Ok(()), From fd15b55c2ef607525d45f26ab3e7f3fc600e29af Mon Sep 17 00:00:00 2001 From: will Date: Mon, 9 Feb 2026 15:46:40 +0000 Subject: [PATCH 3/5] lint: use requirements.txt --- ci/lint/01_install.sh | 5 +---- ci/lint/requirements.txt | 3 +++ ci/lint_imagefile | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 ci/lint/requirements.txt diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 0d9ccfbbac5..9c2020a9a15 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -29,10 +29,7 @@ export PATH="/python_env/bin:${PATH}" command -v python3 python3 --version -uv pip install --python /python_env \ - lief==0.17.5 \ - mypy==1.19.1 \ - pyzmq==27.1.0 +uv pip install --python /python_env --requirements /ci/lint/requirements.txt SHELLCHECK_VERSION=v0.11.0 curl --fail -L "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.$(uname --machine).tar.xz" | \ diff --git a/ci/lint/requirements.txt b/ci/lint/requirements.txt new file mode 100644 index 00000000000..e8abf041dc7 --- /dev/null +++ b/ci/lint/requirements.txt @@ -0,0 +1,3 @@ +lief==0.17.5 +mypy==1.19.1 +pyzmq==27.1.0 diff --git a/ci/lint_imagefile b/ci/lint_imagefile index 7dc21203f53..a6c49a7c861 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -14,6 +14,7 @@ COPY --from=ghcr.io/astral-sh/ruff:0.15 /ruff /bin/ COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version +COPY ./ci/lint/requirements.txt /ci/lint/requirements.txt COPY ./ci/lint/01_install.sh /install.sh RUN /install.sh && \ From 5fefa5a654e2a1146c8abc9673e72d7bcbf2f757 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Fri, 25 Jul 2025 13:18:36 +0200 Subject: [PATCH 4/5] Don't pin Python patch version .python-version always matches the minimum supported Python version. It's main purpose is to catch accidental use of too modern syntax in scripts and functional tests. We (currently) don't specify a minimum patch version, so it's not necessary to do so here. The minor verion is enough. This also avoids requiring users to keep a potentially unsafe old patch version installed. --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index 1445aee866c..c8cfe395918 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.14 +3.10 From 2424e5283672e10bc45cdbca1a8851308716b50c Mon Sep 17 00:00:00 2001 From: will Date: Sat, 14 Mar 2026 09:58:13 +0000 Subject: [PATCH 5/5] lint: doc: detail lint tool install methods Installing tools in the dockerfile using `COPY --from` is better , but not all tools we use publish an OCI image to a non-docker.io registry. As we are frequently rate-limited from docker.io, only install tools which publish to another registry, e.g. ghcr.io. --- test/lint/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lint/README.md b/test/lint/README.md index 08703ab8efb..d060df110e6 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -56,7 +56,7 @@ or `--help`: | `py_lint` | [ruff](https://github.com/astral-sh/ruff) | markdown link check | [mlc](https://github.com/becheran/mlc) -In use versions and install instructions are available in the [CI setup](../../ci/lint/01_install.sh). +Dependency versions and installation instructions are available in the [CI setup](../../ci/lint/01_install.sh) and the [lint_imagefile](../../ci/lint_imagefile) (for tools where an OCI imagefile exists). Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated.