highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-06-03 17:54:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge bitcoin/bitcoin#30723: lint: Speed up and fix flake8 checks

Browse Source 
fafdb7df34 lint: Speed up flake8 checks (MarcoFalke)
faf17df7fb lint: Document missing py_lint dependency (MarcoFalke)
faebeb828f lint: Remove python whitespace and shadowing lint rules (MarcoFalke)
7777047835 lint: Remove python lint rules that are SyntaxError (MarcoFalke)
faaf3e53f0 test: [refactor] Fix F841 flake8 (MarcoFalke)
444421db69 test: [refactor] Fix E714 pycodestyle (MarcoFalke)

Pull request description:

  The checks have many issues:

  * Some checks that could in theory hide bugs are not applied -> Fix them and apply them going forward
  * Some checks are redundant Python 2 checks, or of low value -> Remove them
  * The checks are slow -> Speed them up from ~10 seconds to about ~20 milliseconds

ACKs for top commit:
  davidgumberg:
    review and tested reACK fafdb7df34
  kevkevinpal:
    ACK [fafdb7d](fafdb7df34)
  achow101:
    ACK fafdb7df34

Tree-SHA512: a0488b722cfaf7071bd6848cd3be002e0b6c38af80d8b5cbb08613c0b174ef63277289f960db8ac31adb09fe563a4973203b8fb10b83cbcfdc6f0ef39bd04410
This commit is contained in:
Ava Chow
2024-09-04 15:35:52 -04:00
parent 5373aa30e2 fafdb7df34
commit f640b323bd
13 changed files with 80 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/lint/README.md
View File

@@ -45,13 +45,13 @@ or `--help`:
| Lint test | Dependency |
|-----------|:----------:|
| [`lint-python.py`](/test/lint/lint-python.py) | [flake8](https://github.com/PyCQA/flake8)
| [`lint-python.py`](/test/lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF)
| [`lint-python.py`](/test/lint/lint-python.py) | [mypy](https://github.com/python/mypy)
| [`lint-python.py`](/test/lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq)
| [`lint-python-dead-code.py`](/test/lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture)
| [`lint-shell.py`](/test/lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck)
| [`lint-spelling.py`](/test/lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell)
| `py_lint` | [ruff](https://github.com/astral-sh/ruff)
| markdown link check | [mlc](https://github.com/becheran/mlc)
In use versions and install instructions are available in the [CI setup](../../ci/lint/04_install.sh).

96
test/lint/lint-python.py
View File

@@ -5,13 +5,12 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""
Check for specified flake8 and mypy warnings in python files.
Check for specified mypy warnings in python files.
"""
import os
from pathlib import Path
import subprocess
import sys
from importlib.metadata import metadata, PackageNotFoundError
@@ -19,89 +18,12 @@ from importlib.metadata import metadata, PackageNotFoundError
cache_dir = Path(__file__).parent.parent / ".mypy_cache"
os.environ["MYPY_CACHE_DIR"] = str(cache_dir)
DEPS = ['flake8', 'lief', 'mypy', 'pyzmq']
# All .py files, except those in src/ (to exclude subtrees there)
FLAKE_FILES_ARGS = ['git', 'ls-files', '*.py', ':!:src/*.py']
DEPS = ['lief', 'mypy', 'pyzmq']
# Only .py files in test/functional and contrib/devtools have type annotations
# enforced.
MYPY_FILES_ARGS = ['git', 'ls-files', 'test/functional/*.py', 'contrib/devtools/*.py']
ENABLED = (
'E101,' # indentation contains mixed spaces and tabs
'E112,' # expected an indented block
'E113,' # unexpected indentation
'E115,' # expected an indented block (comment)
'E116,' # unexpected indentation (comment)
'E125,' # continuation line with same indent as next logical line
'E129,' # visually indented line with same indent as next logical line
'E131,' # continuation line unaligned for hanging indent
'E133,' # closing bracket is missing indentation
'E223,' # tab before operator
'E224,' # tab after operator
'E242,' # tab after ','
'E266,' # too many leading '#' for block comment
'E271,' # multiple spaces after keyword
'E272,' # multiple spaces before keyword
'E273,' # tab after keyword
'E274,' # tab before keyword
'E275,' # missing whitespace after keyword
'E304,' # blank lines found after function decorator
'E306,' # expected 1 blank line before a nested definition
'E401,' # multiple imports on one line
'E402,' # module level import not at top of file
'E502,' # the backslash is redundant between brackets
'E701,' # multiple statements on one line (colon)
'E702,' # multiple statements on one line (semicolon)
'E703,' # statement ends with a semicolon
'E711,' # comparison to None should be 'if cond is None:'
'E714,' # test for object identity should be "is not"
'E721,' # do not compare types, use "isinstance()"
'E722,' # do not use bare 'except'
'E742,' # do not define classes named "l", "O", or "I"
'E743,' # do not define functions named "l", "O", or "I"
'E901,' # SyntaxError: invalid syntax
'E902,' # TokenError: EOF in multi-line string
'F401,' # module imported but unused
'F402,' # import module from line N shadowed by loop variable
'F403,' # 'from foo_module import *' used; unable to detect undefined names
'F404,' # future import(s) name after other statements
'F405,' # foo_function may be undefined, or defined from star imports: bar_module
'F406,' # "from module import *" only allowed at module level
'F407,' # an undefined __future__ feature name was imported
'F601,' # dictionary key name repeated with different values
'F602,' # dictionary key variable name repeated with different values
'F621,' # too many expressions in an assignment with star-unpacking
'F622,' # two or more starred expressions in an assignment (a, *b, *c = d)
'F631,' # assertion test is a tuple, which are always True
'F632,' # use ==/!= to compare str, bytes, and int literals
'F701,' # a break statement outside of a while or for loop
'F702,' # a continue statement outside of a while or for loop
'F703,' # a continue statement in a finally block in a loop
'F704,' # a yield or yield from statement outside of a function
'F705,' # a return statement with arguments inside a generator
'F706,' # a return statement outside of a function/method
'F707,' # an except: block as not the last exception handler
'F811,' # redefinition of unused name from line N
'F812,' # list comprehension redefines 'foo' from line N
'F821,' # undefined name 'Foo'
'F822,' # undefined name name in __all__
'F823,' # local variable name … referenced before assignment
'F831,' # duplicate argument name in function definition
'F841,' # local variable 'foo' is assigned to but never used
'W191,' # indentation contains tabs
'W291,' # trailing whitespace
'W292,' # no newline at end of file
'W293,' # blank line contains whitespace
'W601,' # .has_key() is deprecated, use "in"
'W602,' # deprecated form of raising exception
'W603,' # "<>" is deprecated, use "!="
'W604,' # backticks are deprecated, use "repr()"
'W605,' # invalid escape sequence "x"
'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7
)
def check_dependencies():
for dep in DEPS:
@@ -115,20 +37,6 @@ def check_dependencies():
def main():
check_dependencies()
if len(sys.argv) > 1:
flake8_files = sys.argv[1:]
else:
flake8_files = subprocess.check_output(FLAKE_FILES_ARGS).decode("utf-8").splitlines()
flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files
flake8_env = os.environ.copy()
flake8_env["PYTHONWARNINGS"] = "ignore"
try:
subprocess.check_call(flake8_args, env=flake8_env)
except subprocess.CalledProcessError:
exit(1)
mypy_files = subprocess.check_output(MYPY_FILES_ARGS).decode("utf-8").splitlines()
mypy_args = ['mypy', '--show-error-codes'] + mypy_files

56
test/lint/test_runner/src/main.rs
View File

@@ -36,9 +36,9 @@ fn get_linter_list() -> Vec<&'static Linter> {
lint_fn: lint_markdown
},
&Linter {
description: "Check the default arguments in python",
name: "py_mut_arg_default",
lint_fn: lint_py_mut_arg_default,
description: "Lint Python code",
name: "py_lint",
lint_fn: lint_py_lint,
},
&Linter {
description: "Check that std::filesystem is not used directly",
@@ -185,12 +185,50 @@ fn lint_subtree() -> LintResult {
}
}
fn lint_py_mut_arg_default() -> LintResult {
fn lint_py_lint() -> LintResult {
let bin_name = "ruff";
let checks = ["B006", "B008"]
.iter()
.map(|c| format!("--select={}", c))
.collect::<Vec<_>>();
let checks = format!(
"--select={}",
[
"B006", // mutable-argument-default
"B008", // function-call-in-default-argument
"E101", // indentation contains mixed spaces and tabs
"E401", // multiple imports on one line
"E402", // module level import not at top of file
"E701", // multiple statements on one line (colon)
"E702", // multiple statements on one line (semicolon)
"E703", // statement ends with a semicolon
"E711", // comparison to None should be 'if cond is None:'
"E714", // test for object identity should be "is not"
"E721", // do not compare types, use "isinstance()"
"E722", // do not use bare 'except'
"E742", // do not define classes named "l", "O", or "I"
"E743", // do not define functions named "l", "O", or "I"
"F401", // module imported but unused
"F402", // import module from line N shadowed by loop variable
"F403", // 'from foo_module import *' used; unable to detect undefined names
"F404", // future import(s) name after other statements
"F405", // foo_function may be undefined, or defined from star imports: bar_module
"F406", // "from module import *" only allowed at module level
"F407", // an undefined __future__ feature name was imported
"F601", // dictionary key name repeated with different values
"F602", // dictionary key variable name repeated with different values
"F621", // too many expressions in an assignment with star-unpacking
"F631", // assertion test is a tuple, which are always True
"F632", // use ==/!= to compare str, bytes, and int literals
"F811", // redefinition of unused name from line N
"F821", // undefined name 'Foo'
"F822", // undefined name name in __all__
"F823", // local variable name … referenced before assignment
"F841", // local variable 'foo' is assigned to but never used
"W191", // indentation contains tabs
"W291", // trailing whitespace
"W292", // no newline at end of file
"W293", // blank line contains whitespace
"W605", // invalid escape sequence "x"
]
.join(",")
);
let files = check_output(
git()
.args(["ls-files", "--", "*.py"])
@@ -198,7 +236,7 @@ fn lint_py_mut_arg_default() -> LintResult {
)?;
let mut cmd = Command::new(bin_name);
cmd.arg("check").args(checks).args(files.lines());
cmd.args(["check", &checks]).args(files.lines());
match cmd.status() {
Ok(status) if status.success() => Ok(()),