highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-11-12 06:58:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge #20560: fuzz: Link all targets once

Browse Source 
fa13e1b0c5 build: Add option --enable-danger-fuzz-link-all (MarcoFalke)
44444ba759 fuzz: Link all targets once (MarcoFalke)

Pull request description:

  Currently the linker is invoked more than 150 times when compiling with `--enable-fuzz`. This is problematic for several reasons:

  * It wastes disk space north of 20 GB, as all libraries and sanitizers are linked more than 150 times
  * It wastes CPU time, as the link step can practically not be cached (similar to ccache for object files)
  * It makes it a blocker to compile the fuzz tests by default for non-fuzz builds #19388, for the aforementioned reasons
  * The build file is several thousand lines of code, without doing anything meaningful except listing each fuzz target in a highly verbose manner
  * It makes writing new fuzz tests unnecessarily hard, as build system knowledge is required; Compare that to boost unit tests, which can be added by simply editing an existing cpp file
  * It encourages fuzz tests that re-use the `buffer` or assume the `buffer` to be concatenations of seeds, which increases complexity of seeds and complexity for the fuzz engine to explore; Thus reducing the effectiveness of the affected fuzz targets

  Fixes #20088

ACKs for top commit:
  practicalswift:
    Tested ACK fa13e1b0c5
  sipa:
    ACK fa13e1b0c5. Reviewed the code changes, and tested the 3 different test_runner.py modes (run once, merge, generate). I also tested building with the new --enable-danger-fuzz-link-all

Tree-SHA512: 962ab33269ebd51810924c51266ecc62edd6ddf2fcd9a8c359ed906766f58c3f73c223f8d3cc49f2c60f0053f65e8bdd86ce9c19e673f8c2b3cd676e913f2642
This commit is contained in:
MarcoFalke
2020-12-15 18:59:52 +01:00
parent a35a3466ef fa13e1b0c5
commit 8bb40d5f56
100 changed files with 476 additions and 1307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
test/fuzz/test_runner.py
View File

@@ -83,7 +83,7 @@ def main():
sys.exit(1)
# Build list of tests
test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
test_list_all = parse_test_list(fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'))
if not test_list_all:
logging.error("No fuzz targets found")
@@ -126,9 +126,12 @@ def main():
try:
help_output = subprocess.run(
args=[
os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'),
'-help=1',
],
env={
'FUZZ': test_list_selection[0]
},
timeout=20,
check=True,
stderr=subprocess.PIPE,
@@ -177,24 +180,30 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets):
"""
logging.info("Generating corpus seeds to {}".format(seed_dir))
def job(command):
def job(command, t):
logging.debug("Running '{}'\n".format(" ".join(command)))
logging.debug("Command '{}' output:\n'{}'\n".format(
' '.join(command),
subprocess.run(command, check=True, stderr=subprocess.PIPE,
universal_newlines=True).stderr
))
subprocess.run(
command,
env={
'FUZZ': t
},
check=True,
stderr=subprocess.PIPE,
universal_newlines=True,
).stderr))
futures = []
for target in targets:
target_seed_dir = os.path.join(seed_dir, target)
os.makedirs(target_seed_dir, exist_ok=True)
command = [
os.path.join(build_dir, "src", "test", "fuzz", target),
os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
"-runs=100000",
target_seed_dir,
]
futures.append(fuzz_pool.submit(job, command))
futures.append(fuzz_pool.submit(job, command, target))
for future in as_completed(futures):
future.result()
@@ -205,7 +214,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
jobs = []
for t in test_list:
args = [
os.path.join(build_dir, 'src', 'test', 'fuzz', t),
os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
'-merge=1',
'-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
os.path.join(corpus, t),
@@ -216,7 +225,15 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir):
def job(t, args):
output = 'Run {} with args {}\n'.format(t, " ".join(args))
output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
output += subprocess.run(
args,
env={
'FUZZ': t
},
check=True,
stderr=subprocess.PIPE,
universal_newlines=True,
).stderr
logging.debug(output)
jobs.append(fuzz_pool.submit(job, t, args))
@@ -231,7 +248,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
corpus_path = os.path.join(corpus, t)
os.makedirs(corpus_path, exist_ok=True)
args = [
os.path.join(build_dir, 'src', 'test', 'fuzz', t),
os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'),
'-runs=1',
corpus_path,
]
@@ -240,7 +257,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
def job(t, args):
output = 'Run {} with args {}'.format(t, args)
result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True)
output += result.stderr
return output, result
@@ -260,20 +277,16 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind):
sys.exit(1)
def parse_test_list(makefile):
with open(makefile, encoding='utf-8') as makefile_test:
test_list_all = []
read_targets = False
for line in makefile_test.readlines():
line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
if read_targets:
if not line:
break
test_list_all.append(line)
continue
if line == 'FUZZ_TARGETS =':
read_targets = True
def parse_test_list(*, fuzz_bin):
test_list_all = subprocess.run(
fuzz_bin,
env={
'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': ''
},
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
universal_newlines=True,
).stdout.splitlines()
return test_list_all