highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-04-21 12:18:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge bitcoin/bitcoin#25528: ci: run USDT interface tests in the CI

Browse Source 
cc7335edc8 ci: run USDT interface test in a VM (0xb10c)
dba6f82342 test: adopt USDT utxocache interface tests (0xb10c)
220a5a2841 test: hook into PID in tracing tests (0xb10c)

Pull request description:

  Changes a CI task that runs test the previously not run `test/functional/interface_usdt_*.py` functional tests (added in https://github.com/bitcoin/bitcoin/pull/24358).

  This task is run as CirussCI `compute_engine_instance` VM as hooking into the tracepoints is not possible in CirrusCI docker containers (https://github.com/bitcoin/bitcoin/issues/23296#issuecomment-1024920845). We use an unoffical PPA and untrusted  `bpfcc-tools` package in the CI as the Ubuntu jammy and Debian bullseye packages are outdated. We hope use an official package when new Ubuntu/Debian releases are available for the use with Google Compute Engine.

  We make sure to hook into `bitcoind` binaries in USDT interface tests via their PID, instead of their path. This makes sure multiple functional tests running in parallel don't interfere with each other.

  The utxocache USDT interface tests is adopted to a change of the functional test framework that wasn't detected as the tests weren't run in the CI. As the tracepoints expose internals, it can happen that we need to adopt the interface test when internals change. This is a bit awkward, and if it happens to frequently, we should consider generalizing the tests a bit more. For now it's fine, I think.

  See the individual commit messages for more details on the changes.

  Fixes https://github.com/bitcoin/bitcoin/issues/24782
  Fixes https://github.com/bitcoin/bitcoin/issues/23296

  I'd like to hear from reviewers:
  - Are we OK with using the [`hadret/bpfcc`](https://launchpad.net/~hadret/+archive/ubuntu/bpfcc) PPA for now? There is a clear plan when to drop it and as is currently, it could only impact the newly added VM task.
  - ~~Adding a new task increases CI runtime and costs. Should an existing `container` CI task be ported to a VM and reused instead?~~ Yes, see https://github.com/bitcoin/bitcoin/pull/25528#issuecomment-1179509525

ACKs for top commit:
  MarcoFalke:
    cr ACK cc7335edc8

Tree-SHA512: b7fddccc0a77d82371229d048abe0bf2c4ccaa45906497ef3040cf99e7f05561890aef4c253c40e4afc96bb838c9787fae81c8454c6fd9db583276e005a4ccb3
This commit is contained in:
MacroFake
2022-08-01 11:27:23 +02:00
parent da23320998 cc7335edc8
commit eeb5a94e27
7 changed files with 52 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/functional/interface_usdt_net.py
View File

@@ -109,7 +109,7 @@ class NetTracepointTest(BitcoinTestFramework):
self.log.info(
"hook into the net:inbound_message and net:outbound_message tracepoints")
ctx = USDT(path=str(self.options.bitcoind))
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="net:inbound_message",
fn_name="trace_inbound_message")
ctx.enable_probe(probe="net:outbound_message",

40
test/functional/interface_usdt_utxocache.py
View File

@@ -173,7 +173,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16)
self.log.info("hooking into the utxocache:uncache tracepoint")
ctx = USDT(path=str(self.options.bitcoind))
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="utxocache:uncache",
fn_name="trace_utxocache_uncache")
bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
@@ -238,7 +238,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
self.log.info(
"hook into the utxocache:add and utxocache:spent tracepoints")
ctx = USDT(path=str(self.options.bitcoind))
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add")
ctx.enable_probe(probe="utxocache:spent",
fn_name="trace_utxocache_spent")
@@ -334,7 +334,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
self.log.info("test the utxocache:flush tracepoint API")
self.log.info("hook into the utxocache:flush tracepoint")
ctx = USDT(path=str(self.options.bitcoind))
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="utxocache:flush",
fn_name="trace_utxocache_flush")
bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
@@ -345,16 +345,17 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
# that the handle_* functions succeeded.
EXPECTED_HANDLE_FLUSH_SUCCESS = 3
handle_flush_succeeds = 0
possible_cache_sizes = set()
expected_flushes = []
expected_flushes = list()
def handle_utxocache_flush(_, data, __):
nonlocal handle_flush_succeeds
event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents
self.log.info(f"handle_utxocache_flush(): {event}")
expected = expected_flushes.pop(0)
assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode])
possible_cache_sizes.remove(event.size) # fails if size not in set
expected_flushes.remove({
"mode": FLUSHMODE_NAME[event.mode],
"for_prune": event.for_prune,
"size": event.size
})
# sanity checks only
assert(event.memory > 0)
assert(event.duration > 0)
@@ -363,20 +364,19 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
self.log.info("stop the node to flush the UTXO cache")
UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified
UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified
# A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE
# UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the
# second flush, however it can happen that the order changes.
possible_cache_sizes = {UTXOS_IN_CACHE, 0}
flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False}
expected_flushes.extend([flush_for_shutdown, flush_for_shutdown])
expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": UTXOS_IN_CACHE})
expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": 0})
self.stop_node(0)
bpf.perf_buffer_poll(timeout=200)
bpf.cleanup()
self.log.info("check that we don't expect additional flushes")
assert_equal(0, len(expected_flushes))
assert_equal(0, len(possible_cache_sizes))
self.log.info("restart the node with -prune")
self.start_node(0, ["-fastprune=1", "-prune=1"])
@@ -384,12 +384,17 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
BLOCKS_TO_MINE = 350
self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune")
self.generate(self.wallet, BLOCKS_TO_MINE)
# we added BLOCKS_TO_MINE coinbase UTXOs to the cache
possible_cache_sizes = {BLOCKS_TO_MINE}
expected_flushes.append(
{"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE})
self.log.info("test the utxocache:flush tracepoint API with pruning")
self.log.info("hook into the utxocache:flush tracepoint")
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="utxocache:flush",
fn_name="trace_utxocache_flush")
bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
self.log.info(f"prune blockchain to trigger a flush for pruning")
expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0})
self.nodes[0].pruneblockchain(315)
bpf.perf_buffer_poll(timeout=500)
@@ -398,7 +403,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
self.log.info(
f"check that we don't expect additional flushes and that the handle_* function succeeded")
assert_equal(0, len(expected_flushes))
assert_equal(0, len(possible_cache_sizes))
assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds)

2
test/functional/interface_usdt_validation.py
View File

@@ -94,7 +94,7 @@ class ValidationTracepointTest(BitcoinTestFramework):
expected_blocks = list()
self.log.info("hook into the validation:block_connected tracepoint")
ctx = USDT(path=str(self.options.bitcoind))
ctx = USDT(pid=self.nodes[0].process.pid)
ctx.enable_probe(probe="validation:block_connected",
fn_name="trace_block_connected")
bpf = BPF(text=validation_blockconnected_program,