Compare commits

...

8 Commits

Author SHA1 Message Date
Cory Fields
eb5ea9623d
Merge dfc074cd5c1e499b27849969cec61f98f3ce6adc into 5f4422d68dc3530c353af1f87499de1c864b60ad 2025-03-17 03:52:17 +01:00
merge-script
5f4422d68d
Merge bitcoin/bitcoin#32010: qa: Fix TxIndex race conditions
3301d2cbe8c3b76c97285d75fa59637cb6952d0b qa: Wait for txindex to avoid race condition (Hodlinator)
9bfb0d75ba10591cc6c9620f9fd1ecc0e55e7a48 qa: Remove unnecessary -txindex args (Hodlinator)
7ac281c19cd3d11f316dbbb3308eabf1ad4f26d6 qa: Add missing coverage of corrupt indexes (Hodlinator)

Pull request description:

  - Add synchronization in 3 places where if the Transaction Index happens to be slow, we get rare test failures when querying it for transactions (one such case experienced on Windows, prompting investigation).
  - Remove unnecessary TxIndex initialization in some tests.
  - Add some test coverage where TxIndex aspect could be tested in feature_init.py.

ACKs for top commit:
  fjahr:
    re-ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  mzumsande:
    Code Review ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  furszy:
    Code review ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  Prabhat1308:
    Concept ACK [`3301d2c`](3301d2cbe8)

Tree-SHA512: 7c2019e38455f344856aaf6b381faafbd88d53dc88d13309deb718c1dcfbee4ccca7c7f1b66917395503a6f94c3b216a007ad432cc8b93d0309db9805f38d602
2025-03-17 10:28:14 +08:00
Hodlinator
3301d2cbe8
qa: Wait for txindex to avoid race condition
Can be verified to be necessary through adding std::this_thread::sleep_for(0.5s) at the beginning of TxIndex::CustomAppend.
2025-03-10 15:24:16 +01:00
Hodlinator
9bfb0d75ba
qa: Remove unnecessary -txindex args
(Parent commit ensured indexes in feature_init.py are actually used, otherwise they would be removed here as well).
2025-03-07 22:22:31 +01:00
Hodlinator
7ac281c19c
qa: Add missing coverage of corrupt indexes 2025-03-07 22:22:31 +01:00
Cory Fields
dfc074cd5c build: create Depends build type for depends and use it by default
Rather than modifying the existing build types and adding flags for each,
create a new type specifically for depends.

This allows the forwarding of optional flags into the CMake build for the
"Depends" build type, but the user can now optionally use an existing
(Debug/RelWitDebInfo/etc.) type to ignore the optimization flags set by depends
and use the ones from that type instead.

As an example, a user may do:
make -C depends
cmake --toolchain depends/arm64-apple-darwin/toolchain.cmake -DCMAKE_BUILD_TYPE=Debug

This would compile depends with the default optimization flags for depends
(-O2) but build Core with the default optimization flags from CMake.

Depends note:
For hosts, $host_*FLAGS variables now represent the mandatory flags for
compiling and will be forwarded to all Core builds in the toolchain file,
regardless of the build type. For most platforms they should be empty, but
are useful at least for darwin.

When setting (for example) CFLAGS from the command line when building depends,
these flags will be stored in $host_release_CFLAGS (or _debug_), and will only
be forwarded to the Depends build type.
2025-02-20 21:05:41 +00:00
Cory Fields
f7906de7e8 depends: set env-provided flags to per-release
Users can now safely set flags and have them override the optional
per-release-type flags.

This also means that user-provided flags will now be forwarded to CMake's
build-type flags variables so that they'll correctly override the existing
values.
2025-02-20 20:39:38 +00:00
Cory Fields
40a01e9d30 depends: hard-code necessary c(xx)flags rather than setting them per-host
The per-host flag values now represent the mandatory flags that cannot be
overridden by the environment. Additionally, these flags (-pipe and -std=xx)
will no longer be passed into the CMake build.
2025-02-20 20:39:38 +00:00
17 changed files with 82 additions and 62 deletions

View File

@ -205,10 +205,10 @@ check_linker_supports_pie(configure_warnings)
# It is a usage requirement for all targets except for secp256k1, which
# gets its flags by other means.
add_library(core_interface INTERFACE)
add_library(core_interface_relwithdebinfo INTERFACE)
add_library(core_interface_depends INTERFACE)
add_library(core_interface_debug INTERFACE)
target_link_libraries(core_interface INTERFACE
$<$<CONFIG:RelWithDebInfo>:core_interface_relwithdebinfo>
$<$<CONFIG:Depends>:core_interface_depends>
$<$<CONFIG:Debug>:core_interface_debug>
)
@ -576,8 +576,7 @@ else()
endif()
target_compile_definitions(core_interface INTERFACE ${DEPENDS_COMPILE_DEFINITIONS})
target_compile_definitions(core_interface_relwithdebinfo INTERFACE ${DEPENDS_COMPILE_DEFINITIONS_RELWITHDEBINFO})
target_compile_definitions(core_interface_debug INTERFACE ${DEPENDS_COMPILE_DEFINITIONS_DEBUG})
target_compile_definitions(core_interface_depends INTERFACE ${DEPENDS_CONFIG_COMPILE_DEFINITIONS})
# If the {CXX,LD}FLAGS environment variables are defined during building depends
# and configuring this build system, their content might be duplicated.

View File

@ -215,17 +215,13 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina
-e 's|@OBJCOPY@|$(host_OBJCOPY)|' \
-e 's|@OBJDUMP@|$(host_OBJDUMP)|' \
-e 's|@CFLAGS@|$(strip $(host_CFLAGS))|' \
-e 's|@CFLAGS_RELEASE@|$(strip $(host_release_CFLAGS))|' \
-e 's|@CFLAGS_DEBUG@|$(strip $(host_debug_CFLAGS))|' \
-e 's|@CFLAGS_DEPENDS_CONFIG@|$(strip $(host_$(release_type)_CFLAGS))|' \
-e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS))|' \
-e 's|@CXXFLAGS_RELEASE@|$(strip $(host_release_CXXFLAGS))|' \
-e 's|@CXXFLAGS_DEBUG@|$(strip $(host_debug_CXXFLAGS))|' \
-e 's|@CXXFLAGS_DEPENDS_CONFIG@|$(strip $(host_$(release_type)_CXXFLAGS))|' \
-e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS))|' \
-e 's|@CPPFLAGS_RELEASE@|$(strip $(host_release_CPPFLAGS))|' \
-e 's|@CPPFLAGS_DEBUG@|$(strip $(host_debug_CPPFLAGS))|' \
-e 's|@CPPFLAGS_DEPENDS_CONFIG@|$(strip $(host_$(release_type)_CPPFLAGS))|' \
-e 's|@LDFLAGS@|$(strip $(host_LDFLAGS))|' \
-e 's|@LDFLAGS_RELEASE@|$(strip $(host_release_LDFLAGS))|' \
-e 's|@LDFLAGS_DEBUG@|$(strip $(host_debug_LDFLAGS))|' \
-e 's|@LDFLAGS_DEPENDS_CONFIG@|$(strip $(host_$(release_type)_LDFLAGS))|' \
-e 's|@qt_packages@|$(qt_packages_)|' \
-e 's|@qrencode_packages@|$(qrencode_packages_)|' \
-e 's|@zmq_packages@|$(zmq_packages_)|' \

View File

@ -8,9 +8,11 @@ $(1)_ar=$$($$($(1)_type)_AR)
$(1)_ranlib=$$($$($(1)_type)_RANLIB)
$(1)_nm=$$($$($(1)_type)_NM)
$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
$$($$($(1)_type)_$$(release_type)_CFLAGS)
$$($$($(1)_type)_$$(release_type)_CFLAGS) \
-pipe -std=$(C_STANDARD)
$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
$$($$($(1)_type)_$$(release_type)_CXXFLAGS)
$$($$($(1)_type)_$$(release_type)_CXXFLAGS) \
-pipe -std=$(CXX_STANDARD)
$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
$$($$($(1)_type)_$$(release_type)_LDFLAGS) \
-L$$($($(1)_type)_prefix)/lib

View File

@ -60,8 +60,8 @@ darwin_CXX=$(clangxx_prog) --target=$(host) \
-iwithsysroot/usr/include/c++/v1 \
-iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks
darwin_CFLAGS=-pipe -std=$(C_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION)
darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacos-version-min=$(OSX_MIN_VERSION)
darwin_CFLAGS=-mmacos-version-min=$(OSX_MIN_VERSION)
darwin_CXXFLAGS=-mmacos-version-min=$(OSX_MIN_VERSION)
darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION)
ifneq ($(build_os),darwin)

View File

@ -29,12 +29,11 @@ endef
define add_host_flags_func
ifeq ($(filter $(origin $1),undefined default),)
$(host_arch)_$(host_os)_$1 = $($1)
$(host_arch)_$(host_os)_$(release_type)_$1 =
$(host_arch)_$(host_os)_$(release_type)_$1 = $($1)
else
$(host_arch)_$(host_os)_$1 += $($(host_os)_$1)
$(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1)
endif
$(host_arch)_$(host_os)_$1 += $($(host_os)_$1)
host_$1 = $$($(host_arch)_$(host_os)_$1)
host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1)
endef

View File

@ -1,5 +1,5 @@
freebsd_CFLAGS=-pipe -std=$(C_STANDARD)
freebsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD)
freebsd_CFLAGS=
freebsd_CXXFLAGS=
freebsd_release_CFLAGS=-O2
freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS)

View File

@ -1,5 +1,5 @@
linux_CFLAGS=-pipe -std=$(C_STANDARD)
linux_CXXFLAGS=-pipe -std=$(CXX_STANDARD)
linux_CFLAGS=
linux_CXXFLAGS=
ifneq ($(LTO),)
linux_AR = $(host_toolchain)gcc-ar

View File

@ -5,8 +5,8 @@ ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),)
mingw32_CXX := $(host)-g++-posix
endif
mingw32_CFLAGS=-pipe -std=$(C_STANDARD)
mingw32_CXXFLAGS=-pipe -std=$(CXX_STANDARD)
mingw32_CFLAGS=
mingw32_CXXFLAGS=
ifneq ($(LTO),)
mingw32_AR = $(host_toolchain)gcc-ar

View File

@ -1,5 +1,5 @@
netbsd_CFLAGS=-pipe -std=$(C_STANDARD)
netbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD)
netbsd_CFLAGS=
netbsd_CXXFLAGS=
ifneq ($(LTO),)
netbsd_AR = $(host_toolchain)gcc-ar

View File

@ -1,5 +1,5 @@
openbsd_CFLAGS=-pipe -std=$(C_STANDARD)
openbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD)
openbsd_CFLAGS=
openbsd_CXXFLAGS=
openbsd_release_CFLAGS=-O2
openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS)

View File

@ -15,14 +15,27 @@ if(@depends_crosscompiling@)
set(CMAKE_SYSTEM_PROCESSOR @host_arch@)
endif()
# Create a build type specifically for depends builds and default to it. When
# building from depends toolchain file, this will ensure that the flags from
# depends (or those provided by the user during depends build) will not be
# overridden by CMake's build-type flags.
# The user can still change the build-type to a non-Depends-type
# (Release/Debug/RelWithDebInfo/etc), which will inform the build to ignore the
# depends flags.
if(NOT SET_UP_CONFIGURATIONS_DONE)
set(SET_UP_CONFIGURATIONS_DONE TRUE)
list(PREPEND CMAKE_CONFIGURATION_TYPES "Depends")
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Depends)
endif()
endif()
if(NOT DEFINED CMAKE_C_FLAGS_INIT)
set(CMAKE_C_FLAGS_INIT "@CFLAGS@")
endif()
if(NOT DEFINED CMAKE_C_FLAGS_RELWITHDEBINFO_INIT)
set(CMAKE_C_FLAGS_RELWITHDEBINFO_INIT "@CFLAGS_RELEASE@")
endif()
if(NOT DEFINED CMAKE_C_FLAGS_DEBUG_INIT)
set(CMAKE_C_FLAGS_DEBUG_INIT "@CFLAGS_DEBUG@")
if(NOT DEFINED CMAKE_C_FLAGS_DEPENDS_INIT)
set(CMAKE_C_FLAGS_DEPENDS_INIT "@CFLAGS_DEPENDS_CONFIG@")
endif()
if(NOT DEFINED CMAKE_C_COMPILER)
@ -33,13 +46,9 @@ if(NOT DEFINED CMAKE_CXX_FLAGS_INIT)
set(CMAKE_CXX_FLAGS_INIT "@CXXFLAGS@")
set(CMAKE_OBJCXX_FLAGS_INIT "@CXXFLAGS@")
endif()
if(NOT DEFINED CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT)
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO_INIT "@CXXFLAGS_RELEASE@")
set(CMAKE_OBJCXX_FLAGS_RELWITHDEBINFO_INIT "@CXXFLAGS_RELEASE@")
endif()
if(NOT DEFINED CMAKE_CXX_FLAGS_DEBUG_INIT)
set(CMAKE_CXX_FLAGS_DEBUG_INIT "@CXXFLAGS_DEBUG@")
set(CMAKE_OBJCXX_FLAGS_DEBUG_INIT "@CXXFLAGS_DEBUG@")
if(NOT DEFINED CMAKE_CXX_FLAGS_DEPENDS_INIT)
set(CMAKE_CXX_FLAGS_DEPENDS_INIT "@CXXFLAGS_DEPENDS_CONFIG@")
set(CMAKE_OBJCXX_FLAGS_DEPENDS_INIT "@CXXFLAGS_DEPENDS_CONFIG@")
endif()
if(NOT DEFINED CMAKE_CXX_COMPILER)
@ -49,8 +58,7 @@ endif()
# The DEPENDS_COMPILE_DEFINITIONS* variables are to be treated as lists.
set(DEPENDS_COMPILE_DEFINITIONS @CPPFLAGS@)
set(DEPENDS_COMPILE_DEFINITIONS_RELWITHDEBINFO @CPPFLAGS_RELEASE@)
set(DEPENDS_COMPILE_DEFINITIONS_DEBUG @CPPFLAGS_DEBUG@)
set(DEPENDS_CONFIG_COMPILE_DEFINITIONS @CPPFLAGS_DEPENDS_CONFIG@)
if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_INIT)
set(CMAKE_EXE_LINKER_FLAGS_INIT "@LDFLAGS@")
@ -58,17 +66,11 @@ endif()
if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_INIT)
set(CMAKE_SHARED_LINKER_FLAGS_INIT "@LDFLAGS@")
endif()
if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO_INIT)
set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO_INIT "@LDFLAGS_RELEASE@")
if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_DEPENDS_INIT)
set(CMAKE_SHARED_LINKER_FLAGS_DEPENDS_INIT "@LDFLAGS_DEPENDS_CONFIG@")
endif()
if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO_INIT)
set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO_INIT "@LDFLAGS_RELEASE@")
endif()
if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_DEBUG_INIT)
set(CMAKE_EXE_LINKER_FLAGS_DEBUG_INIT "@LDFLAGS_DEBUG@")
endif()
if(NOT DEFINED CMAKE_SHARED_LINKER_FLAGS_DEBUG_INIT)
set(CMAKE_SHARED_LINKER_FLAGS_DEBUG_INIT "@LDFLAGS_DEBUG@")
if(NOT DEFINED CMAKE_EXE_LINKER_FLAGS_DEPENDS_INIT)
set(CMAKE_EXE_LINKER_FLAGS_DEPENDS_INIT "@LDFLAGS_DEPENDS_CONFIG@")
endif()
set(CMAKE_AR "@AR@")

View File

@ -88,7 +88,7 @@ class InitTest(BitcoinTestFramework):
args = ['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']
for terminate_line in lines_to_terminate_after:
self.log.info(f"Starting node and will exit after line {terminate_line}")
self.log.info(f"Starting node and will terminate after line {terminate_line}")
with node.busy_wait_for_debug_log([terminate_line]):
if platform.system() == 'Windows':
# CREATE_NEW_PROCESS_GROUP is required in order to be able
@ -108,12 +108,22 @@ class InitTest(BitcoinTestFramework):
'blocks/index/*.ldb': 'Error opening block database.',
'chainstate/*.ldb': 'Error opening coins database.',
'blocks/blk*.dat': 'Error loading block database.',
'indexes/txindex/MANIFEST*': 'LevelDB error: Corruption: CURRENT points to a non-existent file',
# Removing these files does not result in a startup error:
# 'indexes/blockfilter/basic/*.dat', 'indexes/blockfilter/basic/db/*.*', 'indexes/coinstats/db/*.*',
# 'indexes/txindex/*.log', 'indexes/txindex/CURRENT', 'indexes/txindex/LOCK'
}
files_to_perturb = {
'blocks/index/*.ldb': 'Error loading block database.',
'chainstate/*.ldb': 'Error opening coins database.',
'blocks/blk*.dat': 'Corrupted block database detected.',
'indexes/blockfilter/basic/db/*.*': 'LevelDB error: Corruption',
'indexes/coinstats/db/*.*': 'LevelDB error: Corruption',
'indexes/txindex/*.log': 'LevelDB error: Corruption',
'indexes/txindex/CURRENT': 'LevelDB error: Corruption',
# Perturbing these files does not result in a startup error:
# 'indexes/blockfilter/basic/*.dat', 'indexes/txindex/MANIFEST*', 'indexes/txindex/LOCK'
}
for file_patt, err_fragment in files_to_delete.items():
@ -135,9 +145,10 @@ class InitTest(BitcoinTestFramework):
self.stop_node(0)
self.log.info("Test startup errors after perturbing certain essential files")
dirs = ["blocks", "chainstate", "indexes"]
for file_patt, err_fragment in files_to_perturb.items():
shutil.copytree(node.chain_path / "blocks", node.chain_path / "blocks_bak")
shutil.copytree(node.chain_path / "chainstate", node.chain_path / "chainstate_bak")
for dir in dirs:
shutil.copytree(node.chain_path / dir, node.chain_path / f"{dir}_bak")
target_files = list(node.chain_path.glob(file_patt))
for target_file in target_files:
@ -151,10 +162,9 @@ class InitTest(BitcoinTestFramework):
start_expecting_error(err_fragment)
shutil.rmtree(node.chain_path / "blocks")
shutil.rmtree(node.chain_path / "chainstate")
shutil.move(node.chain_path / "blocks_bak", node.chain_path / "blocks")
shutil.move(node.chain_path / "chainstate_bak", node.chain_path / "chainstate")
for dir in dirs:
shutil.rmtree(node.chain_path / dir)
shutil.move(node.chain_path / f"{dir}_bak", node.chain_path / dir)
def init_pid_test(self):
BITCOIN_PID_FILENAME_CUSTOM = "my_fancy_bitcoin_pid_file.foobar"

View File

@ -45,6 +45,7 @@ from test_framework.util import (
assert_equal,
assert_greater_than,
assert_raises_rpc_error,
sync_txindex,
)
from test_framework.wallet import MiniWallet
from test_framework.wallet_util import generate_keypair
@ -270,6 +271,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
self.log.info('A coinbase transaction')
# Pick the input of the first tx we created, so it has to be a coinbase tx
sync_txindex(self, node)
raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid'])
tx = tx_from_hex(raw_tx_coinbase_spent)
self.check_mempool_result(

View File

@ -34,6 +34,7 @@ from test_framework.util import (
assert_equal,
assert_greater_than,
assert_raises_rpc_error,
sync_txindex,
)
from test_framework.wallet import (
getnewdestination,
@ -70,7 +71,7 @@ class RawTransactionsTest(BitcoinTestFramework):
self.num_nodes = 3
self.extra_args = [
["-txindex"],
["-txindex"],
[],
["-fastprune", "-prune=1"],
]
# whitelist peers to speed up tx relay / mempool sync
@ -109,6 +110,7 @@ class RawTransactionsTest(BitcoinTestFramework):
self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex")
if n == 0:
sync_txindex(self, self.nodes[n])
# With -txindex.
# 1. valid parameters - only supply txid
assert_equal(self.nodes[n].getrawtransaction(txId), tx['hex'])

View File

@ -12,6 +12,7 @@ from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
assert_raises_rpc_error,
sync_txindex,
)
from test_framework.wallet import MiniWallet
@ -77,6 +78,7 @@ class MerkleBlockTest(BitcoinTestFramework):
assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2]))), sorted(txlist))
assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid2, txid1]))), sorted(txlist))
# We can always get a proof if we have a -txindex
sync_txindex(self, self.nodes[1])
assert_equal(self.nodes[0].verifytxoutproof(self.nodes[1].gettxoutproof([txid_spent])), [txid_spent])
# We can't get a proof if we specify transactions from different blocks
assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[0].gettxoutproof, [txid1, txid3])

View File

@ -592,3 +592,10 @@ def find_vout_for_address(node, txid, addr):
if addr == tx["vout"][i]["scriptPubKey"]["address"]:
return i
raise RuntimeError("Vout not found for address: txid=%s, addr=%s" % (txid, addr))
def sync_txindex(test_framework, node):
test_framework.log.debug("Waiting for node txindex to sync")
sync_start = int(time.time())
test_framework.wait_until(lambda: node.getindexinfo("txindex")["txindex"]["synced"])
test_framework.log.debug(f"Synced in {time.time() - sync_start} seconds")

View File

@ -117,7 +117,6 @@ class AddressInputTypeGrouping(BitcoinTestFramework):
self.extra_args = [
[
"-addresstype=bech32",
"-txindex",
],
[
"-addresstype=p2sh-segwit",