diff --git a/Makefile b/Makefile index 6b28bcc49..2f0c8c9ba 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,7 @@ clean-itest-logs: itest-only: clean-itest-logs db-instance @$(call print, "Running integration tests with ${backend} backend.") date - EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS) -test.v + EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) -test.v $(COLLECT_ITEST_COVERAGE) #? itest: Build and run integration tests @@ -233,7 +233,7 @@ itest-race: build-itest-race itest-only itest-parallel: clean-itest-logs build-itest db-instance @$(call print, "Running tests") date - EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS) + EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS) $(COLLECT_ITEST_COVERAGE) #? itest-clean: Kill all running itest processes diff --git a/itest/list_on_test.go b/itest/list_on_test.go index a192a423f..67915dee8 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -2,7 +2,9 @@ package itest -import "github.com/lightningnetwork/lnd/lntest" +import ( + "github.com/lightningnetwork/lnd/lntest" +) var allTestCases = []*lntest.TestCase{ { diff --git a/itest/lnd_test.go b/itest/lnd_test.go index cb0871b96..f2a345fa1 100644 --- a/itest/lnd_test.go +++ b/itest/lnd_test.go @@ -20,6 +20,7 @@ import ( "github.com/lightningnetwork/lnd/lntest/port" "github.com/lightningnetwork/lnd/lntest/wait" "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" "google.golang.org/grpc/grpclog" ) @@ -61,6 +62,13 @@ var ( "0-based index specified by the -runtranche flag", ) + // shuffleSeedFlag is the source of randomness used to shuffle the test + // cases. If not specified, the test cases won't be shuffled. + shuffleSeedFlag = flag.Uint64( + "shuffleseed", 0, "if set, shuffles the test cases using this "+ + "as the source of randomness", + ) + // testCasesRunTranche is the 0-based index of the split test cases // tranche to run in the current invocation. testCasesRunTranche = flag.Uint( @@ -160,6 +168,32 @@ func TestLightningNetworkDaemon(t *testing.T) { harnessTest.CurrentHeight()-height) } +// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is +// set and not 0. In parallel tests we want to shuffle the test cases so they +// are executed in a random order. This is done to even out the blocks mined in +// each test tranche so they can run faster. +// +// NOTE: Because the parallel tests are initialized with the same seed (job +// ID), they will always have the same order. +func maybeShuffleTestCases() { + // Exit if not set. + if shuffleSeedFlag == nil { + return + } + + // Exit if set to 0. + if *shuffleSeedFlag == 0 { + return + } + + // Init the seed and shuffle the test cases. + rand.Seed(*shuffleSeedFlag) + rand.Shuffle(len(allTestCases), func(i, j int) { + allTestCases[i], allTestCases[j] = + allTestCases[j], allTestCases[i] + }) +} + // getTestCaseSplitTranche returns the sub slice of the test cases that should // be run as the current split tranche as well as the index and slice offset of // the tranche. @@ -182,6 +216,9 @@ func getTestCaseSplitTranche() ([]*lntest.TestCase, uint, uint) { runTranche = 0 } + // Shuffle the test cases if the `shuffleseed` flag is set. + maybeShuffleTestCases() + numCases := uint(len(allTestCases)) testsPerTranche := numCases / numTranches trancheOffset := runTranche * testsPerTranche diff --git a/make/testing_flags.mk b/make/testing_flags.mk index b2db6861c..ba5b3fb37 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -10,6 +10,7 @@ COVER_PKG = $$(go list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)' | grep -v NUM_ITEST_TRANCHES = 4 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) POSTGRES_START_DELAY = 5 +SHUFFLE_SEED = 0 # If rpc option is set also add all extra RPC tags to DEV_TAGS ifneq ($(with-rpc),) @@ -27,6 +28,11 @@ ifneq ($(parallel),) ITEST_PARALLELISM = $(parallel) endif +# Set the seed for shuffling the test cases. +ifneq ($(shuffleseed),) +SHUFFLE_SEED = $(shuffleseed) +endif + # Windows needs to append a .exe suffix to all executable files, otherwise it # won't run them. ifneq ($(windows),) diff --git a/scripts/itest_parallel.sh b/scripts/itest_parallel.sh index 2ae7f3f53..ab1b3efa2 100755 --- a/scripts/itest_parallel.sh +++ b/scripts/itest_parallel.sh @@ -3,9 +3,10 @@ # Get all the variables. PROCESSES=$1 TRANCHES=$2 +SHUFFLE_SEED=$3 -# Here we also shift 2 times and get the rest of our flags to pass on in $@. -shift 2 +# Here we also shift 3 times and get the rest of our flags to pass on in $@. +shift 3 # Create a variable to hold the final exit code. exit_code=0 @@ -13,7 +14,7 @@ exit_code=0 # Run commands using xargs in parallel and capture their PIDs pids=() for ((i=0; i