From e38aa6d1e0f19fa335352682bc6ef87824b2d685 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 23 Jun 2025 10:05:03 +0200 Subject: [PATCH] multi: itest jobs with SQL graph backend In this commit, a new `test_native_sql` build flag is defined. If this build flag is used along with the `--use-native-sql` config option, then the SQLStore implementation of the graphdb.V1Store will be initialised. This is then used to run our itest suite against the new SQLStore graph implementation. NOTE that this only works for new nodes currently - the migration from kv-to-sql is yet to be implemeneted. --- .github/workflows/main.yml | 4 +++ config_builder.go | 52 ++++++++++++++++++++++++-------------- config_prod.go | 18 +++++++++++++ config_test_native_sql.go | 31 +++++++++++++++++++++++ sqldb/migrations_dev.go | 2 +- sqldb/migrations_prod.go | 2 +- 6 files changed, 88 insertions(+), 21 deletions(-) create mode 100644 config_prod.go create mode 100644 config_test_native_sql.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 951637def..0b59eb985 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -368,10 +368,14 @@ jobs: args: backend=bitcoind dbbackend=sqlite - name: bitcoind-sqlite-nativesql args: backend=bitcoind dbbackend=sqlite nativesql=true + - name: bitcoind-sqlite=nativesql-experiment + args: backend=bitcoind dbbackend=sqlite nativesql=true tags=test_native_sql - name: bitcoind-postgres args: backend=bitcoind dbbackend=postgres - name: bitcoind-postgres-nativesql args: backend=bitcoind dbbackend=postgres nativesql=true + - name: bitcoind-postgres-nativesql-experiment + args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql steps: - name: git checkout uses: actions/checkout@v4 diff --git a/config_builder.go b/config_builder.go index 95155b007..e47b29b7d 100644 --- a/config_builder.go +++ b/config_builder.go @@ -1046,23 +1046,6 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( ) } - graphStore, err := graphdb.NewKVStore( - databaseBackends.GraphDB, graphDBOptions..., - ) - if err != nil { - return nil, nil, err - } - - dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...) - if err != nil { - cleanUp() - - err = fmt.Errorf("unable to open graph DB: %w", err) - d.logger.Error(err) - - return nil, nil, err - } - dbOptions := []channeldb.OptionModifier{ channeldb.OptionDryRunMigration(cfg.DryRunMigration), channeldb.OptionKeepFailedPaymentAttempts( @@ -1098,6 +1081,10 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil, nil, err } + // The graph store implementation we will use depends on whether + // native SQL is enabled or not. + var graphStore graphdb.V1Store + // Instantiate a native SQL store if the flag is set. if d.cfg.DB.UseNativeSQL { migrations := sqldb.GetMigrations() @@ -1156,17 +1143,27 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // the base DB and transaction executor for the native SQL // invoice store. baseDB := dbs.NativeSQLStore.GetBaseDB() - executor := sqldb.NewTransactionExecutor( + invoiceExecutor := sqldb.NewTransactionExecutor( baseDB, func(tx *sql.Tx) invoices.SQLInvoiceQueries { return baseDB.WithTx(tx) }, ) sqlInvoiceDB := invoices.NewSQLStore( - executor, clock.NewDefaultClock(), + invoiceExecutor, clock.NewDefaultClock(), ) dbs.InvoiceDB = sqlInvoiceDB + + graphStore, err = d.getGraphStore( + baseDB, databaseBackends.GraphDB, graphDBOptions..., + ) + if err != nil { + err = fmt.Errorf("unable to get graph store: %w", err) + d.logger.Error(err) + + return nil, nil, err + } } else { // Check if the invoice bucket tombstone is set. If it is, we // need to return and ask the user switch back to using the @@ -1188,6 +1185,23 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( } dbs.InvoiceDB = dbs.ChanStateDB + + graphStore, err = graphdb.NewKVStore( + databaseBackends.GraphDB, graphDBOptions..., + ) + if err != nil { + return nil, nil, err + } + } + + dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...) + if err != nil { + cleanUp() + + err = fmt.Errorf("unable to open channel graph DB: %w", err) + d.logger.Error(err) + + return nil, nil, err } // Wrap the watchtower client DB and make sure we clean up. diff --git a/config_prod.go b/config_prod.go new file mode 100644 index 000000000..b480c2bb8 --- /dev/null +++ b/config_prod.go @@ -0,0 +1,18 @@ +//go:build !test_native_sql + +package lnd + +import ( + graphdb "github.com/lightningnetwork/lnd/graph/db" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/sqldb" +) + +// getGraphStore returns a graphdb.V1Store backed by a graphdb.KVStore +// implementation. +func (d *DefaultDatabaseBuilder) getGraphStore(_ *sqldb.BaseDB, + kvBackend kvdb.Backend, + opts ...graphdb.StoreOptionModifier) (graphdb.V1Store, error) { + + return graphdb.NewKVStore(kvBackend, opts...) +} diff --git a/config_test_native_sql.go b/config_test_native_sql.go new file mode 100644 index 000000000..4a9469deb --- /dev/null +++ b/config_test_native_sql.go @@ -0,0 +1,31 @@ +//go:build test_native_sql + +package lnd + +import ( + "database/sql" + + graphdb "github.com/lightningnetwork/lnd/graph/db" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/sqldb" +) + +// getGraphStore returns a graphdb.V1Store backed by a graphdb.SQLStore +// implementation. +func (d *DefaultDatabaseBuilder) getGraphStore(baseDB *sqldb.BaseDB, + _ kvdb.Backend, opts ...graphdb.StoreOptionModifier) (graphdb.V1Store, + error) { + + graphExecutor := sqldb.NewTransactionExecutor( + baseDB, func(tx *sql.Tx) graphdb.SQLQueries { + return baseDB.WithTx(tx) + }, + ) + + return graphdb.NewSQLStore( + &graphdb.SQLStoreConfig{ + ChainHash: *d.cfg.ActiveNetParams.GenesisHash, + }, + graphExecutor, opts..., + ) +} diff --git a/sqldb/migrations_dev.go b/sqldb/migrations_dev.go index 112e68dd0..b8721f927 100644 --- a/sqldb/migrations_dev.go +++ b/sqldb/migrations_dev.go @@ -1,4 +1,4 @@ -//go:build test_db_postgres || test_db_sqlite +//go:build test_db_postgres || test_db_sqlite || test_native_sql package sqldb diff --git a/sqldb/migrations_prod.go b/sqldb/migrations_prod.go index 551cf1fbb..faa41f419 100644 --- a/sqldb/migrations_prod.go +++ b/sqldb/migrations_prod.go @@ -1,4 +1,4 @@ -//go:build !test_db_postgres && !test_db_sqlite +//go:build !test_db_postgres && !test_db_sqlite && !test_native_sql package sqldb