From 6a31e06817d051a922d7b3790d8870dff79f1f36 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 11 Aug 2025 13:09:01 +0200 Subject: [PATCH] graph/db+sqldb: find best default query cfg values for sqlite & postgres This commit adds a BenchmarkFindOptimalSQLQueryConfig test in the graph/db package which runs ForEachNode and ForEachChannel queries against a local backend using various different values for the sql QueryConfig struct. This is done to determine good default values to use for the config options for sqlite vs postgres. --- graph/db/benchmark_test.go | 138 +++++++++++++++++++++++++++++++++---- graph/db/test_postgres.go | 2 +- graph/db/test_sqlite.go | 2 +- lncfg/db.go | 10 +-- sqldb/paginate.go | 32 +++++++++ 5 files changed, 162 insertions(+), 22 deletions(-) diff --git a/graph/db/benchmark_test.go b/graph/db/benchmark_test.go index 1022197c5..783a2ce08 100644 --- a/graph/db/benchmark_test.go +++ b/graph/db/benchmark_test.go @@ -60,10 +60,6 @@ var ( WithBatchCommitInterval(500 * time.Millisecond), } - // testSQLPaginationCfg is used to configure the pagination settings for - // the SQL stores we open for testing. - testSQLPaginationCfg = sqldb.DefaultQueryConfig() - // testSqlitePragmaOpts is a set of SQLite pragma options that we apply // to the SQLite databases we open for testing. testSqlitePragmaOpts = []string{ @@ -111,7 +107,8 @@ var ( name: "native-sqlite", open: func(b testing.TB) V1Store { return connectNativeSQLite( - b, nativeSQLSqlitePath, nativeSQLSqliteFile, + b, sqldb.DefaultSQLiteConfig(), + nativeSQLSqlitePath, nativeSQLSqliteFile, ) }, } @@ -130,15 +127,20 @@ var ( nativeSQLPostgresConn = dbConnection{ name: "native-postgres", open: func(b testing.TB) V1Store { - return connectNativePostgres(b, nativeSQLPostgresDNS) + return connectNativePostgres( + b, sqldb.DefaultPostgresConfig(), + nativeSQLPostgresDNS, + ) }, } ) // connectNativePostgres creates a V1Store instance backed by a native Postgres // database for testing purposes. -func connectNativePostgres(t testing.TB, dsn string) V1Store { - return newSQLStore(t, sqlPostgres(t, dsn)) +func connectNativePostgres(t testing.TB, cfg *sqldb.QueryConfig, + dsn string) V1Store { + + return newSQLStore(t, cfg, sqlPostgres(t, dsn)) } // sqlPostgres creates a sqldb.DB instance backed by a native Postgres database @@ -158,8 +160,10 @@ func sqlPostgres(t testing.TB, dsn string) BatchedSQLQueries { // connectNativeSQLite creates a V1Store instance backed by a native SQLite // database for testing purposes. -func connectNativeSQLite(t testing.TB, dbPath, file string) V1Store { - return newSQLStore(t, sqlSQLite(t, dbPath, file)) +func connectNativeSQLite(t testing.TB, cfg *sqldb.QueryConfig, dbPath, + file string) V1Store { + + return newSQLStore(t, cfg, sqlSQLite(t, dbPath, file)) } // sqlSQLite creates a sqldb.DB instance backed by a native SQLite database for @@ -277,11 +281,13 @@ func newSQLExecutor(t testing.TB, db sqldb.DB) BatchedSQLQueries { // newSQLStore creates a new SQLStore instance for testing using a provided // sqldb.DB instance. -func newSQLStore(t testing.TB, db BatchedSQLQueries) V1Store { +func newSQLStore(t testing.TB, cfg *sqldb.QueryConfig, + db BatchedSQLQueries) V1Store { + store, err := NewSQLStore( &SQLStoreConfig{ ChainHash: dbTestChain, - QueryCfg: testSQLPaginationCfg, + QueryCfg: cfg, }, db, testStoreOptions..., ) @@ -763,3 +769,111 @@ func BenchmarkGraphReadMethods(b *testing.B) { } } } + +// BenchmarkFindOptimalSQLQueryConfig uses the ForEachNode and ForEachChannel +// methods to find the optimal maximum sqldb QueryConfig values for a given +// database backend. This is useful for determining the best default values for +// each backend. The ForEachNode and ForEachChannel methods are used since +// they make use of both batching and pagination. +func BenchmarkFindOptimalSQLQueryConfig(b *testing.B) { + // NOTE: Set this to true if you want to test with a postgres backend. + testPostgres := false + + // NOTE: Set this to true if you want to test with various batch sizes. + // By default, page sizes will be tested. + testBatching := false + + // Set the various page sizes we want to test. + // + // NOTE: these are the sqlite paging testing values. + testSizes := []int{20, 50, 100, 150, 500} + + configOption := "MaxPageSize" + if testBatching { + configOption = "MaxBatchSize" + + testSizes = []int{ + 50, 100, 150, 200, 250, 300, 350, + } + } + + dbName := "sqlite" + if testPostgres { + dbName = "postgres" + + // Set the various page sizes we want to test. + // + // NOTE: these are the postgres paging values. + testSizes = []int{5000, 7000, 10000, 12000} + + if testBatching { + testSizes = []int{ + 1000, 2000, 5000, 7000, 10000, + } + } + } + + for _, size := range testSizes { + b.Run(fmt.Sprintf("%s-%s-%d", configOption, dbName, size), + func(b *testing.B) { + ctx := context.Background() + + cfg := sqldb.DefaultSQLiteConfig() + if testPostgres { + cfg = sqldb.DefaultPostgresConfig() + } + + if testBatching { + cfg.MaxBatchSize = size + } else { + cfg.MaxPageSize = int32(size) + } + + store := connectNativeSQLite( + b, cfg, nativeSQLSqlitePath, + nativeSQLSqliteFile, + ) + if testPostgres { + store = connectNativePostgres( + b, cfg, nativeSQLPostgresDNS, + ) + } + + // Reset timer to exclude setup time. + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var ( + numNodes = 0 + numChannels = 0 + ) + + //nolint:ll + err := store.ForEachNode( + ctx, + func(_ *models.LightningNode) error { + numNodes++ + + return nil + }, func() {}, + ) + require.NoError(b, err) + + //nolint:ll + err = store.ForEachChannel( + ctx, + func(_ *models.ChannelEdgeInfo, + _, + _ *models.ChannelEdgePolicy) error { + + numChannels++ + + return nil + }, func() {}, + ) + require.NoError(b, err) + } + }, + ) + } +} diff --git a/graph/db/test_postgres.go b/graph/db/test_postgres.go index 463a89db2..6134f0114 100644 --- a/graph/db/test_postgres.go +++ b/graph/db/test_postgres.go @@ -43,7 +43,7 @@ func NewTestDBWithFixture(t testing.TB, store, err := NewSQLStore( &SQLStoreConfig{ ChainHash: *chaincfg.MainNetParams.GenesisHash, - QueryCfg: sqldb.DefaultQueryConfig(), + QueryCfg: sqldb.DefaultPostgresConfig(), }, querier, ) require.NoError(t, err) diff --git a/graph/db/test_sqlite.go b/graph/db/test_sqlite.go index 3442bfd6e..c1c6d808f 100644 --- a/graph/db/test_sqlite.go +++ b/graph/db/test_sqlite.go @@ -28,7 +28,7 @@ func NewTestDBWithFixture(t testing.TB, _ *sqldb.TestPgFixture) V1Store { store, err := NewSQLStore( &SQLStoreConfig{ ChainHash: *chaincfg.MainNetParams.GenesisHash, - QueryCfg: sqldb.DefaultQueryConfig(), + QueryCfg: sqldb.DefaultSQLiteConfig(), }, newBatchQuerier(t), ) require.NoError(t, err) diff --git a/lncfg/db.go b/lncfg/db.go index 927e76759..0f55c736e 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -115,18 +115,12 @@ func DefaultDB() *DB { }, Postgres: &sqldb.PostgresConfig{ MaxConnections: defaultPostgresMaxConnections, - QueryConfig: sqldb.QueryConfig{ - MaxBatchSize: 250, - MaxPageSize: 10000, - }, + QueryConfig: *sqldb.DefaultPostgresConfig(), }, Sqlite: &sqldb.SqliteConfig{ MaxConnections: defaultSqliteMaxConnections, BusyTimeout: defaultSqliteBusyTimeout, - QueryConfig: sqldb.QueryConfig{ - MaxBatchSize: 250, - MaxPageSize: 10000, - }, + QueryConfig: *sqldb.DefaultSQLiteConfig(), }, UseNativeSQL: false, SkipNativeSQLMigration: false, diff --git a/sqldb/paginate.go b/sqldb/paginate.go index e71f2e1c3..279893add 100644 --- a/sqldb/paginate.go +++ b/sqldb/paginate.go @@ -15,6 +15,20 @@ const ( // included in a batch query IN clause for Postgres. This was determined // using the TestSQLSliceQueries test. maxPostgresBatchSize = 65535 + + // defaultSQLitePageSize is the default page size for SQLite queries. + defaultSQLitePageSize = 100 + + // defaultPostgresPageSize is the default page size for Postgres + // queries. + defaultPostgresPageSize = 10500 + + // defaultSQLiteBatchSize is the default batch size for SQLite queries. + defaultSQLiteBatchSize = 250 + + // defaultPostgresBatchSize is the default batch size for Postgres + // queries. + defaultPostgresBatchSize = 5000 ) // QueryConfig holds configuration values for SQL queries. @@ -66,6 +80,24 @@ func DefaultQueryConfig() *QueryConfig { } } +// DefaultSQLiteConfig returns a default configuration for SQL queries to a +// SQLite backend. +func DefaultSQLiteConfig() *QueryConfig { + return &QueryConfig{ + MaxBatchSize: defaultSQLiteBatchSize, + MaxPageSize: defaultSQLitePageSize, + } +} + +// DefaultPostgresConfig returns a default configuration for SQL queries to a +// Postgres backend. +func DefaultPostgresConfig() *QueryConfig { + return &QueryConfig{ + MaxBatchSize: defaultPostgresBatchSize, + MaxPageSize: defaultPostgresPageSize, + } +} + // BatchQueryFunc represents a function that takes a batch of converted items // and returns results. type BatchQueryFunc[T any, R any] func(context.Context, []T) ([]R, error)