config+sqldb: make native SQL query config options configurable

Here, we make the sql query option params (batch size and pagination
size) configurable. The defaults for SQLite vs Postgres are still the
same but will be changed in an upcoming commit.

config_test_native_sql.go

@@ -10,6 +10,7 @@ import (
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	graphdb "github.com/lightningnetwork/lnd/graph/db"
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lncfg"
 	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 )
@@ -30,10 +31,15 @@ func (d *DefaultDatabaseBuilder) getGraphStore(baseDB *sqldb.BaseDB,
 		},
 	)
 
+	queryConfig := d.cfg.DB.Sqlite.QueryConfig
+	if d.cfg.DB.Backend == lncfg.PostgresBackend {
+		queryConfig = d.cfg.DB.Postgres.QueryConfig
+	}
+
 	return graphdb.NewSQLStore(
 		&graphdb.SQLStoreConfig{
 			ChainHash: *d.cfg.ActiveNetParams.GenesisHash,
-			QueryCfg:  sqldb.DefaultQueryConfig(),
+			QueryCfg:  &queryConfig,
 		},
 		graphExecutor, opts...,
 	)

lncfg/db.go

@@ -115,10 +115,18 @@ func DefaultDB() *DB {
 		},
 		Postgres: &sqldb.PostgresConfig{
 			MaxConnections: defaultPostgresMaxConnections,
+			QueryConfig: sqldb.QueryConfig{
+				MaxBatchSize: 250,
+				MaxPageSize:  10000,
+			},
 		},
 		Sqlite: &sqldb.SqliteConfig{
 			MaxConnections: defaultSqliteMaxConnections,
 			BusyTimeout:    defaultSqliteBusyTimeout,
+			QueryConfig: sqldb.QueryConfig{
+				MaxBatchSize: 250,
+				MaxPageSize:  10000,
+			},
 		},
 		UseNativeSQL:           false,
 		SkipNativeSQLMigration: false,

sample-lnd.conf

@@ -1601,6 +1601,12 @@
 ; Whether to skip executing schema migrations.
 ; db.postgres.skipmigrations=false
 
+; The maximum number of elements to use in a native-SQL batch query IN clause.
+; db.postgres.query.max-batch-size=5000
+
+; The maximum number of records to fetch at a time in a paginated query during
+; native-SQL calls.
+; db.postgres.query.max-page-size=10500
 
 [sqlite]
 
@@ -1617,6 +1623,13 @@
 ; The maximum amount of time to wait to execute a query if the db is locked.
 ; db.sqlite.busytimeout=5s
 
+; The maximum number of elements to use in a native-SQL batch query IN clause.
+; db.sqlite.query.max-batch-size=250
+
+; The maximum number of records to fetch at a time in a paginated query during
+; native-SQL calls.
+; db.sqlite.query.max-page-size=100
+
 ; Raw pragma option pairs to be used when opening the sqlite db. The flag
 ; can be specified multiple times to set multiple options.
 ; Default:

sqldb/config.go

@@ -28,6 +28,7 @@ type SqliteConfig struct {
 	MaxConnections int           `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."`
 	PragmaOptions  []string      `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."`
 	SkipMigrations bool          `long:"skipmigrations" description:"Skip applying migrations on startup."`
+	QueryConfig    `group:"query" namespace:"query"`
 }
 
 // PostgresConfig holds the postgres database configuration.
@@ -38,6 +39,7 @@ type PostgresConfig struct {
 	Timeout        time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."`
 	MaxConnections int           `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."`
 	SkipMigrations bool          `long:"skipmigrations" description:"Skip applying migrations on startup."`
+	QueryConfig    `group:"query" namespace:"query"`
 }
 
 func (p *PostgresConfig) Validate() error {

sqldb/paginate.go

@@ -6,20 +6,19 @@ import (
 )
 
 // QueryConfig holds configuration values for SQL queries.
+//
+//nolint:ll
 type QueryConfig struct {
 	// MaxBatchSize is the maximum number of items included in a batch
 	// query IN clauses list.
-	MaxBatchSize int
+	MaxBatchSize int `long:"max-batch-size" description:"The maximum number of items to include in a batch query IN clause. This is used for queries that fetch results based on a list of identifiers."`
 
 	// MaxPageSize is the maximum number of items returned in a single page
 	// of results. This is used for paginated queries.
-	MaxPageSize int32
+	MaxPageSize int32 `long:"max-page-size" description:"The maximum number of items to return in a single page of results. This is used for paginated queries."`
 }
 
 // DefaultQueryConfig returns a default configuration for SQL queries.
-//
-// TODO(elle): make configurable & have different defaults for SQLite and
-// Postgres.
 func DefaultQueryConfig() *QueryConfig {
 	return &QueryConfig{
 		MaxBatchSize: 250,