diff --git a/config_test_native_sql.go b/config_test_native_sql.go index 01baacfae..2994e9e08 100644 --- a/config_test_native_sql.go +++ b/config_test_native_sql.go @@ -46,7 +46,7 @@ func (d *DefaultDatabaseBuilder) getGraphStore(baseDB *sqldb.BaseDB, // graphSQLMigration is the version number for the graph migration // that migrates the KV graph to the native SQL schema. -const graphSQLMigration = 9 +const graphSQLMigration = 10 // getSQLMigration returns a migration function for the given version. func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, diff --git a/docs/release-notes/release-notes-0.20.0.md b/docs/release-notes/release-notes-0.20.0.md index 579c5f3fe..83dd1e20b 100644 --- a/docs/release-notes/release-notes-0.20.0.md +++ b/docs/release-notes/release-notes-0.20.0.md @@ -211,6 +211,9 @@ reader of a payment request. ## Database +* Add missing [sql index](https://github.com/lightningnetwork/lnd/pull/10155) + for settled invoices to increase query speed. + ## Code Health ## Tooling and Documentation diff --git a/sqldb/SCHEMA_UPDATES.md b/sqldb/SCHEMA_UPDATES.md new file mode 100644 index 000000000..895eec2b9 --- /dev/null +++ b/sqldb/SCHEMA_UPDATES.md @@ -0,0 +1,224 @@ +# SQL Schema Updates in LND + +This document provides guidance for adding new SQL schema migrations in the LND codebase. When adding a new SQL schema, multiple files need to be updated to ensure consistency across different build configurations and environments. + +## Overview + +LND uses a dual approach for database migrations: +1. **SQL Schema Migrations**: Standard SQL files handled by golang-migrate +2. **Custom Application-Level Migrations**: Go functions that perform data migration or complex logic + +The migration system supports both development and production configurations through build tags. + +## Required Updates Checklist + +When adding a new SQL schema migration, you **MUST** update the following locations: + +### 1. SQL Migration Files +Create the migration SQL files in the appropriate directory: +``` +sqldb/sqlc/migrations/ +├── 000XXX_your_migration_name.up.sql # Schema changes +└── 000XXX_your_migration_name.down.sql # Rollback changes +``` + +### 2. Migration Configuration +Add the migration entry to **ONE** of these files based on your target environment: + +#### For Production Releases: +**File**: `sqldb/migrations.go` +```go + +migrationConfig = append([]MigrationConfig{ + // ... existing migrations + { + Name: "000XXX_your_migration_name", + Version: X, + SchemaVersion: X, + // MigrationFn: customMigrationFunction, // Only if custom logic needed + }, +``` + +#### For Development/Testing: +**File**: `sqldb/migrations_dev.go` +```go +//go:build test_db_postgres || test_db_sqlite || test_native_sql + +var migrationAdditions = []MigrationConfig{ + // ... existing migrations + { + Name: "000XXX_your_migration_name", + Version: X, + SchemaVersion: X, + // MigrationFn: customMigrationFunction, // Only if custom logic needed + }, +} +``` + +### 3. Custom Migration Functions (If Needed) +If your migration requires custom application logic (data transformation, complex operations), you need to: + +**⚠️ Important**: Custom migrations do NOT have corresponding SQL files in the `sqlc/migrations/` directory. They are purely application-level migrations defined in Go code. + +#### 3a. Add Custom Migration Entry to Migration Configuration + +same as in section 2 without the number prefix. See the example for more information. + + +#### 3b. Add Migration Function Reference +Add the migration function to the appropriate config files: + +**For Production**: `config_builder.go` +```go +func (d *DefaultDatabaseBuilder) BuildDatabase( + ctx context.Context) (*DatabaseInstances, func(), error) { + + invoiceMig := func(tx *sqlc.Queries) error { + err := invoices.MigrateInvoicesToSQL( + ctx, dbs.ChanStateDB.Backend, + dbs.ChanStateDB, tx, + invoiceMigrationBatchSize, + ) + if err != nil { + return fmt.Errorf("failed to migrate "+ + "invoices to SQL: %w", err) + } + + // Set the invoice bucket tombstone to indicate + // that the migration has been completed. + d.logger.Debugf("Setting invoice bucket " + + "tombstone") + + return dbs.ChanStateDB.SetInvoiceBucketTombstone() //nolint:ll + +``` + +**For Testing**: `config_test_native_sql.go` +```go +func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, + version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) { + + switch version { + case X: // Use the version number from your migrationAdditions + return func(tx *sqlc.Queries) error { + // Your custom migration logic here + return nil + }, true + } + + return nil, false +} +``` + +#### 3b. Implement Migration Function +Create the actual migration function (typically in the same package that owns the data): +```go +func yourCustomMigrationFunction(tx *sqlc.Queries) error { + // Your custom migration logic here + // Example: data transformation, index creation, etc. + return nil +} +``` + +## Important Guidelines + +### Version Numbering +- **Schema Version**: Must match the SQL file number (e.g., `000008` → `SchemaVersion: 8`) +- **Migration Version**: Must be sequential and unique across all migrations +- **File Naming**: Use format `000XXX_descriptive_name.{up|down}.sql` + +### Build Tags +- **Production builds** (default): Use `migrations_prod.go` and `config_prod.go` +- **Test builds**: Use `migrations_dev.go` and `config_test_native_sql.go` +- The build system automatically includes the appropriate files based on build tags + +### Custom Migrations +- **No SQL files**: Custom migrations do NOT have `.up.sql` or `.down.sql` files - they are pure Go code +- Custom KV→SQL migrations should use the same `SchemaVersion` as their corresponding SQL migration +- KV migration `Version` should be higher than the SQL migration version +- Custom migrations only require config entries in `migrationAdditions` and implementation in `getSQLMigration` + +## Examples + +### Example 1: Simple Schema Migration - `000008_graph` + +The `000008_graph` migration is a real example from the codebase: + +1. **SQL files exist**: + ```sql + -- sqldb/sqlc/migrations/000008_graph.up.sql + -- (Contains graph schema creation SQL) + ``` + +2. **Migration config in migrations_dev.go**: + ```go + var migrationAdditions = []MigrationConfig{ + { + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, + }, + } + ``` + +### Example 2: Schema Migration with Custom Logic - `000008_graph` + `kv_graph_migration` + +This shows how `000008_graph` (SQL schema) works together with `kv_graph_migration` (custom logic): + +1. **SQL files for schema**: + ```sql + -- sqldb/sqlc/migrations/000008_graph.up.sql + -- Creates graph tables and indexes + ``` + +2. **Schema migration in migrations_dev.go**: + ```go + var migrationAdditions = []MigrationConfig{ + { + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, + }, + { + Name: "kv_graph_migration", + Version: 10, + SchemaVersion: 8, // Same schema version as 000008_graph + }, + } + ``` + +3. **Custom migration logic in config_test_native_sql.go**: + ```go + const graphSQLMigration = 10 + + func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, + version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) { + + switch version { + case graphSQLMigration: // version 10 (kv_graph_migration) + return func(tx *sqlc.Queries) error { + err := graphdb.MigrateGraphToSQL(ctx, cfg, kvBackend, tx) + if err != nil { + return fmt.Errorf("failed to migrate graph to SQL: %w", err) + } + return nil + }, true + } + + return nil, false + } + ``` + +This pattern shows: +- Version 9: SQL schema creation (`000008_graph`) +- Version 10: Data migration from KV to SQL (`kv_graph_migration`) + + +## Common Mistakes to Avoid + +1. **Forgetting migration config entry**: Every SQL file must have a corresponding config entry +2. **Version number mismatch**: Schema version must match SQL file number +3. **Missing custom migration reference**: If using `MigrationFn`, must update appropriate config file +4. **Wrong build tag file**: Use prod files for releases, dev files for testing +5. **Non-sequential versions**: Migration versions must be sequential without gaps +6. **Inconsistent cross-environment**: Ensure both prod and dev configurations are updated if needed diff --git a/sqldb/migrations.go b/sqldb/migrations.go index 98c63734d..0f293d22a 100644 --- a/sqldb/migrations.go +++ b/sqldb/migrations.go @@ -73,6 +73,11 @@ var ( // schema. This is optional and can be disabled by the // user if necessary. }, + { + Name: "000007_invoice_add_settled_index", + Version: 8, + SchemaVersion: 7, + }, }, migrationAdditions...) // ErrMigrationMismatch is returned when a migrated record does not diff --git a/sqldb/migrations_dev.go b/sqldb/migrations_dev.go index 38501b58b..a9b753696 100644 --- a/sqldb/migrations_dev.go +++ b/sqldb/migrations_dev.go @@ -4,14 +4,14 @@ package sqldb var migrationAdditions = []MigrationConfig{ { - Name: "000007_graph", - Version: 8, - SchemaVersion: 7, + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, }, { Name: "kv_graph_migration", - Version: 9, - SchemaVersion: 7, + Version: 10, + SchemaVersion: 8, // A migration function may be attached to this // migration to migrate KV graph to the native SQL // schema. This is optional and can be disabled by the diff --git a/sqldb/migrations_prod.go b/sqldb/migrations_prod.go index faa41f419..15f5f5844 100644 --- a/sqldb/migrations_prod.go +++ b/sqldb/migrations_prod.go @@ -2,4 +2,9 @@ package sqldb +// migrationAdditions is a list of migrations that are added to the +// migrationConfig slice. +// +// NOTE: This should always be empty and instead migrations for production +// should be added into the main line (see migrations.go). var migrationAdditions []MigrationConfig diff --git a/sqldb/sqlc/migrations/000001_invoices.down.sql b/sqldb/sqlc/migrations/000001_invoices.down.sql index 0e275b344..4d1bd739a 100644 --- a/sqldb/sqlc/migrations/000001_invoices.down.sql +++ b/sqldb/sqlc/migrations/000001_invoices.down.sql @@ -1,7 +1,3 @@ -DROP INDEX IF EXISTS invoice_payments_invoice_id_idx; -DROP INDEX IF EXISTS invoice_payments_settled_at_idx; -DROP TABLE IF EXISTS invoice_payments; - DROP INDEX IF EXISTS invoice_htlc_custom_records_htlc_id_idx; DROP TABLE IF EXISTS invoice_htlc_custom_records; @@ -11,6 +7,7 @@ DROP TABLE IF EXISTS invoice_htlcs; DROP INDEX IF EXISTS invoice_feature_invoice_id_idx; DROP TABLE IF EXISTS invoice_features; +DROP INDEX IF EXISTS invoices_settled_at_idx; DROP INDEX IF EXISTS invoices_created_at_idx; DROP INDEX IF EXISTS invoices_state_idx; DROP INDEX IF EXISTS invoices_payment_addr_idx; diff --git a/sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql new file mode 100644 index 000000000..3cee600ac --- /dev/null +++ b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS invoices_settle_index_idx; \ No newline at end of file diff --git a/sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql new file mode 100644 index 000000000..a8a3d1992 --- /dev/null +++ b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS invoices_settle_index_idx ON invoices(settle_index); \ No newline at end of file diff --git a/sqldb/sqlc/migrations/000007_graph.down.sql b/sqldb/sqlc/migrations/000008_graph.down.sql similarity index 100% rename from sqldb/sqlc/migrations/000007_graph.down.sql rename to sqldb/sqlc/migrations/000008_graph.down.sql diff --git a/sqldb/sqlc/migrations/000007_graph.up.sql b/sqldb/sqlc/migrations/000008_graph.up.sql similarity index 100% rename from sqldb/sqlc/migrations/000007_graph.up.sql rename to sqldb/sqlc/migrations/000008_graph.up.sql