From 22fbbee83793d1ab566d0c312aef4d798a036e6a Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 13 Aug 2025 15:48:10 +0200 Subject: [PATCH 1/5] sqldb: delete wrong index names and add missing one --- sqldb/sqlc/migrations/000001_invoices.down.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sqldb/sqlc/migrations/000001_invoices.down.sql b/sqldb/sqlc/migrations/000001_invoices.down.sql index 0e275b344..4d1bd739a 100644 --- a/sqldb/sqlc/migrations/000001_invoices.down.sql +++ b/sqldb/sqlc/migrations/000001_invoices.down.sql @@ -1,7 +1,3 @@ -DROP INDEX IF EXISTS invoice_payments_invoice_id_idx; -DROP INDEX IF EXISTS invoice_payments_settled_at_idx; -DROP TABLE IF EXISTS invoice_payments; - DROP INDEX IF EXISTS invoice_htlc_custom_records_htlc_id_idx; DROP TABLE IF EXISTS invoice_htlc_custom_records; @@ -11,6 +7,7 @@ DROP TABLE IF EXISTS invoice_htlcs; DROP INDEX IF EXISTS invoice_feature_invoice_id_idx; DROP TABLE IF EXISTS invoice_features; +DROP INDEX IF EXISTS invoices_settled_at_idx; DROP INDEX IF EXISTS invoices_created_at_idx; DROP INDEX IF EXISTS invoices_state_idx; DROP INDEX IF EXISTS invoices_payment_addr_idx; From 506d226eb342cec5d4601cd5aafa94c3637cc37b Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 13 Aug 2025 15:48:50 +0200 Subject: [PATCH 2/5] sqldb: add missing index for settled invoices --- sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql | 1 + sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql create mode 100644 sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql diff --git a/sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql b/sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql new file mode 100644 index 000000000..3cee600ac --- /dev/null +++ b/sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS invoices_settle_index_idx; \ No newline at end of file diff --git a/sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql b/sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql new file mode 100644 index 000000000..a8a3d1992 --- /dev/null +++ b/sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS invoices_settle_index_idx ON invoices(settle_index); \ No newline at end of file From ea9fb80eff6dcfcf89d990e4caccdf96637e57bb Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 13 Aug 2025 22:26:03 +0200 Subject: [PATCH 3/5] sqldb: use the new schema We put this new schema update into the main line and change the versions of the schema updates which are currently only available in dev builds. The schemas need to be chronological therefore we also need to rename the file numbers. --- config_test_native_sql.go | 2 +- sqldb/migrations.go | 5 +++++ sqldb/migrations_dev.go | 10 +++++----- sqldb/migrations_prod.go | 5 +++++ ...n.sql => 000007_invoice_add_settled_index.down.sql} | 0 ....up.sql => 000007_invoice_add_settled_index.up.sql} | 0 .../{000007_graph.down.sql => 000008_graph.down.sql} | 0 .../{000007_graph.up.sql => 000008_graph.up.sql} | 0 8 files changed, 16 insertions(+), 6 deletions(-) rename sqldb/sqlc/migrations/{000008_invoice_add_settled_index.down.sql => 000007_invoice_add_settled_index.down.sql} (100%) rename sqldb/sqlc/migrations/{000008_invoice_add_settled_index.up.sql => 000007_invoice_add_settled_index.up.sql} (100%) rename sqldb/sqlc/migrations/{000007_graph.down.sql => 000008_graph.down.sql} (100%) rename sqldb/sqlc/migrations/{000007_graph.up.sql => 000008_graph.up.sql} (100%) diff --git a/config_test_native_sql.go b/config_test_native_sql.go index 01baacfae..2994e9e08 100644 --- a/config_test_native_sql.go +++ b/config_test_native_sql.go @@ -46,7 +46,7 @@ func (d *DefaultDatabaseBuilder) getGraphStore(baseDB *sqldb.BaseDB, // graphSQLMigration is the version number for the graph migration // that migrates the KV graph to the native SQL schema. -const graphSQLMigration = 9 +const graphSQLMigration = 10 // getSQLMigration returns a migration function for the given version. func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, diff --git a/sqldb/migrations.go b/sqldb/migrations.go index 98c63734d..0f293d22a 100644 --- a/sqldb/migrations.go +++ b/sqldb/migrations.go @@ -73,6 +73,11 @@ var ( // schema. This is optional and can be disabled by the // user if necessary. }, + { + Name: "000007_invoice_add_settled_index", + Version: 8, + SchemaVersion: 7, + }, }, migrationAdditions...) // ErrMigrationMismatch is returned when a migrated record does not diff --git a/sqldb/migrations_dev.go b/sqldb/migrations_dev.go index 38501b58b..a9b753696 100644 --- a/sqldb/migrations_dev.go +++ b/sqldb/migrations_dev.go @@ -4,14 +4,14 @@ package sqldb var migrationAdditions = []MigrationConfig{ { - Name: "000007_graph", - Version: 8, - SchemaVersion: 7, + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, }, { Name: "kv_graph_migration", - Version: 9, - SchemaVersion: 7, + Version: 10, + SchemaVersion: 8, // A migration function may be attached to this // migration to migrate KV graph to the native SQL // schema. This is optional and can be disabled by the diff --git a/sqldb/migrations_prod.go b/sqldb/migrations_prod.go index faa41f419..15f5f5844 100644 --- a/sqldb/migrations_prod.go +++ b/sqldb/migrations_prod.go @@ -2,4 +2,9 @@ package sqldb +// migrationAdditions is a list of migrations that are added to the +// migrationConfig slice. +// +// NOTE: This should always be empty and instead migrations for production +// should be added into the main line (see migrations.go). var migrationAdditions []MigrationConfig diff --git a/sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql similarity index 100% rename from sqldb/sqlc/migrations/000008_invoice_add_settled_index.down.sql rename to sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql diff --git a/sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql b/sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql similarity index 100% rename from sqldb/sqlc/migrations/000008_invoice_add_settled_index.up.sql rename to sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql diff --git a/sqldb/sqlc/migrations/000007_graph.down.sql b/sqldb/sqlc/migrations/000008_graph.down.sql similarity index 100% rename from sqldb/sqlc/migrations/000007_graph.down.sql rename to sqldb/sqlc/migrations/000008_graph.down.sql diff --git a/sqldb/sqlc/migrations/000007_graph.up.sql b/sqldb/sqlc/migrations/000008_graph.up.sql similarity index 100% rename from sqldb/sqlc/migrations/000007_graph.up.sql rename to sqldb/sqlc/migrations/000008_graph.up.sql From b98cc24cfe4f13b8f556f6235901b12d325417e7 Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 13 Aug 2025 15:46:13 +0200 Subject: [PATCH 4/5] docs: add release-notes --- docs/release-notes/release-notes-0.20.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release-notes/release-notes-0.20.0.md b/docs/release-notes/release-notes-0.20.0.md index 579c5f3fe..83dd1e20b 100644 --- a/docs/release-notes/release-notes-0.20.0.md +++ b/docs/release-notes/release-notes-0.20.0.md @@ -211,6 +211,9 @@ reader of a payment request. ## Database +* Add missing [sql index](https://github.com/lightningnetwork/lnd/pull/10155) + for settled invoices to increase query speed. + ## Code Health ## Tooling and Documentation From 35e9979d674b23f90ae769b71efa69e5ecd30de2 Mon Sep 17 00:00:00 2001 From: ziggie Date: Thu, 14 Aug 2025 09:28:00 +0200 Subject: [PATCH 5/5] docs: add schema update doc --- sqldb/SCHEMA_UPDATES.md | 224 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 sqldb/SCHEMA_UPDATES.md diff --git a/sqldb/SCHEMA_UPDATES.md b/sqldb/SCHEMA_UPDATES.md new file mode 100644 index 000000000..895eec2b9 --- /dev/null +++ b/sqldb/SCHEMA_UPDATES.md @@ -0,0 +1,224 @@ +# SQL Schema Updates in LND + +This document provides guidance for adding new SQL schema migrations in the LND codebase. When adding a new SQL schema, multiple files need to be updated to ensure consistency across different build configurations and environments. + +## Overview + +LND uses a dual approach for database migrations: +1. **SQL Schema Migrations**: Standard SQL files handled by golang-migrate +2. **Custom Application-Level Migrations**: Go functions that perform data migration or complex logic + +The migration system supports both development and production configurations through build tags. + +## Required Updates Checklist + +When adding a new SQL schema migration, you **MUST** update the following locations: + +### 1. SQL Migration Files +Create the migration SQL files in the appropriate directory: +``` +sqldb/sqlc/migrations/ +├── 000XXX_your_migration_name.up.sql # Schema changes +└── 000XXX_your_migration_name.down.sql # Rollback changes +``` + +### 2. Migration Configuration +Add the migration entry to **ONE** of these files based on your target environment: + +#### For Production Releases: +**File**: `sqldb/migrations.go` +```go + +migrationConfig = append([]MigrationConfig{ + // ... existing migrations + { + Name: "000XXX_your_migration_name", + Version: X, + SchemaVersion: X, + // MigrationFn: customMigrationFunction, // Only if custom logic needed + }, +``` + +#### For Development/Testing: +**File**: `sqldb/migrations_dev.go` +```go +//go:build test_db_postgres || test_db_sqlite || test_native_sql + +var migrationAdditions = []MigrationConfig{ + // ... existing migrations + { + Name: "000XXX_your_migration_name", + Version: X, + SchemaVersion: X, + // MigrationFn: customMigrationFunction, // Only if custom logic needed + }, +} +``` + +### 3. Custom Migration Functions (If Needed) +If your migration requires custom application logic (data transformation, complex operations), you need to: + +**⚠️ Important**: Custom migrations do NOT have corresponding SQL files in the `sqlc/migrations/` directory. They are purely application-level migrations defined in Go code. + +#### 3a. Add Custom Migration Entry to Migration Configuration + +same as in section 2 without the number prefix. See the example for more information. + + +#### 3b. Add Migration Function Reference +Add the migration function to the appropriate config files: + +**For Production**: `config_builder.go` +```go +func (d *DefaultDatabaseBuilder) BuildDatabase( + ctx context.Context) (*DatabaseInstances, func(), error) { + + invoiceMig := func(tx *sqlc.Queries) error { + err := invoices.MigrateInvoicesToSQL( + ctx, dbs.ChanStateDB.Backend, + dbs.ChanStateDB, tx, + invoiceMigrationBatchSize, + ) + if err != nil { + return fmt.Errorf("failed to migrate "+ + "invoices to SQL: %w", err) + } + + // Set the invoice bucket tombstone to indicate + // that the migration has been completed. + d.logger.Debugf("Setting invoice bucket " + + "tombstone") + + return dbs.ChanStateDB.SetInvoiceBucketTombstone() //nolint:ll + +``` + +**For Testing**: `config_test_native_sql.go` +```go +func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, + version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) { + + switch version { + case X: // Use the version number from your migrationAdditions + return func(tx *sqlc.Queries) error { + // Your custom migration logic here + return nil + }, true + } + + return nil, false +} +``` + +#### 3b. Implement Migration Function +Create the actual migration function (typically in the same package that owns the data): +```go +func yourCustomMigrationFunction(tx *sqlc.Queries) error { + // Your custom migration logic here + // Example: data transformation, index creation, etc. + return nil +} +``` + +## Important Guidelines + +### Version Numbering +- **Schema Version**: Must match the SQL file number (e.g., `000008` → `SchemaVersion: 8`) +- **Migration Version**: Must be sequential and unique across all migrations +- **File Naming**: Use format `000XXX_descriptive_name.{up|down}.sql` + +### Build Tags +- **Production builds** (default): Use `migrations_prod.go` and `config_prod.go` +- **Test builds**: Use `migrations_dev.go` and `config_test_native_sql.go` +- The build system automatically includes the appropriate files based on build tags + +### Custom Migrations +- **No SQL files**: Custom migrations do NOT have `.up.sql` or `.down.sql` files - they are pure Go code +- Custom KV→SQL migrations should use the same `SchemaVersion` as their corresponding SQL migration +- KV migration `Version` should be higher than the SQL migration version +- Custom migrations only require config entries in `migrationAdditions` and implementation in `getSQLMigration` + +## Examples + +### Example 1: Simple Schema Migration - `000008_graph` + +The `000008_graph` migration is a real example from the codebase: + +1. **SQL files exist**: + ```sql + -- sqldb/sqlc/migrations/000008_graph.up.sql + -- (Contains graph schema creation SQL) + ``` + +2. **Migration config in migrations_dev.go**: + ```go + var migrationAdditions = []MigrationConfig{ + { + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, + }, + } + ``` + +### Example 2: Schema Migration with Custom Logic - `000008_graph` + `kv_graph_migration` + +This shows how `000008_graph` (SQL schema) works together with `kv_graph_migration` (custom logic): + +1. **SQL files for schema**: + ```sql + -- sqldb/sqlc/migrations/000008_graph.up.sql + -- Creates graph tables and indexes + ``` + +2. **Schema migration in migrations_dev.go**: + ```go + var migrationAdditions = []MigrationConfig{ + { + Name: "000008_graph", + Version: 9, + SchemaVersion: 8, + }, + { + Name: "kv_graph_migration", + Version: 10, + SchemaVersion: 8, // Same schema version as 000008_graph + }, + } + ``` + +3. **Custom migration logic in config_test_native_sql.go**: + ```go + const graphSQLMigration = 10 + + func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context, + version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) { + + switch version { + case graphSQLMigration: // version 10 (kv_graph_migration) + return func(tx *sqlc.Queries) error { + err := graphdb.MigrateGraphToSQL(ctx, cfg, kvBackend, tx) + if err != nil { + return fmt.Errorf("failed to migrate graph to SQL: %w", err) + } + return nil + }, true + } + + return nil, false + } + ``` + +This pattern shows: +- Version 9: SQL schema creation (`000008_graph`) +- Version 10: Data migration from KV to SQL (`kv_graph_migration`) + + +## Common Mistakes to Avoid + +1. **Forgetting migration config entry**: Every SQL file must have a corresponding config entry +2. **Version number mismatch**: Schema version must match SQL file number +3. **Missing custom migration reference**: If using `MigrationFn`, must update appropriate config file +4. **Wrong build tag file**: Use prod files for releases, dev files for testing +5. **Non-sequential versions**: Migration versions must be sequential without gaps +6. **Inconsistent cross-environment**: Ensure both prod and dev configurations are updated if needed