Merge pull request #10155 from ziggie1984/add-missing-invoice-settle-index

Add missing invoice index for native sql

config_test_native_sql.go

@@ -46,7 +46,7 @@ func (d *DefaultDatabaseBuilder) getGraphStore(baseDB *sqldb.BaseDB,
 
 // graphSQLMigration is the version number for the graph migration
 // that migrates the KV graph to the native SQL schema.
-const graphSQLMigration = 9
+const graphSQLMigration = 10
 
 // getSQLMigration returns a migration function for the given version.
 func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context,

docs/release-notes/release-notes-0.20.0.md

@@ -211,6 +211,9 @@ reader of a payment request.
 
 ## Database
 
+* Add missing [sql index](https://github.com/lightningnetwork/lnd/pull/10155)
+  for settled invoices to increase query speed.
+
 ## Code Health
 
 ## Tooling and Documentation

sqldb/SCHEMA_UPDATES.md

@@ -0,0 +1,224 @@
+# SQL Schema Updates in LND
+
+This document provides guidance for adding new SQL schema migrations in the LND codebase. When adding a new SQL schema, multiple files need to be updated to ensure consistency across different build configurations and environments.
+
+## Overview
+
+LND uses a dual approach for database migrations:
+1. **SQL Schema Migrations**: Standard SQL files handled by golang-migrate
+2. **Custom Application-Level Migrations**: Go functions that perform data migration or complex logic
+
+The migration system supports both development and production configurations through build tags.
+
+## Required Updates Checklist
+
+When adding a new SQL schema migration, you **MUST** update the following locations:
+
+### 1. SQL Migration Files
+Create the migration SQL files in the appropriate directory:
+```
+sqldb/sqlc/migrations/
+├── 000XXX_your_migration_name.up.sql    # Schema changes
+└── 000XXX_your_migration_name.down.sql  # Rollback changes
+```
+
+### 2. Migration Configuration
+Add the migration entry to **ONE** of these files based on your target environment:
+
+#### For Production Releases:
+**File**: `sqldb/migrations.go`
+```go
+
+migrationConfig = append([]MigrationConfig{
+    // ... existing migrations
+    {
+        Name:          "000XXX_your_migration_name",
+        Version:       X,
+        SchemaVersion: X,
+        // MigrationFn: customMigrationFunction, // Only if custom logic needed
+    },
+```
+
+#### For Development/Testing:
+**File**: `sqldb/migrations_dev.go`
+```go
+//go:build test_db_postgres || test_db_sqlite || test_native_sql
+
+var migrationAdditions = []MigrationConfig{
+    // ... existing migrations
+    {
+        Name:          "000XXX_your_migration_name",
+        Version:       X,
+        SchemaVersion: X,
+        // MigrationFn: customMigrationFunction, // Only if custom logic needed
+    },
+}
+```
+
+### 3. Custom Migration Functions (If Needed)
+If your migration requires custom application logic (data transformation, complex operations), you need to:
+
+**⚠️ Important**: Custom migrations do NOT have corresponding SQL files in the `sqlc/migrations/` directory. They are purely application-level migrations defined in Go code.
+
+#### 3a. Add Custom Migration Entry to Migration Configuration
+
+same as in section 2 without the number prefix. See the example for more information.
+
+
+#### 3b. Add Migration Function Reference
+Add the migration function to the appropriate config files:
+
+**For Production**: `config_builder.go`
+```go
+func (d *DefaultDatabaseBuilder) BuildDatabase(
+    	ctx context.Context) (*DatabaseInstances, func(), error) {
+
+            	invoiceMig := func(tx *sqlc.Queries) error {
+				err := invoices.MigrateInvoicesToSQL(
+					ctx, dbs.ChanStateDB.Backend,
+					dbs.ChanStateDB, tx,
+					invoiceMigrationBatchSize,
+				)
+				if err != nil {
+					return fmt.Errorf("failed to migrate "+
+						"invoices to SQL: %w", err)
+				}
+
+				// Set the invoice bucket tombstone to indicate
+				// that the migration has been completed.
+				d.logger.Debugf("Setting invoice bucket " +
+					"tombstone")
+
+				return dbs.ChanStateDB.SetInvoiceBucketTombstone() //nolint:ll
+			
+```
+
+**For Testing**: `config_test_native_sql.go`
+```go
+func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context,
+    version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) {
+
+    switch version {
+    case X: // Use the version number from your migrationAdditions  
+        return func(tx *sqlc.Queries) error {
+            // Your custom migration logic here
+            return nil
+        }, true
+    }
+
+    return nil, false
+}
+```
+
+#### 3b. Implement Migration Function
+Create the actual migration function (typically in the same package that owns the data):
+```go
+func yourCustomMigrationFunction(tx *sqlc.Queries) error {
+    // Your custom migration logic here
+    // Example: data transformation, index creation, etc.
+    return nil
+}
+```
+
+## Important Guidelines
+
+### Version Numbering
+- **Schema Version**: Must match the SQL file number (e.g., `000008` → `SchemaVersion: 8`)
+- **Migration Version**: Must be sequential and unique across all migrations
+- **File Naming**: Use format `000XXX_descriptive_name.{up|down}.sql`
+
+### Build Tags
+- **Production builds** (default): Use `migrations_prod.go` and `config_prod.go`
+- **Test builds**: Use `migrations_dev.go` and `config_test_native_sql.go`
+- The build system automatically includes the appropriate files based on build tags
+
+### Custom Migrations
+- **No SQL files**: Custom migrations do NOT have `.up.sql` or `.down.sql` files - they are pure Go code
+- Custom KV→SQL migrations should use the same `SchemaVersion` as their corresponding SQL migration
+- KV migration `Version` should be higher than the SQL migration version
+- Custom migrations only require config entries in `migrationAdditions` and implementation in `getSQLMigration`
+
+## Examples
+
+### Example 1: Simple Schema Migration - `000008_graph`
+
+The `000008_graph` migration is a real example from the codebase:
+
+1. **SQL files exist**:
+   ```sql
+   -- sqldb/sqlc/migrations/000008_graph.up.sql
+   -- (Contains graph schema creation SQL)
+   ```
+
+2. **Migration config in migrations_dev.go**:
+   ```go
+   var migrationAdditions = []MigrationConfig{
+       {
+           Name:          "000008_graph",
+           Version:       9,
+           SchemaVersion: 8,
+       },
+   }
+   ```
+
+### Example 2: Schema Migration with Custom Logic - `000008_graph` + `kv_graph_migration`
+
+This shows how `000008_graph` (SQL schema) works together with `kv_graph_migration` (custom logic):
+
+1. **SQL files for schema**:
+   ```sql
+   -- sqldb/sqlc/migrations/000008_graph.up.sql
+   -- Creates graph tables and indexes
+   ```
+
+2. **Schema migration in migrations_dev.go**:
+   ```go
+   var migrationAdditions = []MigrationConfig{
+       {
+           Name:          "000008_graph", 
+           Version:       9,
+           SchemaVersion: 8,
+       },
+       {
+           Name:          "kv_graph_migration",
+           Version:       10,
+           SchemaVersion: 8, // Same schema version as 000008_graph
+       },
+   }
+   ```
+
+3. **Custom migration logic in config_test_native_sql.go**:
+   ```go
+   const graphSQLMigration = 10
+
+   func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context,
+       version int, kvBackend kvdb.Backend) (func(tx *sqlc.Queries) error, bool) {
+
+       switch version {
+       case graphSQLMigration: // version 10 (kv_graph_migration)
+           return func(tx *sqlc.Queries) error {
+               err := graphdb.MigrateGraphToSQL(ctx, cfg, kvBackend, tx)
+               if err != nil {
+                   return fmt.Errorf("failed to migrate graph to SQL: %w", err)
+               }
+               return nil
+           }, true
+       }
+
+       return nil, false
+   }
+   ```
+
+This pattern shows:
+- Version 9: SQL schema creation (`000008_graph`)  
+- Version 10: Data migration from KV to SQL (`kv_graph_migration`)
+
+
+## Common Mistakes to Avoid
+
+1. **Forgetting migration config entry**: Every SQL file must have a corresponding config entry
+2. **Version number mismatch**: Schema version must match SQL file number
+3. **Missing custom migration reference**: If using `MigrationFn`, must update appropriate config file
+4. **Wrong build tag file**: Use prod files for releases, dev files for testing
+5. **Non-sequential versions**: Migration versions must be sequential without gaps
+6. **Inconsistent cross-environment**: Ensure both prod and dev configurations are updated if needed

sqldb/migrations.go

@@ -73,6 +73,11 @@ var (
 			// schema. This is optional and can be disabled by the
 			// user if necessary.
 		},
+		{
+			Name:          "000007_invoice_add_settled_index",
+			Version:       8,
+			SchemaVersion: 7,
+		},
 	}, migrationAdditions...)
 
 	// ErrMigrationMismatch is returned when a migrated record does not

sqldb/migrations_dev.go

@@ -4,14 +4,14 @@ package sqldb
 
 var migrationAdditions = []MigrationConfig{
 	{
-		Name:          "000007_graph",
-		Version:       8,
-		SchemaVersion: 7,
+		Name:          "000008_graph",
+		Version:       9,
+		SchemaVersion: 8,
 	},
 	{
 		Name:          "kv_graph_migration",
-		Version:       9,
-		SchemaVersion: 7,
+		Version:       10,
+		SchemaVersion: 8,
 		// A migration function may be attached to this
 		// migration to migrate KV graph to the native SQL
 		// schema. This is optional and can be disabled by the

sqldb/migrations_prod.go

@@ -2,4 +2,9 @@
 
 package sqldb
 
+// migrationAdditions is a list of migrations that are added to the
+// migrationConfig slice.
+//
+// NOTE: This should always be empty and instead migrations for production
+// should be added into the main line (see migrations.go).
 var migrationAdditions []MigrationConfig

sqldb/sqlc/migrations/000001_invoices.down.sql

@@ -1,7 +1,3 @@
-DROP INDEX IF EXISTS invoice_payments_invoice_id_idx;
-DROP INDEX IF EXISTS invoice_payments_settled_at_idx;
-DROP TABLE IF EXISTS invoice_payments;
-
 DROP INDEX IF EXISTS invoice_htlc_custom_records_htlc_id_idx;
 DROP TABLE IF EXISTS invoice_htlc_custom_records;
 
@@ -11,6 +7,7 @@ DROP TABLE IF EXISTS invoice_htlcs;
 DROP INDEX IF EXISTS invoice_feature_invoice_id_idx;
 DROP TABLE IF EXISTS invoice_features;
 
+DROP INDEX IF EXISTS invoices_settled_at_idx;
 DROP INDEX IF EXISTS invoices_created_at_idx;
 DROP INDEX IF EXISTS invoices_state_idx;
 DROP INDEX IF EXISTS invoices_payment_addr_idx;

sqldb/sqlc/migrations/000007_invoice_add_settled_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS invoices_settle_index_idx;

sqldb/sqlc/migrations/000007_invoice_add_settled_index.up.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS invoices_settle_index_idx ON invoices(settle_index);