From ae6cf72b46b088042a5ea2f6f677357fea93315e Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 13 Jul 2021 12:08:36 +0200 Subject: [PATCH] lncfg: add postgres --- docs/postgres.md | 31 ++++++++ docs/release-notes/release-notes-0.14.0.md | 21 +++++ lncfg/db.go | 93 +++++++++++++++++++++- sample-lnd.conf | 7 ++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 docs/postgres.md diff --git a/docs/postgres.md b/docs/postgres.md new file mode 100644 index 000000000..1b46e7a74 --- /dev/null +++ b/docs/postgres.md @@ -0,0 +1,31 @@ +# Postgres support in LND + +With the introduction of the `kvdb` interface, LND can support multiple database +backends. One of the supported backends is Postgres. This document +describes how it can be configured. + +## Building LND with postgres support + +To build LND with postgres support, include the following build tag: + +```shell +⛰ make tags="kvdb_postgres" +``` + +## Configuring Postgres for LND + +In order for LND to run on Postgres, an empty database should already exist. A +database can be created via the usual ways (psql, pgadmin, etc). A user with +access to this database is also required. + +Creation of a schema and the tables is handled by LND automatically. + +## Configuring LND for Postgres + +LND is configured for Postgres through the following configuration options: + +* `db.backend=postgres` to select the Postgres backend. +* `db.postgres.dsn=...` to set the database connection string that includes + database, user and password. +* `db.postgres.timeout=...` to set the connection timeout. If not set, no + timeout applies. diff --git a/docs/release-notes/release-notes-0.14.0.md b/docs/release-notes/release-notes-0.14.0.md index c19f941c1..c0717d5b4 100644 --- a/docs/release-notes/release-notes-0.14.0.md +++ b/docs/release-notes/release-notes-0.14.0.md @@ -38,6 +38,27 @@ instantaneous. Read the [guide on leader election](https://github.com/lightningnetwork/lnd/blob/master/docs/leader_election.md) for more information. +### Postgres database support + +This release adds [support for Postgres as a database +backend](https://github.com/lightningnetwork/lnd/pull/5366) to lnd. Postgres +has several advantages over the default bbolt backend: +* Better handling of large data sets. +* On-the-fly database compaction (auto vacuum). +* Database replication. +* Inspect data while lnd is running (bbolt opens the database exclusively). +* Usage of industry-standard tools to manage the stored data, get performance + metrics, etc. + +Furthermore, the SQL platform opens up possibilities to improve lnd's +performance in the future. Bbolt's single-writer model is a severe performance +bottleneck, whereas Postgres offers a variety of locking models. Additionally, +structured tables reduce the need for custom serialization/deserialization code +in `lnd`, saving developer time and limiting the potential for bugs. + +Instructions for enabling Postgres can be found in +[docs/postgres.md](../postgres.md). + ## Protocol Extensions ### Explicit Channel Negotiation diff --git a/lncfg/db.go b/lncfg/db.go index aa0dcd602..70e6671c8 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -7,6 +7,7 @@ import ( "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/kvdb/etcd" + "github.com/lightningnetwork/lnd/kvdb/postgres" "github.com/lightningnetwork/lnd/lnwallet/btcwallet" ) @@ -19,6 +20,7 @@ const ( BoltBackend = "bolt" EtcdBackend = "etcd" + PostgresBackend = "postgres" DefaultBatchCommitInterval = 500 * time.Millisecond // NSChannelDB is the namespace name that we use for the combined graph @@ -53,6 +55,8 @@ type DB struct { Etcd *etcd.Config `group:"etcd" namespace:"etcd" description:"Etcd settings."` Bolt *kvdb.BoltConfig `group:"bolt" namespace:"bolt" description:"Bolt settings."` + + Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."` } // DefaultDB creates and returns a new default DB config. @@ -71,6 +75,10 @@ func DefaultDB() *DB { func (db *DB) Validate() error { switch db.Backend { case BoltBackend: + case PostgresBackend: + if db.Postgres.Dsn == "" { + return fmt.Errorf("postgres dsn must be set") + } case EtcdBackend: if !db.Etcd.Embedded && db.Etcd.Host == "" { @@ -175,7 +183,8 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } }() - if db.Backend == EtcdBackend { + switch db.Backend { + case EtcdBackend: // As long as the graph data, channel state and height hint // cache are all still in the channel.db file in bolt, we // replicate the same behavior here and use the same etcd @@ -266,6 +275,88 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, Remote: true, CloseFuncs: closeFuncs, }, nil + + case PostgresBackend: + postgresBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSChannelDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres graph "+ + "DB: %v", err) + } + closeFuncs[NSChannelDB] = postgresBackend.Close + + postgresMacaroonBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSMacaroonDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres "+ + "macaroon DB: %v", err) + } + closeFuncs[NSMacaroonDB] = postgresMacaroonBackend.Close + + postgresDecayedLogBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSDecayedLogDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres "+ + "decayed log DB: %v", err) + } + closeFuncs[NSDecayedLogDB] = postgresDecayedLogBackend.Close + + postgresTowerClientBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSTowerClientDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres tower "+ + "client DB: %v", err) + } + closeFuncs[NSTowerClientDB] = postgresTowerClientBackend.Close + + postgresTowerServerBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSTowerServerDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres tower "+ + "server DB: %v", err) + } + closeFuncs[NSTowerServerDB] = postgresTowerServerBackend.Close + + postgresWalletBackend, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, + db.Postgres, NSWalletDB, + ) + if err != nil { + return nil, fmt.Errorf("error opening postgres macaroon "+ + "DB: %v", err) + } + closeFuncs[NSWalletDB] = postgresWalletBackend.Close + + returnEarly = false + return &DatabaseBackends{ + GraphDB: postgresBackend, + ChanStateDB: postgresBackend, + HeightHintDB: postgresBackend, + MacaroonDB: postgresMacaroonBackend, + DecayedLogDB: postgresDecayedLogBackend, + TowerClientDB: postgresTowerClientBackend, + TowerServerDB: postgresTowerServerBackend, + // The wallet loader will attempt to use/create the + // wallet in the replicated remote DB if we're running + // in a clustered environment. This will ensure that all + // members of the cluster have access to the same wallet + // state. + WalletDB: btcwallet.LoaderWithExternalWalletDB( + postgresWalletBackend, + ), + Remote: true, + CloseFuncs: closeFuncs, + }, nil } // We're using all bbolt based databases by default. diff --git a/sample-lnd.conf b/sample-lnd.conf index d06dae4fe..1b889ed13 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -1135,6 +1135,13 @@ litecoin.node=ltcd ; testing with embedded etcd. ; db.etcd.embedded_log_file=/path/etcd.log +[postgres] +; Postgres connection string. +; db.postgres.dsn=postgres://lnd:lnd@localhost:45432/lnd?sslmode=disable + +; Postgres connection timeout. Valid time units are {s, m, h}. Set to zero to +; disable. +; db.postgres.timeout= [cluster]