diff --git a/chainntnfs/test_utils.go b/chainntnfs/test_utils.go index 715eae752..a9d95eaa5 100644 --- a/chainntnfs/test_utils.go +++ b/chainntnfs/test_utils.go @@ -268,7 +268,7 @@ func NewNeutrinoBackend(t *testing.T, minerAddr string) (*neutrino.ChainService, } dbName := filepath.Join(spvDir, "neutrino.db") - spvDatabase, err := walletdb.Create("bdb", dbName) + spvDatabase, err := walletdb.Create("bdb", dbName, true) if err != nil { os.RemoveAll(spvDir) t.Fatalf("unable to create walletdb: %v", err) diff --git a/chainregistry.go b/chainregistry.go index a209cc662..7be92afd9 100644 --- a/chainregistry.go +++ b/chainregistry.go @@ -703,7 +703,7 @@ func initNeutrinoBackend(chainDir string) (*neutrino.ChainService, func(), error } dbName := filepath.Join(dbPath, "neutrino.db") - db, err := walletdb.Create("bdb", dbName) + db, err := walletdb.Create("bdb", dbName, !cfg.SyncFreelist) if err != nil { return nil, nil, fmt.Errorf("unable to create neutrino "+ "database: %v", err) diff --git a/channeldb/db.go b/channeldb/db.go index 4bcfa0f8a..667ea418d 100644 --- a/channeldb/db.go +++ b/channeldb/db.go @@ -85,7 +85,7 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) { // Specify bbolt freelist options to reduce heap pressure in case the // freelist grows to be very large. options := &bbolt.Options{ - NoFreelistSync: true, + NoFreelistSync: opts.NoFreelistSync, FreelistType: bbolt.FreelistMapType, } diff --git a/channeldb/options.go b/channeldb/options.go index a96ebfa90..38ac05efd 100644 --- a/channeldb/options.go +++ b/channeldb/options.go @@ -21,6 +21,11 @@ type Options struct { // ChannelCacheSize is the maximum number of ChannelEdges to hold in the // channel cache. ChannelCacheSize int + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool } // DefaultOptions returns an Options populated with default values. @@ -28,6 +33,7 @@ func DefaultOptions() Options { return Options{ RejectCacheSize: DefaultRejectCacheSize, ChannelCacheSize: DefaultChannelCacheSize, + NoFreelistSync: true, } } @@ -47,3 +53,10 @@ func OptionSetChannelCacheSize(n int) OptionModifier { o.ChannelCacheSize = n } } + +// OptionSetSyncFreelist allows the database to sync its freelist. +func OptionSetSyncFreelist(b bool) OptionModifier { + return func(o *Options) { + o.NoFreelistSync = !b + } +} diff --git a/config.go b/config.go index 0800345cd..dc71a75c9 100644 --- a/config.go +++ b/config.go @@ -234,6 +234,7 @@ type config struct { LndDir string `long:"lnddir" description:"The base directory that contains lnd's data, logs, configuration file, etc."` ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` DataDir string `short:"b" long:"datadir" description:"The directory to store lnd's data within"` + SyncFreelist bool `long:"sync-freelist" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."` TLSCertPath string `long:"tlscertpath" description:"Path to write the TLS certificate for lnd's RPC and REST services"` TLSKeyPath string `long:"tlskeypath" description:"Path to write the TLS private key for lnd's RPC and REST services"` TLSExtraIPs []string `long:"tlsextraip" description:"Adds an extra ip to the generated certificate"` diff --git a/go.mod b/go.mod index d6aea5618..0a0188dc3 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d - github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 + github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 - github.com/btcsuite/btcwallet/walletdb v1.0.0 + github.com/btcsuite/btcwallet/walletdb v1.1.0 github.com/btcsuite/btcwallet/wtxmgr v1.0.0 github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 github.com/coreos/bbolt v1.3.3 diff --git a/go.sum b/go.sum index e9ed3de6b..9865e20b1 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 h1:NG3SmXd3KMOF4/BHVQaJuayrlXBosJgwUjeHcX4k198= -github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI= +github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb h1:W4l92i/ImG2qx4msNWSzhKqnvfQM3d8weQ27D++lhxk= +github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI= github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c= github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU= github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w= @@ -35,6 +35,8 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 h1:6DxkcoMnCPY4E9cUDPB5tbuuf github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs= github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw= github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= +github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y= +github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= github.com/btcsuite/btcwallet/wtxmgr v1.0.0 h1:aIHgViEmZmZfe0tQQqF1xyd2qBqFWxX5vZXkkbjtbeA= github.com/btcsuite/btcwallet/wtxmgr v1.0.0/go.mod h1:vc4gBprll6BP0UJ+AIGDaySoc7MdAmZf8kelfNb8CFY= github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 h1:kij1x2aL7VE6gtx8KMIt8PGPgI5GV9LgtHFG5KaEMPY= diff --git a/keychain/interface_test.go b/keychain/interface_test.go index 82e70e7c7..a85718558 100644 --- a/keychain/interface_test.go +++ b/keychain/interface_test.go @@ -45,7 +45,7 @@ func createTestBtcWallet(coinType uint32) (func(), *wallet.Wallet, error) { if err != nil { return nil, nil, err } - loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, 0) + loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, true, 0) pass := []byte("test") diff --git a/lnd.go b/lnd.go index 9bf0426d8..8c0b74aac 100644 --- a/lnd.go +++ b/lnd.go @@ -190,6 +190,7 @@ func Main(lisCfg ListenerCfg) error { graphDir, channeldb.OptionSetRejectCacheSize(cfg.Caches.RejectCacheSize), channeldb.OptionSetChannelCacheSize(cfg.Caches.ChannelCacheSize), + channeldb.OptionSetSyncFreelist(cfg.SyncFreelist), ) if err != nil { err := fmt.Errorf("Unable to open channeldb: %v", err) @@ -1000,7 +1001,8 @@ func waitForWalletPassword(restEndpoints []net.Addr, cfg.AdminMacPath, cfg.ReadMacPath, cfg.InvoiceMacPath, } pwService := walletunlocker.New( - chainConfig.ChainDir, activeNetParams.Params, macaroonFiles, + chainConfig.ChainDir, activeNetParams.Params, !cfg.SyncFreelist, + macaroonFiles, ) lnrpc.RegisterWalletUnlockerServer(grpcServer, pwService) @@ -1093,7 +1095,8 @@ func waitForWalletPassword(restEndpoints []net.Addr, chainConfig.ChainDir, activeNetParams.Params, ) loader := wallet.NewLoader( - activeNetParams.Params, netDir, uint32(recoveryWindow), + activeNetParams.Params, netDir, !cfg.SyncFreelist, + recoveryWindow, ) // With the seed, we can now use the wallet loader to create diff --git a/lnwallet/btcwallet/btcwallet.go b/lnwallet/btcwallet/btcwallet.go index 31c2e3206..84945c86a 100644 --- a/lnwallet/btcwallet/btcwallet.go +++ b/lnwallet/btcwallet/btcwallet.go @@ -89,8 +89,10 @@ func New(cfg Config) (*BtcWallet, error) { } else { pubPass = cfg.PublicPass } - loader := base.NewLoader(cfg.NetParams, netDir, - cfg.RecoveryWindow) + loader := base.NewLoader( + cfg.NetParams, netDir, cfg.NoFreelistSync, + cfg.RecoveryWindow, + ) walletExists, err := loader.WalletExists() if err != nil { return nil, err diff --git a/lnwallet/btcwallet/config.go b/lnwallet/btcwallet/config.go index 3516d5cb1..2dedf4e3d 100644 --- a/lnwallet/btcwallet/config.go +++ b/lnwallet/btcwallet/config.go @@ -75,6 +75,11 @@ type Config struct { // encrypted at all, in which case it should be attempted to be loaded // normally when creating the BtcWallet. Wallet *wallet.Wallet + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool } // NetworkDir returns the directory name of a network directory to hold wallet diff --git a/lnwallet/interface_test.go b/lnwallet/interface_test.go index 99ef9b7b2..04af443fa 100644 --- a/lnwallet/interface_test.go +++ b/lnwallet/interface_test.go @@ -2804,8 +2804,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, // Start Alice - open a database, start a neutrino // instance, and initialize a btcwallet driver for it. - aliceDB, err := walletdb.Create("bdb", - tempTestDirAlice+"/neutrino.db") + aliceDB, err := walletdb.Create( + "bdb", tempTestDirAlice+"/neutrino.db", true, + ) if err != nil { t.Fatalf("unable to create DB: %v", err) } @@ -2831,8 +2832,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, // Start Bob - open a database, start a neutrino // instance, and initialize a btcwallet driver for it. - bobDB, err := walletdb.Create("bdb", - tempTestDirBob+"/neutrino.db") + bobDB, err := walletdb.Create( + "bdb", tempTestDirBob+"/neutrino.db", true, + ) if err != nil { t.Fatalf("unable to create DB: %v", err) } diff --git a/routing/chainview/interface_test.go b/routing/chainview/interface_test.go index 24789b772..14f9461d3 100644 --- a/routing/chainview/interface_test.go +++ b/routing/chainview/interface_test.go @@ -846,7 +846,7 @@ var interfaceImpls = []struct { } dbName := filepath.Join(spvDir, "neutrino.db") - spvDatabase, err := walletdb.Create("bdb", dbName) + spvDatabase, err := walletdb.Create("bdb", dbName, true) if err != nil { return nil, nil, err } diff --git a/walletunlocker/service.go b/walletunlocker/service.go index a4e030e0a..90e84bae8 100644 --- a/walletunlocker/service.go +++ b/walletunlocker/service.go @@ -96,13 +96,14 @@ type UnlockerService struct { // sent. UnlockMsgs chan *WalletUnlockMsg - chainDir string - netParams *chaincfg.Params - macaroonFiles []string + chainDir string + noFreelistSync bool + netParams *chaincfg.Params + macaroonFiles []string } // New creates and returns a new UnlockerService. -func New(chainDir string, params *chaincfg.Params, +func New(chainDir string, params *chaincfg.Params, noFreelistSync bool, macaroonFiles []string) *UnlockerService { return &UnlockerService{ @@ -128,7 +129,7 @@ func (u *UnlockerService) GenSeed(ctx context.Context, // Before we start, we'll ensure that the wallet hasn't already created // so we don't show a *new* seed to the user if one already exists. netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, 0) + loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0) walletExists, err := loader.WalletExists() if err != nil { return nil, err @@ -257,7 +258,9 @@ func (u *UnlockerService) InitWallet(ctx context.Context, // We'll then open up the directory that will be used to store the // wallet's files so we can check if the wallet already exists. netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, uint32(recoveryWindow)) + loader := wallet.NewLoader( + u.netParams, netDir, u.noFreelistSync, uint32(recoveryWindow), + ) walletExists, err := loader.WalletExists() if err != nil { @@ -314,7 +317,9 @@ func (u *UnlockerService) UnlockWallet(ctx context.Context, recoveryWindow := uint32(in.RecoveryWindow) netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, recoveryWindow) + loader := wallet.NewLoader( + u.netParams, netDir, u.noFreelistSync, recoveryWindow, + ) // Check if wallet already exists. walletExists, err := loader.WalletExists() @@ -365,7 +370,7 @@ func (u *UnlockerService) ChangePassword(ctx context.Context, in *lnrpc.ChangePasswordRequest) (*lnrpc.ChangePasswordResponse, error) { netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, 0) + loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0) // First, we'll make sure the wallet exists for the specific chain and // network. diff --git a/walletunlocker/service_test.go b/walletunlocker/service_test.go index 8eec845a0..9b497ede1 100644 --- a/walletunlocker/service_test.go +++ b/walletunlocker/service_test.go @@ -36,7 +36,7 @@ var ( func createTestWallet(t *testing.T, dir string, netParams *chaincfg.Params) { netDir := btcwallet.NetworkDir(dir, netParams) - loader := wallet.NewLoader(netParams, netDir, 0) + loader := wallet.NewLoader(netParams, netDir, true, 0) _, err := loader.CreateNewWallet( testPassword, testPassword, testSeed, time.Time{}, ) @@ -62,7 +62,7 @@ func TestGenSeed(t *testing.T) { } defer os.RemoveAll(testDir) - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. @@ -103,7 +103,7 @@ func TestGenSeedGenerateEntropy(t *testing.T) { defer func() { os.RemoveAll(testDir) }() - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. Note that we don't actually @@ -143,7 +143,7 @@ func TestGenSeedInvalidEntropy(t *testing.T) { defer func() { os.RemoveAll(testDir) }() - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. However, we'll be using an @@ -181,7 +181,7 @@ func TestInitWallet(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Once we have the unlocker service created, we'll now instantiate a // new cipher seed instance. @@ -282,7 +282,7 @@ func TestCreateWalletInvalidEntropy(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // We'll attempt to init the wallet with an invalid cipher seed and // passphrase. @@ -315,7 +315,7 @@ func TestUnlockWallet(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) ctx := context.Background() req := &lnrpc.UnlockWalletRequest{ @@ -389,7 +389,7 @@ func TestChangeWalletPassword(t *testing.T) { } // Create a new UnlockerService with our temp files. - service := walletunlocker.New(testDir, testNetParams, tempFiles) + service := walletunlocker.New(testDir, testNetParams, true, tempFiles) ctx := context.Background() newPassword := []byte("hunter2???")