diff --git a/kvdb/backend.go b/kvdb/backend.go index da2161c1c..76599311b 100644 --- a/kvdb/backend.go +++ b/kvdb/backend.go @@ -275,6 +275,20 @@ func GetTestBackend(path, name string) (Backend, func(), error) { ) return backend, cancel, err + case SqliteBackend: + dbPath := filepath.Join(path, name) + keyHash := sha256.Sum256([]byte(dbPath)) + sqliteDb, err := StartSqliteTestBackend( + path, name, "test_"+hex.EncodeToString(keyHash[:]), + ) + if err != nil { + return nil, empty, err + } + + return sqliteDb, func() { + _ = sqliteDb.Close() + }, nil + default: db, err := GetBoltBackend(&BoltBackendConfig{ DBPath: path, diff --git a/kvdb/config.go b/kvdb/config.go index b3a923093..08fcc3ad9 100644 --- a/kvdb/config.go +++ b/kvdb/config.go @@ -18,6 +18,11 @@ const ( // by a live instance of postgres. PostgresBackendName = "postgres" + // SqliteBackendName is the name of the backend that should be passed + // into kvdb.Create to initialize a new instance of kvdb.Backend backed + // by a live instance of sqlite. + SqliteBackendName = "sqlite" + // DefaultBoltAutoCompactMinAge is the default minimum time that must // have passed since a bolt database file was last compacted for the // compaction to be considered again. diff --git a/kvdb/go.mod b/kvdb/go.mod index a5c91d973..baba6ccfe 100644 --- a/kvdb/go.mod +++ b/kvdb/go.mod @@ -15,6 +15,7 @@ require ( go.etcd.io/etcd/client/v3 v3.5.0 go.etcd.io/etcd/server/v3 v3.5.0 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + modernc.org/sqlite v1.20.3 ) require ( @@ -29,6 +30,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -42,10 +44,12 @@ require ( github.com/jackc/pgtype v1.8.1 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.11 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/lib/pq v1.10.3 // indirect github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mholt/archiver/v3 v3.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -57,6 +61,7 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -81,15 +86,27 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + golang.org/x/tools v0.1.2 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect google.golang.org/grpc v1.38.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.22.2 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.4.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/kvdb/go.sum b/kvdb/go.sum index aa327cd03..30ae3ace3 100644 --- a/kvdb/go.sum +++ b/kvdb/go.sum @@ -148,14 +148,17 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -247,6 +250,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -287,6 +292,9 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= @@ -349,6 +357,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -505,6 +515,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -578,8 +589,9 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -707,6 +719,30 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs= +modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= +modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/kvdb/kvdb_no_sqlite.go b/kvdb/kvdb_no_sqlite.go new file mode 100644 index 000000000..ff265e6ca --- /dev/null +++ b/kvdb/kvdb_no_sqlite.go @@ -0,0 +1,25 @@ +//go:build !kvdb_sqlite || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64)) + +package kvdb + +import ( + "fmt" + "runtime" + + "github.com/btcsuite/btcwallet/walletdb" +) + +var errSqliteNotAvailable = fmt.Errorf("sqlite backend not available either "+ + "due to the `kvdb_sqlite` build tag not being set, or due to this "+ + "OS(%s) and/or architecture(%s) not being supported", runtime.GOOS, + runtime.GOARCH) + +// SqliteBackend is conditionally set to false when the kvdb_sqlite build tag is +// not defined. This will allow testing of other database backends. +const SqliteBackend = false + +// StartSqliteTestBackend is a stub returning nil, and errSqliteNotAvailable +// error. +func StartSqliteTestBackend(path, name, table string) (walletdb.DB, error) { + return nil, errSqliteNotAvailable +} diff --git a/kvdb/kvdb_sqlite.go b/kvdb/kvdb_sqlite.go new file mode 100644 index 000000000..ec880da63 --- /dev/null +++ b/kvdb/kvdb_sqlite.go @@ -0,0 +1,39 @@ +//go:build kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + +package kvdb + +import ( + "context" + "os" + "time" + + "github.com/btcsuite/btcwallet/walletdb" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" +) + +const ( + // SqliteBackend is conditionally set to true when the kvdb_sqlite build + // tag is defined. This will allow testing of other database backends. + SqliteBackend = true + + testMaxConnections = 50 +) + +// StartSqliteTestBackend starts a sqlite backed wallet.DB instance +func StartSqliteTestBackend(path, name, table string) (walletdb.DB, error) { + if !fileExists(path) { + err := os.Mkdir(path, 0700) + if err != nil { + return nil, err + } + } + + sqlbase.Init(testMaxConnections) + return sqlite.NewSqliteBackend( + context.Background(), &sqlite.Config{ + Timeout: time.Second * 30, + BusyTimeout: time.Second * 5, + }, path, name, table, + ) +} diff --git a/kvdb/sqlbase/db.go b/kvdb/sqlbase/db.go index 3e5d3c8e1..fda628a74 100644 --- a/kvdb/sqlbase/db.go +++ b/kvdb/sqlbase/db.go @@ -1,4 +1,4 @@ -//go:build kvdb_postgres +//go:build kvdb_postgres || (kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))) package sqlbase @@ -235,7 +235,7 @@ func (db *db) executeTransaction(f func(tx walletdb.ReadWriteTx) error, // PrintStats returns all collected stats pretty printed into a string. func (db *db) PrintStats() string { - return "stats not supported by Postgres driver" + return "stats not supported by SQL driver" } // BeginReadWriteTx opens a database read+write transaction. diff --git a/kvdb/sqlbase/no_sql.go b/kvdb/sqlbase/no_sql.go index 92c002d9e..666175782 100644 --- a/kvdb/sqlbase/no_sql.go +++ b/kvdb/sqlbase/no_sql.go @@ -1,4 +1,4 @@ -//go:build !kvdb_postgres +//go:build !kvdb_postgres && (!kvdb_sqlite || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64))) package sqlbase diff --git a/kvdb/sqlbase/readwrite_bucket.go b/kvdb/sqlbase/readwrite_bucket.go index c1eff6876..f8913723f 100644 --- a/kvdb/sqlbase/readwrite_bucket.go +++ b/kvdb/sqlbase/readwrite_bucket.go @@ -1,4 +1,4 @@ -//go:build kvdb_postgres +//go:build kvdb_postgres || (kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))) package sqlbase diff --git a/kvdb/sqlbase/readwrite_cursor.go b/kvdb/sqlbase/readwrite_cursor.go index bb5a0251e..3124f915c 100644 --- a/kvdb/sqlbase/readwrite_cursor.go +++ b/kvdb/sqlbase/readwrite_cursor.go @@ -1,4 +1,4 @@ -//go:build kvdb_postgres +//go:build kvdb_postgres || (kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))) package sqlbase diff --git a/kvdb/sqlbase/readwrite_tx.go b/kvdb/sqlbase/readwrite_tx.go index 7e82a1570..7286f1db9 100644 --- a/kvdb/sqlbase/readwrite_tx.go +++ b/kvdb/sqlbase/readwrite_tx.go @@ -1,4 +1,4 @@ -//go:build kvdb_postgres +//go:build kvdb_postgres || (kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))) package sqlbase diff --git a/kvdb/sqlbase/schema.go b/kvdb/sqlbase/schema.go index 6cced1c3d..1ff3aefb9 100644 --- a/kvdb/sqlbase/schema.go +++ b/kvdb/sqlbase/schema.go @@ -1,4 +1,4 @@ -//go:build kvdb_postgres +//go:build kvdb_postgres || (kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))) package sqlbase diff --git a/kvdb/sqlite/config.go b/kvdb/sqlite/config.go new file mode 100644 index 000000000..f60abcfc0 --- /dev/null +++ b/kvdb/sqlite/config.go @@ -0,0 +1,13 @@ +package sqlite + +import "time" + +// Config holds sqlite configuration data. +// +//nolint:lll +type Config struct { + Timeout time.Duration `long:"timeout" description:"The time after which a database query should be timed out."` + BusyTimeout time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."` +} diff --git a/kvdb/sqlite/db.go b/kvdb/sqlite/db.go new file mode 100644 index 000000000..f04bec3be --- /dev/null +++ b/kvdb/sqlite/db.go @@ -0,0 +1,83 @@ +//go:build kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + +package sqlite + +import ( + "context" + "fmt" + "net/url" + "path/filepath" + + "github.com/btcsuite/btcwallet/walletdb" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + _ "modernc.org/sqlite" // Register relevant drivers. +) + +const ( + // sqliteOptionPrefix is the string prefix sqlite uses to set various + // options. This is used in the following format: + // * sqliteOptionPrefix || option_name = option_value. + sqliteOptionPrefix = "_pragma" + + // sqliteTxLockImmediate is a dsn option used to ensure that write + // transactions are started immediately. + sqliteTxLockImmediate = "_txlock=immediate" +) + +// NewSqliteBackend returns a db object initialized with the passed backend +// config. If a sqlite connection cannot be established, then an error is +// returned. +func NewSqliteBackend(ctx context.Context, cfg *Config, dbPath, fileName, + prefix string) (walletdb.DB, error) { + + // First, we add a set of mandatory pragma options to the query. + pragmaOptions := []struct { + name string + value string + }{ + { + name: "busy_timeout", + value: fmt.Sprintf( + "%d", cfg.BusyTimeout.Milliseconds(), + ), + }, + { + name: "foreign_keys", + value: "on", + }, + { + name: "journal_mode", + value: "WAL", + }, + } + sqliteOptions := make(url.Values) + for _, option := range pragmaOptions { + sqliteOptions.Add( + sqliteOptionPrefix, + fmt.Sprintf("%v=%v", option.name, option.value), + ) + } + + // Then we add any user specified pragma options. Note that these can + // be of the form: "key=value", "key(N)" or "key". + for _, option := range cfg.PragmaOptions { + sqliteOptions.Add(sqliteOptionPrefix, option) + } + + // Construct the DSN which is just the database file name, appended + // with the series of pragma options as a query URL string. For more + // details on the formatting here, see the modernc.org/sqlite docs: + // https://pkg.go.dev/modernc.org/sqlite#Driver.Open. + dsn := fmt.Sprintf( + "%v?%v&%v", filepath.Join(dbPath, fileName), + sqliteOptions.Encode(), sqliteTxLockImmediate, + ) + sqlCfg := &sqlbase.Config{ + DriverName: "sqlite", + Dsn: dsn, + Timeout: cfg.Timeout, + TableNamePrefix: prefix, + } + + return sqlbase.NewSqlBackend(ctx, sqlCfg) +} diff --git a/kvdb/sqlite/db_test.go b/kvdb/sqlite/db_test.go new file mode 100644 index 000000000..f12acf47f --- /dev/null +++ b/kvdb/sqlite/db_test.go @@ -0,0 +1,35 @@ +//go:build kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + +package sqlite + +import ( + "testing" + "time" + + "github.com/btcsuite/btcwallet/walletdb/walletdbtest" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +// TestInterface performs all interfaces tests for this database driver. +func TestInterface(t *testing.T) { + // dbType is the database type name for this driver. + dir := t.TempDir() + ctx := context.Background() + + sqlbase.Init(0) + + cfg := &Config{ + BusyTimeout: time.Second * 5, + } + + sqlDB, err := NewSqliteBackend(ctx, cfg, dir, "tmp.db", "table") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, sqlDB.Close()) + }) + + walletdbtest.TestInterface(t, dbType, ctx, cfg, dir, "tmp.db", "temp") +} diff --git a/kvdb/sqlite/driver.go b/kvdb/sqlite/driver.go new file mode 100644 index 000000000..efd7892b1 --- /dev/null +++ b/kvdb/sqlite/driver.go @@ -0,0 +1,97 @@ +//go:build kvdb_sqlite && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + +package sqlite + +import ( + "context" + "fmt" + + "github.com/btcsuite/btcwallet/walletdb" +) + +const ( + dbType = "sqlite" +) + +// parseArgs parses the arguments from the walletdb Open/Create methods. +func parseArgs(funcName string, args ...interface{}) (context.Context, *Config, + string, string, string, error) { + + if len(args) != 5 { + return nil, nil, "", "", "", fmt.Errorf("invalid number of "+ + "arguments to %s.%s -- expected: context.Context, "+ + "sql.Config, string, string, string", dbType, funcName) + } + + ctx, ok := args[0].(context.Context) + if !ok { + return nil, nil, "", "", "", fmt.Errorf("argument 0 to %s.%s "+ + "is invalid -- expected: context.Context", dbType, + funcName) + } + + config, ok := args[1].(*Config) + if !ok { + return nil, nil, "", "", "", fmt.Errorf("argument 1 to %s.%s "+ + "is invalid -- expected: sqlite.Config", dbType, + funcName) + } + + dbPath, ok := args[2].(string) + if !ok { + return nil, nil, "", "", "", fmt.Errorf("argument 2 to %s.%s "+ + "is invalid -- expected string", dbType, dbPath) + } + + fileName, ok := args[3].(string) + if !ok { + return nil, nil, "", "", "", fmt.Errorf("argument 3 to %s.%s "+ + "is invalid -- expected string", dbType, funcName) + } + + prefix, ok := args[4].(string) + if !ok { + return nil, nil, "", "", "", fmt.Errorf("argument 4 to %s.%s "+ + "is invalid -- expected string", dbType, funcName, + ) + } + + return ctx, config, dbPath, fileName, prefix, nil +} + +// createDBDriver is the callback provided during driver registration that +// creates, initializes, and opens a database for use. +func createDBDriver(args ...interface{}) (walletdb.DB, error) { + ctx, config, dbPath, filename, prefix, err := parseArgs( + "Create", args..., + ) + if err != nil { + return nil, err + } + + return NewSqliteBackend(ctx, config, dbPath, filename, prefix) +} + +// openDBDriver is the callback provided during driver registration that opens +// an existing database for use. +func openDBDriver(args ...interface{}) (walletdb.DB, error) { + ctx, config, dbPath, filename, prefix, err := parseArgs("Open", args...) + if err != nil { + return nil, err + } + + return NewSqliteBackend(ctx, config, dbPath, filename, prefix) +} + +func init() { + // Register the driver. + driver := walletdb.Driver{ + DbType: dbType, + Create: createDBDriver, + Open: openDBDriver, + } + if err := walletdb.RegisterDriver(driver); err != nil { + panic(fmt.Sprintf("Failed to regiser database driver '%s': %v", + dbType, err)) + } +}