sqldb: add the sqldb package

This commit provides the scaffolding for using the new sql stores.
The new interfaces, structs and methods are in sync with other projects
like Taproot Assets.

- Transactional Queries: the sqldb package defines the interfaces required
to execute transactional queries to our storage interface.

- Migration Files Embedded: the migration files are embedded into the binary.

- Database Migrations: I kept the use of 'golang-migrate' to ensure our
codebase remains in sync with the other projects, but can be changed.

- Build Flags for Conditional DB Target: flexibility to specify our database
target at compile-time based on the build flags in the same way we do
with our kv stores.

- Update modules: ran `go mod tidy`.

sqldb/sqlite.go

@@ -0,0 +1,176 @@
+package sqldb
+
+import (
+	"database/sql"
+	"fmt"
+	"net/url"
+	"path/filepath"
+	"testing"
+	"time"
+
+	sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
+	"github.com/lightningnetwork/lnd/sqldb/sqlc"
+	"github.com/stretchr/testify/require"
+	_ "modernc.org/sqlite" // Register relevant drivers.
+)
+
+const (
+	// sqliteOptionPrefix is the string prefix sqlite uses to set various
+	// options. This is used in the following format:
+	//   * sqliteOptionPrefix || option_name = option_value.
+	sqliteOptionPrefix = "_pragma"
+
+	// sqliteTxLockImmediate is a dsn option used to ensure that write
+	// transactions are started immediately.
+	sqliteTxLockImmediate = "_txlock=immediate"
+
+	// defaultMaxConns is the number of permitted active and idle
+	// connections. We want to limit this so it isn't unlimited. We use the
+	// same value for the number of idle connections as, this can speed up
+	// queries given a new connection doesn't need to be established each
+	// time.
+	defaultMaxConns = 25
+
+	// connIdleLifetime is the amount of time a connection can be idle.
+	connIdleLifetime = 5 * time.Minute
+)
+
+// SqliteConfig holds all the config arguments needed to interact with our
+// sqlite DB.
+//
+//nolint:lll
+type SqliteConfig struct {
+	// SkipMigrations if true, then all the tables will be created on start
+	// up if they don't already exist.
+	SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."`
+
+	// DatabaseFileName is the full file path where the database file can be
+	// found.
+	DatabaseFileName string `long:"dbfile" description:"The full path to the database."`
+}
+
+// SqliteStore is a database store implementation that uses a sqlite backend.
+type SqliteStore struct {
+	cfg *SqliteConfig
+
+	*BaseDB
+}
+
+// NewSqliteStore attempts to open a new sqlite database based on the passed
+// config.
+func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
+	// The set of pragma options are accepted using query options. For now
+	// we only want to ensure that foreign key constraints are properly
+	// enforced.
+	pragmaOptions := []struct {
+		name  string
+		value string
+	}{
+		{
+			name:  "foreign_keys",
+			value: "on",
+		},
+		{
+			name:  "journal_mode",
+			value: "WAL",
+		},
+		{
+			name:  "busy_timeout",
+			value: "5000",
+		},
+		{
+			// With the WAL mode, this ensures that we also do an
+			// extra WAL sync after each transaction. The normal
+			// sync mode skips this and gives better performance,
+			// but risks durability.
+			name:  "synchronous",
+			value: "full",
+		},
+		{
+			// This is used to ensure proper durability for users
+			// running on Mac OS. It uses the correct fsync system
+			// call to ensure items are fully flushed to disk.
+			name:  "fullfsync",
+			value: "true",
+		},
+	}
+	sqliteOptions := make(url.Values)
+	for _, option := range pragmaOptions {
+		sqliteOptions.Add(
+			sqliteOptionPrefix,
+			fmt.Sprintf("%v=%v", option.name, option.value),
+		)
+	}
+
+	// Construct the DSN which is just the database file name, appended
+	// with the series of pragma options as a query URL string. For more
+	// details on the formatting here, see the modernc.org/sqlite docs:
+	// https://pkg.go.dev/modernc.org/sqlite#Driver.Open.
+	dsn := fmt.Sprintf(
+		"%v?%v&%v", cfg.DatabaseFileName, sqliteOptions.Encode(),
+		sqliteTxLockImmediate,
+	)
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	db.SetMaxOpenConns(defaultMaxConns)
+	db.SetMaxIdleConns(defaultMaxConns)
+	db.SetConnMaxLifetime(connIdleLifetime)
+
+	if !cfg.SkipMigrations {
+		// Now that the database is open, populate the database with
+		// our set of schemas based on our embedded in-memory file
+		// system.
+		//
+		// First, we'll need to open up a new migration instance for
+		// our current target database: sqlite.
+		driver, err := sqlite_migrate.WithInstance(
+			db, &sqlite_migrate.Config{},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		err = applyMigrations(
+			sqlSchemas, driver, "sqlc/migrations", "sqlc",
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	queries := sqlc.New(db)
+
+	return &SqliteStore{
+		cfg: cfg,
+		BaseDB: &BaseDB{
+			DB:      db,
+			Queries: queries,
+		},
+	}, nil
+}
+
+// NewTestSqliteDB is a helper function that creates an SQLite database for
+// testing.
+func NewTestSqliteDB(t *testing.T) *SqliteStore {
+	t.Helper()
+
+	t.Logf("Creating new SQLite DB for testing")
+
+	// TODO(roasbeef): if we pass :memory: for the file name, then we get
+	// an in mem version to speed up tests
+	dbFileName := filepath.Join(t.TempDir(), "tmp.db")
+	sqlDB, err := NewSqliteStore(&SqliteConfig{
+		DatabaseFileName: dbFileName,
+		SkipMigrations:   false,
+	})
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		require.NoError(t, sqlDB.DB.Close())
+	})
+
+	return sqlDB
+}