add sqlite3 storage (#40)

* add sqlite3 storage

* filter by content

* remove needless files

* update go.mod

* do not unquote Search

storage/sqlite3/delete.go

@@ -0,0 +1,6 @@
+package sqlite3
+
+func (b SQLite3Backend) DeleteEvent(id string, pubkey string) error {
+	_, err := b.DB.Exec("DELETE FROM event WHERE id = $1 AND pubkey = $2", id, pubkey)
+	return err
+}

storage/sqlite3/init.go

@@ -0,0 +1,33 @@
+package sqlite3
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/jmoiron/sqlx/reflectx"
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func (b *SQLite3Backend) Init() error {
+	db, err := sqlx.Connect("sqlite3", b.DatabaseURL)
+	if err != nil {
+		return err
+	}
+
+	// sqlx default is 0 (unlimited), while sqlite3 by default accepts up to 100 connections
+	db.SetMaxOpenConns(80)
+
+	db.Mapper = reflectx.NewMapperFunc("json", sqlx.NameMapper)
+	b.DB = db
+
+	_, err = b.DB.Exec(`
+CREATE TABLE IF NOT EXISTS event (
+  id text NOT NULL,
+  pubkey text NOT NULL,
+  created_at integer NOT NULL,
+  kind integer NOT NULL,
+  tags jsonb NOT NULL,
+  content text NOT NULL,
+  sig text NOT NULL
+);
+    `)
+	return err
+}

storage/sqlite3/query.go

@@ -0,0 +1,167 @@
+package sqlite3
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b SQLite3Backend) QueryEvents(filter *nostr.Filter) (events []nostr.Event, err error) {
+	var conditions []string
+	var params []any
+
+	if filter == nil {
+		err = errors.New("filter cannot be null")
+		return
+	}
+
+	if filter.IDs != nil {
+		if len(filter.IDs) > 500 {
+			// too many ids, fail everything
+			return
+		}
+
+		likeids := make([]string, 0, len(filter.IDs))
+		for _, id := range filter.IDs {
+			// to prevent sql attack here we will check if
+			// these ids are valid 32byte hex
+			parsed, err := hex.DecodeString(id)
+			if err != nil || len(parsed) != 32 {
+				continue
+			}
+			likeids = append(likeids, fmt.Sprintf("id LIKE '%x%%'", parsed))
+		}
+		if len(likeids) == 0 {
+			// ids being [] mean you won't get anything
+			return
+		}
+		conditions = append(conditions, "("+strings.Join(likeids, " OR ")+")")
+	}
+
+	if filter.Authors != nil {
+		if len(filter.Authors) > 500 {
+			// too many authors, fail everything
+			return
+		}
+
+		likekeys := make([]string, 0, len(filter.Authors))
+		for _, key := range filter.Authors {
+			// to prevent sql attack here we will check if
+			// these keys are valid 32byte hex
+			parsed, err := hex.DecodeString(key)
+			if err != nil || len(parsed) != 32 {
+				continue
+			}
+			likekeys = append(likekeys, fmt.Sprintf("pubkey LIKE '%x%%'", parsed))
+		}
+		if len(likekeys) == 0 {
+			// authors being [] mean you won't get anything
+			return
+		}
+		conditions = append(conditions, "("+strings.Join(likekeys, " OR ")+")")
+	}
+
+	if filter.Kinds != nil {
+		if len(filter.Kinds) > 10 {
+			// too many kinds, fail everything
+			return
+		}
+
+		if len(filter.Kinds) == 0 {
+			// kinds being [] mean you won't get anything
+			return
+		}
+		// no sql injection issues since these are ints
+		inkinds := make([]string, len(filter.Kinds))
+		for i, kind := range filter.Kinds {
+			inkinds[i] = strconv.Itoa(kind)
+		}
+		conditions = append(conditions, `kind IN (`+strings.Join(inkinds, ",")+`)`)
+	}
+
+	tagQuery := make([]string, 0, 1)
+	for _, values := range filter.Tags {
+		if len(values) == 0 {
+			// any tag set to [] is wrong
+			return
+		}
+
+		// add these tags to the query
+		tagQuery = append(tagQuery, values...)
+
+		if len(tagQuery) > 10 {
+			// too many tags, fail everything
+			return
+		}
+	}
+
+	if len(tagQuery) > 0 {
+		arrayBuild := make([]string, len(tagQuery))
+		for i, tagValue := range tagQuery {
+			arrayBuild[i] = "?"
+			params = append(params, tagValue)
+		}
+
+		// we use a very bad implementation in which we only check the tag values and
+		// ignore the tag names
+		conditions = append(conditions,
+			"instr(',"+strings.Join(arrayBuild, ",")+",',tags) IS NOT NULL")
+	}
+
+	if filter.Since != nil {
+		conditions = append(conditions, "created_at > ?")
+		params = append(params, filter.Since.Unix())
+	}
+	if filter.Until != nil {
+		conditions = append(conditions, "created_at < ?")
+		params = append(params, filter.Until.Unix())
+	}
+	if filter.Search != "" {
+		conditions = append(conditions, "content LIKE ?")
+		params = append(params, "%"+filter.Search+"%")
+	}
+
+	if len(conditions) == 0 {
+		// fallback
+		conditions = append(conditions, "true")
+	}
+
+	if filter.Limit < 1 || filter.Limit > 100 {
+		params = append(params, 100)
+	} else {
+		params = append(params, filter.Limit)
+	}
+
+	query := b.DB.Rebind(`SELECT
+      id, pubkey, created_at, kind, tags, content, sig
+    FROM event WHERE ` +
+		strings.Join(conditions, " AND ") +
+		" ORDER BY created_at DESC LIMIT ?")
+
+	rows, err := b.DB.Query(query, params...)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, fmt.Errorf("failed to fetch events using query %q: %w", query, err)
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var evt nostr.Event
+		var timestamp int64
+		err := rows.Scan(&evt.ID, &evt.PubKey, &timestamp,
+			&evt.Kind, &evt.Tags, &evt.Content, &evt.Sig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+		evt.CreatedAt = time.Unix(timestamp, 0)
+		events = append(events, evt)
+	}
+
+	return events, nil
+}

storage/sqlite3/save.go

@@ -0,0 +1,53 @@
+package sqlite3
+
+import (
+	"encoding/json"
+
+	"github.com/fiatjaf/relayer/storage"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func (b *SQLite3Backend) SaveEvent(evt *nostr.Event) error {
+	// react to different kinds of events
+	if evt.Kind == nostr.KindSetMetadata || evt.Kind == nostr.KindContactList || (10000 <= evt.Kind && evt.Kind < 20000) {
+		// delete past events from this user
+		b.DB.Exec(`DELETE FROM event WHERE pubkey = $1 AND kind = $2`, evt.PubKey, evt.Kind)
+	} else if evt.Kind == nostr.KindRecommendServer {
+		// delete past recommend_server events equal to this one
+		b.DB.Exec(`DELETE FROM event WHERE pubkey = $1 AND kind = $2 AND content = $3`,
+			evt.PubKey, evt.Kind, evt.Content)
+	}
+
+	// insert
+	tagsj, _ := json.Marshal(evt.Tags)
+	res, err := b.DB.Exec(`
+        INSERT INTO event (id, pubkey, created_at, kind, tags, content, sig)
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+    `, evt.ID, evt.PubKey, evt.CreatedAt.Unix(), evt.Kind, tagsj, evt.Content, evt.Sig)
+	if err != nil {
+		return err
+	}
+
+	nr, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+
+	if nr == 0 {
+		return storage.ErrDupEvent
+	}
+
+	return nil
+}
+
+func (b *SQLite3Backend) BeforeSave(evt *nostr.Event) {
+	// do nothing
+}
+
+func (b *SQLite3Backend) AfterSave(evt *nostr.Event) {
+	// delete all but the 100 most recent ones for each key
+	b.DB.Exec(`DELETE FROM event WHERE pubkey = $1 AND kind = $2 AND created_at < (
+      SELECT created_at FROM event WHERE pubkey = $1
+      ORDER BY created_at DESC OFFSET 100 LIMIT 1
+    )`, evt.PubKey, evt.Kind)
+}

storage/sqlite3/sqlite3.go

@@ -0,0 +1,10 @@
+package sqlite3
+
+import (
+	"github.com/jmoiron/sqlx"
+)
+
+type SQLite3Backend struct {
+	*sqlx.DB
+	DatabaseURL string
+}