highperfocused/khatru
1
0
Fork 0
mirror of https://github.com/fiatjaf/khatru.git synced 2025-03-28 18:51:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(postgres): configurable query limit

Browse Source 
Adds a QueryLimit to `type PostgresBackend` and retains the
current default value of 100.

Closes #60
This commit is contained in:
bndw 2023-05-23 20:41:48 -07:00 committed by fiatjaf_
parent 47b8ee106f
commit 03ecbb9e6c
4 changed files with 117 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
storage/postgresql/init.go
View File

@ -46,5 +46,9 @@ CREATE INDEX IF NOT EXISTS timeidx ON event (created_at DESC);
CREATE INDEX IF NOT EXISTS kindidx ON event (kind); CREATE INDEX IF NOT EXISTS kindidx ON event (kind);
CREATE INDEX IF NOT EXISTS arbitrarytagvalues ON event USING gin (tagvalues); CREATE INDEX IF NOT EXISTS arbitrarytagvalues ON event USING gin (tagvalues);
`) `)
if b.QueryLimit < 1 {
b.QueryLimit = 100
}
return err return err
} }

1
storage/postgresql/postgresql.go
View File

@ -7,4 +7,5 @@ import (
type PostgresBackend struct { type PostgresBackend struct {
*sqlx.DB *sqlx.DB
DatabaseURL string DatabaseURL string
QueryLimit int
} }

10
storage/postgresql/query.go
View File

@ -15,7 +15,7 @@ import (
func (b PostgresBackend) QueryEvents(ctx context.Context, filter *nostr.Filter) (ch chan *nostr.Event, err error) { func (b PostgresBackend) QueryEvents(ctx context.Context, filter *nostr.Filter) (ch chan *nostr.Event, err error) {
ch = make(chan *nostr.Event) ch = make(chan *nostr.Event)
query, params, err := queryEventsSql(filter, false) query, params, err := b.queryEventsSql(filter, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -45,7 +45,7 @@ func (b PostgresBackend) QueryEvents(ctx context.Context, filter *nostr.Filter)
} }
func (b PostgresBackend) CountEvents(ctx context.Context, filter *nostr.Filter) (int64, error) { func (b PostgresBackend) CountEvents(ctx context.Context, filter *nostr.Filter) (int64, error) {
query, params, err := queryEventsSql(filter, true) query, params, err := b.queryEventsSql(filter, true)
if err != nil { if err != nil {
return 0, err return 0, err
} }
@ -57,7 +57,7 @@ func (b PostgresBackend) CountEvents(ctx context.Context, filter *nostr.Filter)
return count, nil return count, nil
} }
func queryEventsSql(filter *nostr.Filter, doCount bool) (string, []any, error) { func (b PostgresBackend) queryEventsSql(filter *nostr.Filter, doCount bool) (string, []any, error) {
var conditions []string var conditions []string
var params []any var params []any
@ -172,8 +172,8 @@ func queryEventsSql(filter *nostr.Filter, doCount bool) (string, []any, error) {
conditions = append(conditions, "true") conditions = append(conditions, "true")
} }
if filter.Limit < 1 || filter.Limit > 100 { if filter.Limit < 1 || filter.Limit > b.QueryLimit {
params = append(params, 100) params = append(params, b.QueryLimit)
} else { } else {
params = append(params, filter.Limit) params = append(params, filter.Limit)
} }

158
storage/postgresql/query_test.go
View File

@ -12,21 +12,52 @@ import (
func TestQueryEventsSql(t *testing.T) { func TestQueryEventsSql(t *testing.T) {
var tests = []struct { var tests = []struct {
name string name string
filter *nostr.Filter backend PostgresBackend
query string filter *nostr.Filter
params []any query string
err error params []any
err error
}{ }{
{ {
name: "empty filter", name: "empty filter",
filter: &nostr.Filter{}, backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{},
query: "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event WHERE true ORDER BY created_at DESC LIMIT $1",
params: []any{100},
err: nil,
},
{
name: "large query limit",
backend: PostgresBackend{QueryLimit: 1000},
filter: &nostr.Filter{},
query: "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event WHERE true ORDER BY created_at DESC LIMIT $1",
params: []any{1000},
err: nil,
},
{
name: "valid filter limit",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{
Limit: 50,
},
query: "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event WHERE true ORDER BY created_at DESC LIMIT $1",
params: []any{50},
err: nil,
},
{
name: "too large filter limit",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{
Limit: 2000,
},
query: "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event WHERE true ORDER BY created_at DESC LIMIT $1", query: "SELECT id, pubkey, created_at, kind, tags, content, sig FROM event WHERE true ORDER BY created_at DESC LIMIT $1",
params: []any{100}, params: []any{100},
err: nil, err: nil,
}, },
{ {
name: "ids filter", name: "ids filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: []string{"083ec57f36a7b39ab98a57bedab4f85355b2ee89e4b205bed58d7c3ef9edd294"}, IDs: []string{"083ec57f36a7b39ab98a57bedab4f85355b2ee89e4b205bed58d7c3ef9edd294"},
}, },
@ -38,7 +69,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "kind filter", name: "kind filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: []int{1, 2, 3}, Kinds: []int{1, 2, 3},
}, },
@ -50,7 +82,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "authors filter", name: "authors filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: []string{"7bdef7bdebb8721f77927d0e77c66059360fa62371fdf15f3add93923a613229"}, Authors: []string{"7bdef7bdebb8721f77927d0e77c66059360fa62371fdf15f3add93923a613229"},
}, },
@ -63,14 +96,16 @@ func TestQueryEventsSql(t *testing.T) {
}, },
// errors // errors
{ {
name: "nil filter", name: "nil filter",
filter: nil, backend: PostgresBackend{QueryLimit: 100},
query: "", filter: nil,
params: nil, query: "",
err: fmt.Errorf("filter cannot be null"), params: nil,
err: fmt.Errorf("filter cannot be null"),
}, },
{ {
name: "too many ids", name: "too many ids",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: strSlice(501), IDs: strSlice(501),
}, },
@ -80,7 +115,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "invalid ids", name: "invalid ids",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: []string{"stuff"}, IDs: []string{"stuff"},
}, },
@ -90,7 +126,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many authors", name: "too many authors",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: strSlice(501), Authors: strSlice(501),
}, },
@ -100,7 +137,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "invalid authors", name: "invalid authors",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: []string{"stuff"}, Authors: []string{"stuff"},
}, },
@ -110,7 +148,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many kinds", name: "too many kinds",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: intSlice(11), Kinds: intSlice(11),
}, },
@ -120,7 +159,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "no kinds", name: "no kinds",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: []int{}, Kinds: []int{},
}, },
@ -130,7 +170,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "tags of empty array", name: "tags of empty array",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Tags: nostr.TagMap{ Tags: nostr.TagMap{
"#e": []string{}, "#e": []string{},
@ -142,7 +183,8 @@ func TestQueryEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many tag values", name: "too many tag values",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Tags: nostr.TagMap{ Tags: nostr.TagMap{
"#e": strSlice(11), "#e": strSlice(11),
@ -157,7 +199,7 @@ func TestQueryEventsSql(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
query, params, err := queryEventsSql(tt.filter, false) query, params, err := tt.backend.queryEventsSql(tt.filter, false)
assert.Equal(t, tt.err, err) assert.Equal(t, tt.err, err)
if err != nil { if err != nil {
return return
@ -191,21 +233,24 @@ func strSlice(n int) []string {
func TestCountEventsSql(t *testing.T) { func TestCountEventsSql(t *testing.T) {
var tests = []struct { var tests = []struct {
name string name string
filter *nostr.Filter backend PostgresBackend
query string filter *nostr.Filter
params []any query string
err error params []any
err error
}{ }{
{ {
name: "empty filter", name: "empty filter",
filter: &nostr.Filter{}, backend: PostgresBackend{QueryLimit: 100},
query: "SELECT COUNT(*) FROM event WHERE true ORDER BY created_at DESC LIMIT $1", filter: &nostr.Filter{},
params: []any{100}, query: "SELECT COUNT(*) FROM event WHERE true ORDER BY created_at DESC LIMIT $1",
err: nil, params: []any{100},
err: nil,
}, },
{ {
name: "ids filter", name: "ids filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: []string{"083ec57f36a7b39ab98a57bedab4f85355b2ee89e4b205bed58d7c3ef9edd294"}, IDs: []string{"083ec57f36a7b39ab98a57bedab4f85355b2ee89e4b205bed58d7c3ef9edd294"},
}, },
@ -217,7 +262,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "kind filter", name: "kind filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: []int{1, 2, 3}, Kinds: []int{1, 2, 3},
}, },
@ -229,7 +275,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "authors filter", name: "authors filter",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: []string{"7bdef7bdebb8721f77927d0e77c66059360fa62371fdf15f3add93923a613229"}, Authors: []string{"7bdef7bdebb8721f77927d0e77c66059360fa62371fdf15f3add93923a613229"},
}, },
@ -242,14 +289,16 @@ func TestCountEventsSql(t *testing.T) {
}, },
// errors // errors
{ {
name: "nil filter", name: "nil filter",
filter: nil, backend: PostgresBackend{QueryLimit: 100},
query: "", filter: nil,
params: nil, query: "",
err: fmt.Errorf("filter cannot be null"), params: nil,
err: fmt.Errorf("filter cannot be null"),
}, },
{ {
name: "too many ids", name: "too many ids",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: strSlice(501), IDs: strSlice(501),
}, },
@ -259,7 +308,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "invalid ids", name: "invalid ids",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
IDs: []string{"stuff"}, IDs: []string{"stuff"},
}, },
@ -269,7 +319,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many authors", name: "too many authors",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: strSlice(501), Authors: strSlice(501),
}, },
@ -279,7 +330,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "invalid authors", name: "invalid authors",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Authors: []string{"stuff"}, Authors: []string{"stuff"},
}, },
@ -289,7 +341,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many kinds", name: "too many kinds",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: intSlice(11), Kinds: intSlice(11),
}, },
@ -299,7 +352,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "no kinds", name: "no kinds",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Kinds: []int{}, Kinds: []int{},
}, },
@ -309,7 +363,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "tags of empty array", name: "tags of empty array",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Tags: nostr.TagMap{ Tags: nostr.TagMap{
"#e": []string{}, "#e": []string{},
@ -321,7 +376,8 @@ func TestCountEventsSql(t *testing.T) {
err: nil, err: nil,
}, },
{ {
name: "too many tag values", name: "too many tag values",
backend: PostgresBackend{QueryLimit: 100},
filter: &nostr.Filter{ filter: &nostr.Filter{
Tags: nostr.TagMap{ Tags: nostr.TagMap{
"#e": strSlice(11), "#e": strSlice(11),
@ -336,7 +392,7 @@ func TestCountEventsSql(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
query, params, err := queryEventsSql(tt.filter, true) query, params, err := tt.backend.queryEventsSql(tt.filter, true)
assert.Equal(t, tt.err, err) assert.Equal(t, tt.err, err)
if err != nil { if err != nil {
return return