ratelimits.

* optimized return messages with more context why the policy blocks an event

README.md

@@ -128,3 +128,13 @@ Fear no more. Using the https://github.com/fiatjaf/eventstore module you get a b
 	relay.CountEvents = append(relay.CountEvents, db.CountEvents)
 	relay.DeleteEvent = append(relay.DeleteEvent, db.DeleteEvent)
 ```
+
+### But I don't want to write a bunch of custom policies!
+
+Fear no more. We have a bunch of common policies written in the `github.com/fiatjaf/khatru/policies` package and also a handpicked selection of base sane defaults, which you can apply with:
+
+```go
+	policies.ApplySaneDefaults(relay)
+```
+
+Contributions to this are very much welcomed.

adding.go

@@ -10,17 +10,17 @@ import (
 )
 
 // AddEvent sends an event through then normal add pipeline, as if it was received from a websocket.
-func (rl *Relay) AddEvent(ctx context.Context, evt *nostr.Event) error {
+func (rl *Relay) AddEvent(ctx context.Context, evt *nostr.Event) (skipBroadcast bool, writeError error) {
 	if evt == nil {
-		return errors.New("error: event is nil")
+		return false, errors.New("error: event is nil")
 	}
 
 	for _, reject := range rl.RejectEvent {
 		if reject, msg := reject(ctx, evt); reject {
 			if msg == "" {
-				return errors.New("blocked: no reason")
+				return false, errors.New("blocked: no reason")
 			} else {
-				return errors.New(nostr.NormalizeOKMessage(msg, "blocked"))
+				return false, errors.New(nostr.NormalizeOKMessage(msg, "blocked"))
 			}
 		}
 	}
@@ -31,29 +31,53 @@ func (rl *Relay) AddEvent(ctx context.Context, evt *nostr.Event) error {
 			oee(ctx, evt)
 		}
 	} else {
+		// will store
+
+		// but first check if we already have it
+		filter := nostr.Filter{IDs: []string{evt.ID}}
+		for _, query := range rl.QueryEvents {
+			ch, err := query(ctx, filter)
+			if err != nil {
+				continue
+			}
+			for range ch {
+				// if we run this it means we already have this event, so we just return a success and exit
+				return true, nil
+			}
+		}
+
+		// if it's replaceable we first delete old versions
 		if evt.Kind == 0 || evt.Kind == 3 || (10000 <= evt.Kind && evt.Kind < 20000) {
 			// replaceable event, delete before storing
+			filter := nostr.Filter{Authors: []string{evt.PubKey}, Kinds: []int{evt.Kind}}
 			for _, query := range rl.QueryEvents {
-				ch, err := query(ctx, nostr.Filter{Authors: []string{evt.PubKey}, Kinds: []int{evt.Kind}})
+				ch, err := query(ctx, filter)
 				if err != nil {
 					continue
 				}
-				if previous := <-ch; previous != nil && isOlder(previous, evt) {
-					for _, del := range rl.DeleteEvent {
-						del(ctx, previous)
+				for previous := range ch {
+					if isOlder(previous, evt) {
+						for _, del := range rl.DeleteEvent {
+							del(ctx, previous)
+						}
 					}
 				}
 			}
 		} else if 30000 <= evt.Kind && evt.Kind < 40000 {
 			// parameterized replaceable event, delete before storing
 			d := evt.Tags.GetFirst([]string{"d", ""})
-			if d != nil {
-				for _, query := range rl.QueryEvents {
-					ch, err := query(ctx, nostr.Filter{Authors: []string{evt.PubKey}, Kinds: []int{evt.Kind}, Tags: nostr.TagMap{"d": []string{d.Value()}}})
-					if err != nil {
-						continue
-					}
-					if previous := <-ch; previous != nil && isOlder(previous, evt) {
+			if d == nil {
+				return false, fmt.Errorf("invalid: missing 'd' tag on parameterized replaceable event")
+			}
+
+			filter := nostr.Filter{Authors: []string{evt.PubKey}, Kinds: []int{evt.Kind}, Tags: nostr.TagMap{"d": []string{(*d)[1]}}}
+			for _, query := range rl.QueryEvents {
+				ch, err := query(ctx, filter)
+				if err != nil {
+					continue
+				}
+				for previous := range ch {
+					if isOlder(previous, evt) {
 						for _, del := range rl.DeleteEvent {
 							del(ctx, previous)
 						}
@@ -67,9 +91,9 @@ func (rl *Relay) AddEvent(ctx context.Context, evt *nostr.Event) error {
 			if saveErr := store(ctx, evt); saveErr != nil {
 				switch saveErr {
 				case eventstore.ErrDupEvent:
-					return nil
+					return true, nil
 				default:
-					return fmt.Errorf(nostr.NormalizeOKMessage(saveErr.Error(), "error"))
+					return false, fmt.Errorf(nostr.NormalizeOKMessage(saveErr.Error(), "error"))
 				}
 			}
 		}
@@ -79,5 +103,5 @@ func (rl *Relay) AddEvent(ctx context.Context, evt *nostr.Event) error {
 		}
 	}
 
-	return nil
+	return false, nil
 }

deleting.go

@@ -34,7 +34,9 @@ func (rl *Relay) handleDeleteRequest(ctx context.Context, evt *nostr.Event) erro
 				if acceptDeletion {
 					// delete it
 					for _, del := range rl.DeleteEvent {
-						del(ctx, target)
+						if err := del(ctx, target); err != nil {
+							return err
+						}
 					}
 				} else {
 					// fail and stop here

go.mod

@@ -5,10 +5,9 @@ go 1.21.4
 require (
 	github.com/fasthttp/websocket v1.5.7
 	github.com/fiatjaf/eventstore v0.3.8
-	github.com/nbd-wtf/go-nostr v0.28.1
+	github.com/nbd-wtf/go-nostr v0.30.0
 	github.com/puzpuzpuz/xsync/v3 v3.0.2
 	github.com/rs/cors v1.7.0
-	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a
 )
 
 require (

go.sum

@@ -113,8 +113,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
 github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/nbd-wtf/go-nostr v0.28.1 h1:XQi/lBsigBXHRm7IDBJE7SR9citCh9srgf8sA5iVW3A=
-github.com/nbd-wtf/go-nostr v0.28.1/go.mod h1:OQ8sNLFJnsj17BdqZiLSmjJBIFTfDqckEYC3utS4qoY=
+github.com/nbd-wtf/go-nostr v0.30.0 h1:rN085pe4IxmSBVht8LChZbWLggonjA8hPIk8l4/+Hjk=
+github.com/nbd-wtf/go-nostr v0.30.0/go.mod h1:tiKJY6fWYSujbTQb201Y+IQ3l4szqYVt+fsTnsm7FCk=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -126,8 +126,6 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
 github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
-github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM=
-github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

handlers.go

@@ -34,6 +34,13 @@ func (rl *Relay) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
+	for _, reject := range rl.RejectConnection {
+		if reject(r) {
+			w.WriteHeader(418) // I'm a teapot
+			return
+		}
+	}
+
 	conn, err := rl.upgrader.Upgrade(w, r, nil)
 	if err != nil {
 		rl.Log.Printf("failed to upgrade websocket: %v\n", err)
@@ -161,12 +168,13 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 
 					var ok bool
 					var writeErr error
+					var skipBroadcast bool
 					if env.Event.Kind == 5 {
 						// this always returns "blocked: " whenever it returns an error
 						writeErr = rl.handleDeleteRequest(ctx, &env.Event)
 					} else {
 						// this will also always return a prefixed reason
-						writeErr = rl.AddEvent(ctx, &env.Event)
+						skipBroadcast, writeErr = rl.AddEvent(ctx, &env.Event)
 					}
 
 					var reason string
@@ -175,7 +183,9 @@ func (rl *Relay) HandleWebsocket(w http.ResponseWriter, r *http.Request) {
 						for _, ovw := range rl.OverwriteResponseEvent {
 							ovw(ctx, &env.Event)
 						}
-						notifyListeners(&env.Event)
+						if !skipBroadcast {
+							notifyListeners(&env.Event)
+						}
 					} else {
 						reason = writeErr.Error()
 						if strings.HasPrefix(reason, "auth-required:") {

helpers.go

@@ -1,6 +1,7 @@
 package khatru
 
 import (
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -34,3 +35,43 @@ func getServiceBaseURL(r *http.Request) string {
 	}
 	return proto + "://" + host
 }
+
+var privateMasks = func() []net.IPNet {
+	privateCIDRs := []string{
+		"127.0.0.0/8",
+		"10.0.0.0/8",
+		"172.16.0.0/12",
+		"192.168.0.0/16",
+		"fc00::/7",
+	}
+	masks := make([]net.IPNet, len(privateCIDRs))
+	for i, cidr := range privateCIDRs {
+		_, netw, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil
+		}
+		masks[i] = *netw
+	}
+	return masks
+}()
+
+func isPrivate(ip net.IP) bool {
+	for _, mask := range privateMasks {
+		if mask.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+func GetIPFromRequest(r *http.Request) string {
+	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
+		for _, v := range strings.Split(xffh, ",") {
+			if ip := net.ParseIP(strings.TrimSpace(v)); ip != nil && ip.IsGlobalUnicast() && !isPrivate(ip) {
+				return ip.String()
+			}
+		}
+	}
+	ip, _, _ := net.SplitHostPort(r.RemoteAddr)
+	return ip
+}

policies/events.go

@@ -2,8 +2,9 @@ package policies
 
 import (
 	"context"
-
+	"fmt"
 	"slices"
+	"strings"
 
 	"github.com/nbd-wtf/go-nostr"
 )
@@ -14,6 +15,9 @@ import (
 // If ignoreKinds is given this restriction will not apply to these kinds (useful for allowing a bigger).
 // If onlyKinds is given then all other kinds will be ignored.
 func PreventTooManyIndexableTags(max int, ignoreKinds []int, onlyKinds []int) func(context.Context, *nostr.Event) (bool, string) {
+	slices.Sort(ignoreKinds)
+	slices.Sort(onlyKinds)
+
 	ignore := func(kind int) bool { return false }
 	if len(ignoreKinds) > 0 {
 		ignore = func(kind int) bool {
@@ -74,21 +78,25 @@ func RestrictToSpecifiedKinds(kinds ...uint16) func(context.Context, *nostr.Even
 		}
 	}
 
+	// sort the kinds in increasing order
+	slices.Sort(kinds)
+
 	return func(ctx context.Context, event *nostr.Event) (reject bool, msg string) {
 		// these are cheap and very questionable optimizations, but they exist for a reason:
 		// we would have to ensure that the kind number is within the bounds of a uint16 anyway
 		if event.Kind > max {
-			return true, "event kind not allowed"
+			return true, fmt.Sprintf("event kind not allowed (it should be lower than %d)", max)
 		}
 		if event.Kind < min {
-			return true, "event kind not allowed"
+			return true, fmt.Sprintf("event kind not allowed (it should be higher than %d)", min)
 		}
 
 		// hopefully this map of uint16s is very fast
 		if _, allowed := slices.BinarySearch(kinds, uint16(event.Kind)); allowed {
 			return false, ""
 		}
-		return true, "event kind not allowed"
+
+		return true, fmt.Sprintf("received event kind %d not allowed", event.Kind)
 	}
 }
 
@@ -109,3 +117,7 @@ func PreventTimestampsInTheFuture(thresholdSeconds nostr.Timestamp) func(context
 		return false, ""
 	}
 }
+
+func RejectEventsWithBase64Media(ctx context.Context, evt *nostr.Event) (bool, string) {
+	return strings.Contains(evt.Content, "data:image/") || strings.Contains(evt.Content, "data:video/"), "event with base64 media"
+}

policies/filters.go

@@ -2,7 +2,6 @@ package policies
 
 import (
 	"context"
-
 	"slices"
 
 	"github.com/nbd-wtf/go-nostr"
@@ -48,7 +47,7 @@ func NoSearchQueries(ctx context.Context, filter nostr.Filter) (reject bool, msg
 func RemoveSearchQueries(ctx context.Context, filter *nostr.Filter) {
 	if filter.Search != "" {
 		filter.Search = ""
-		filter.Limit = -1 // signals that this query should be just skipped
+		filter.LimitZero = true // signals that this query should be just skipped
 	}
 }
 
@@ -63,7 +62,7 @@ func RemoveAllButKinds(kinds ...uint16) func(context.Context, *nostr.Filter) {
 			}
 			filter.Kinds = newKinds
 			if len(filter.Kinds) == 0 {
-				filter.Limit = -1 // signals that this query should be just skipped
+				filter.LimitZero = true // signals that this query should be just skipped
 			}
 		}
 	}
@@ -78,7 +77,7 @@ func RemoveAllButTags(tagNames ...string) func(context.Context, *nostr.Filter) {
 				}
 			}
 			if len(filter.Tags) == 0 {
-				filter.Limit = -1 // signals that this query should be just skipped
+				filter.LimitZero = true // signals that this query should be just skipped
 			}
 		}
 	}

policies/helpers.go

@@ -0,0 +1,43 @@
+package policies
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/puzpuzpuz/xsync/v3"
+)
+
+func startRateLimitSystem[K comparable](
+	tokensPerInterval int,
+	interval time.Duration,
+	maxTokens int,
+) func(key K) (ratelimited bool) {
+	negativeBuckets := xsync.NewMapOf[K, *atomic.Int32]()
+	maxTokensInt32 := int32(maxTokens)
+
+	go func() {
+		for {
+			time.Sleep(interval)
+			negativeBuckets.Range(func(key K, bucket *atomic.Int32) bool {
+				newv := bucket.Add(int32(-tokensPerInterval))
+				if newv <= 0 {
+					negativeBuckets.Delete(key)
+				}
+				return true
+			})
+		}
+	}()
+
+	return func(key K) bool {
+		nb, _ := negativeBuckets.LoadOrStore(key, &atomic.Int32{})
+
+		if nb.Load() < maxTokensInt32 {
+			nb.Add(1)
+			// rate limit not reached yet
+			return false
+		}
+
+		// rate limit reached
+		return true
+	}
+}

policies/ratelimits.go

@@ -0,0 +1,42 @@
+package policies
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/fiatjaf/khatru"
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func EventIPRateLimiter(tokensPerInterval int, interval time.Duration, maxTokens int) func(ctx context.Context, _ *nostr.Event) (reject bool, msg string) {
+	rl := startRateLimitSystem[string](tokensPerInterval, interval, maxTokens)
+
+	return func(ctx context.Context, _ *nostr.Event) (reject bool, msg string) {
+		return rl(khatru.GetIP(ctx)), "rate-limited: slow down, please"
+	}
+}
+
+func EventPubKeyRateLimiter(tokensPerInterval int, interval time.Duration, maxTokens int) func(ctx context.Context, _ *nostr.Event) (reject bool, msg string) {
+	rl := startRateLimitSystem[string](tokensPerInterval, interval, maxTokens)
+
+	return func(ctx context.Context, evt *nostr.Event) (reject bool, msg string) {
+		return rl(evt.PubKey), "rate-limited: slow down, please"
+	}
+}
+
+func ConnectionRateLimiter(tokensPerInterval int, interval time.Duration, maxTokens int) func(r *http.Request) bool {
+	rl := startRateLimitSystem[string](tokensPerInterval, interval, maxTokens)
+
+	return func(r *http.Request) bool {
+		return rl(khatru.GetIPFromRequest(r))
+	}
+}
+
+func FilterIPRateLimiter(tokensPerInterval int, interval time.Duration, maxTokens int) func(ctx context.Context, _ nostr.Filter) (reject bool, msg string) {
+	rl := startRateLimitSystem[string](tokensPerInterval, interval, maxTokens)
+
+	return func(ctx context.Context, _ nostr.Filter) (reject bool, msg string) {
+		return rl(khatru.GetIP(ctx)), "rate-limited: there is a bug in the client, no one should be making so many requests"
+	}
+}

policies/sane_defaults.go

@@ -0,0 +1,24 @@
+package policies
+
+import (
+	"time"
+
+	"github.com/fiatjaf/khatru"
+)
+
+func ApplySaneDefaults(relay *khatru.Relay) {
+	relay.RejectEvent = append(relay.RejectEvent,
+		RejectEventsWithBase64Media,
+		EventIPRateLimiter(2, time.Minute*3, 5),
+	)
+
+	relay.RejectFilter = append(relay.RejectFilter,
+		NoEmptyFilters,
+		NoComplexFilters,
+		FilterIPRateLimiter(20, time.Minute, 100),
+	)
+
+	relay.RejectConnection = append(relay.RejectConnection,
+		ConnectionRateLimiter(1, time.Minute*5, 3),
+	)
+}

relay.go

@@ -45,6 +45,7 @@ type Relay struct {
 	RejectEvent               []func(ctx context.Context, event *nostr.Event) (reject bool, msg string)
 	RejectFilter              []func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
 	RejectCountFilter         []func(ctx context.Context, filter nostr.Filter) (reject bool, msg string)
+	RejectConnection          []func(r *http.Request) bool
 	OverwriteDeletionOutcome  []func(ctx context.Context, target *nostr.Event, deletion *nostr.Event) (acceptDeletion bool, msg string)
 	OverwriteResponseEvent    []func(ctx context.Context, event *nostr.Event)
 	OverwriteFilter           []func(ctx context.Context, filter *nostr.Filter)

responding.go

@@ -17,9 +17,8 @@ func (rl *Relay) handleRequest(ctx context.Context, id string, eose *sync.WaitGr
 		ovw(ctx, &filter)
 	}
 
-	if filter.Limit < 0 {
-		// this is a special situation through which the implementor signals to us that it doesn't want
-		// to event perform any queries whatsoever
+	if filter.LimitZero {
+		// don't do any queries, just subscribe to future events
 		return nil
 	}
 
@@ -29,7 +28,6 @@ func (rl *Relay) handleRequest(ctx context.Context, id string, eose *sync.WaitGr
 	// filter we can just reject it)
 	for _, reject := range rl.RejectFilter {
 		if reject, msg := reject(ctx, filter); reject {
-			ws.WriteJSON(nostr.NoticeEnvelope(msg))
 			return errors.New(nostr.NormalizeOKMessage(msg, "blocked"))
 		}
 	}

utils.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/nbd-wtf/go-nostr"
-	"github.com/sebest/xff"
 )
 
 const (
@@ -31,7 +30,7 @@ func GetAuthed(ctx context.Context) string {
 }
 
 func GetIP(ctx context.Context) string {
-	return xff.GetRemoteAddr(GetConnection(ctx).Request)
+	return GetIPFromRequest(GetConnection(ctx).Request)
 }
 
 func GetSubscriptionID(ctx context.Context) string {