From 56e9a5a70970ca44b119a4526199fe218dc30046 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Tue, 25 Feb 2025 17:57:15 -0300 Subject: [PATCH] implement envelope, event and filter parsing with simdjson-go. --- envelopes.go | 253 +++++++++++++++++++++++++++++++++++++++++++-- envelopes_test.go | 153 +++++++++++++++++++++++++++ event_simdjson.go | 84 +++++++++++++++ filter_simdjson.go | 107 +++++++++++++++++++ go.mod | 6 +- go.sum | 8 ++ 6 files changed, 600 insertions(+), 11 deletions(-) create mode 100644 event_simdjson.go create mode 100644 filter_simdjson.go diff --git a/envelopes.go b/envelopes.go index 9b9a539..c6cc12a 100644 --- a/envelopes.go +++ b/envelopes.go @@ -3,14 +3,78 @@ package nostr import ( "bytes" "encoding/hex" + "errors" "fmt" "strconv" "github.com/mailru/easyjson" jwriter "github.com/mailru/easyjson/jwriter" + "github.com/minio/simdjson-go" "github.com/tidwall/gjson" ) +var ( + labelEvent = []byte("EVENT") + labelReq = []byte("REQ") + labelCount = []byte("COUNT") + labelNotice = []byte("NOTICE") + labelEose = []byte("EOSE") + labelOk = []byte("OK") + labelAuth = []byte("AUTH") + labelClosed = []byte("CLOSED") + labelClose = []byte("CLOSE") + + UnknownLabel = errors.New("unknown envelope label") +) + +func ParseMessageSIMD(message []byte, reuse *simdjson.ParsedJson) (Envelope, error) { + parsed, err := simdjson.Parse(message, reuse) + if err != nil { + return nil, fmt.Errorf("simdjson parse failed: %w", err) + } + + iter := parsed.Iter() + iter.AdvanceInto() + if t := iter.Advance(); t != simdjson.TypeArray { + return nil, fmt.Errorf("top-level must be an array") + } + arr, _ := iter.Array(nil) + iter = arr.Iter() + iter.Advance() + label, _ := iter.StringBytes() + + var v Envelope + + switch { + case bytes.Equal(label, labelEvent): + v = &EventEnvelope{} + case bytes.Equal(label, labelReq): + v = &ReqEnvelope{} + case bytes.Equal(label, labelCount): + v = &CountEnvelope{} + case bytes.Equal(label, labelNotice): + x := NoticeEnvelope("") + v = &x + case bytes.Equal(label, labelEose): + x := EOSEEnvelope("") + v = &x + case bytes.Equal(label, labelOk): + v = &OKEnvelope{} + case bytes.Equal(label, labelAuth): + v = &AuthEnvelope{} + case bytes.Equal(label, labelClosed): + v = &ClosedEnvelope{} + case bytes.Equal(label, labelClose): + x := CloseEnvelope("") + v = &x + default: + return nil, UnknownLabel + } + + err = v.UnmarshalSIMD(iter) + return v, err +} + func ParseMessage(message []byte) Envelope { firstComma := bytes.Index(message, []byte{','}) if firstComma == -1 { @@ -20,25 +84,25 @@ func ParseMessage(message []byte) Envelope { var v Envelope switch { - case bytes.Contains(label, []byte("EVENT")): + case bytes.Contains(label, labelEvent): v = &EventEnvelope{} - case bytes.Contains(label, []byte("REQ")): + case bytes.Contains(label, labelReq): v = &ReqEnvelope{} - case bytes.Contains(label, []byte("COUNT")): + case bytes.Contains(label, labelCount): v = &CountEnvelope{} - case bytes.Contains(label, []byte("NOTICE")): + case bytes.Contains(label, labelNotice): x := NoticeEnvelope("") v = &x - case bytes.Contains(label, []byte("EOSE")): + case bytes.Contains(label, labelEose): x := EOSEEnvelope("") v = &x - case bytes.Contains(label, []byte("OK")): + case bytes.Contains(label, labelOk): v = &OKEnvelope{} - case bytes.Contains(label, []byte("AUTH")): + case bytes.Contains(label, labelAuth): v = &AuthEnvelope{} - case bytes.Contains(label, []byte("CLOSED")): + case bytes.Contains(label, labelClosed): v = &ClosedEnvelope{} - case bytes.Contains(label, []byte("CLOSE")): + case bytes.Contains(label, labelClose): x := CloseEnvelope("") v = &x default: @@ -53,6 +117,7 @@ func ParseMessage(message []byte) Envelope { type Envelope interface { Label() string + UnmarshalSIMD(simdjson.Iter) error UnmarshalJSON([]byte) error MarshalJSON() ([]byte, error) String() string @@ -90,6 +155,25 @@ func (v *EventEnvelope) UnmarshalJSON(data []byte) error { } } +func (v *EventEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + // we may or may not have a subscription ID, so peek + if iter.PeekNext() == simdjson.TypeString { + iter.Advance() + // we have a subscription ID + subID, err := iter.String() + if err != nil { + return err + } + v.SubscriptionID = &subID + } + + // now get the event + if typ := iter.Advance(); typ == simdjson.TypeNone { + return fmt.Errorf("missing event") + } + return v.Event.UnmarshalSIMD(&iter) +} + func (v EventEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["EVENT",`) @@ -129,6 +213,44 @@ func (v *ReqEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *ReqEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + var err error + + // we must have a subscription id + if typ := iter.Advance(); typ == simdjson.TypeString { + v.SubscriptionID, err = iter.String() + if err != nil { + return err + } + } else { + return fmt.Errorf("unexpected %s for REQ subscription id", typ) + } + + // now get the filters + v.Filters = make(Filters, 0, 1) + tempIter := &simdjson.Iter{} // make a new iterator here because there may come multiple filters + for { + if typ, err := iter.AdvanceIter(tempIter); err != nil { + return err + } else if typ == simdjson.TypeNone { + break + } else { + } + + var filter Filter + if err := filter.UnmarshalSIMD(tempIter); err != nil { + return err + } + v.Filters = append(v.Filters, filter) + } + + if len(v.Filters) == 0 { + return fmt.Errorf("need at least one filter") + } + + return nil +} + func (v ReqEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["REQ","`) @@ -193,6 +315,53 @@ func (v *CountEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *CountEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + var err error + + // this has two cases: + // in the first case (request from client) this is like REQ except with always one filter + // in the other (response from relay) we have a json object response + // but both cases start with a subscription id + + if typ := iter.Advance(); typ == simdjson.TypeString { + v.SubscriptionID, err = iter.String() + if err != nil { + return err + } + } else { + return fmt.Errorf("unexpected %s for COUNT subscription id", typ) + } + + // now get either a single filter or stuff from the json object + if typ := iter.Advance(); typ == simdjson.TypeNone { + return fmt.Errorf("missing json object") + } + + if el, err := iter.FindElement(nil, "count"); err == nil { + c, _ := el.Iter.Uint() + count := int64(c) + v.Count = &count + if el, err = iter.FindElement(nil, "hll"); err == nil { + if hllHex, err := el.Iter.StringBytes(); err != nil || len(hllHex) != 512 { + return fmt.Errorf("hll is malformed") + } else { + v.HyperLogLog = make([]byte, 256) + if _, err := hex.Decode(v.HyperLogLog, hllHex); err != nil { + return fmt.Errorf("hll is invalid hex") + } + } + } + } else { + var filter Filter + if err := filter.UnmarshalSIMD(&iter); err != nil { + return err + } + v.Filters = Filters{filter} + } + + return nil +} + func (v CountEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["COUNT","`) @@ -237,6 +406,14 @@ func (v *NoticeEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *NoticeEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + msg, _ := iter.String() + *v = NoticeEnvelope(msg) + } + return nil +} + func (v NoticeEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["NOTICE",`) @@ -263,6 +440,14 @@ func (v *EOSEEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *EOSEEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + msg, _ := iter.String() + *v = EOSEEnvelope(msg) + } + return nil +} + func (v EOSEEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["EOSE",`) @@ -291,6 +476,14 @@ func (v *CloseEnvelope) UnmarshalJSON(data []byte) error { } } +func (v *CloseEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + msg, _ := iter.String() + *v = CloseEnvelope(msg) + } + return nil +} + func (v CloseEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["CLOSE",`) @@ -322,6 +515,16 @@ func (v *ClosedEnvelope) UnmarshalJSON(data []byte) error { } } +func (v *ClosedEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + v.SubscriptionID, _ = iter.String() + } + if typ := iter.Advance(); typ == simdjson.TypeString { + v.Reason, _ = iter.String() + } + return nil +} + func (v ClosedEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["CLOSED",`) @@ -357,6 +560,23 @@ func (v *OKEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *OKEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + v.EventID, _ = iter.String() + } else { + return fmt.Errorf("unexpected %s for OK id", typ) + } + if typ := iter.Advance(); typ == simdjson.TypeBool { + v.OK, _ = iter.Bool() + } else { + return fmt.Errorf("unexpected %s for OK status", typ) + } + if typ := iter.Advance(); typ == simdjson.TypeString { + v.Reason, _ = iter.String() + } + return nil +} + func (v OKEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["OK","`) @@ -398,6 +618,21 @@ func (v *AuthEnvelope) UnmarshalJSON(data []byte) error { return nil } +func (v *AuthEnvelope) UnmarshalSIMD(iter simdjson.Iter) error { + if typ := iter.Advance(); typ == simdjson.TypeString { + // we have a challenge + subID, err := iter.String() + if err != nil { + return err + } + v.Challenge = &subID + return nil + } else { + // we have an event + return v.Event.UnmarshalSIMD(&iter) + } +} + func (v AuthEnvelope) MarshalJSON() ([]byte, error) { w := jwriter.Writer{NoEscapeHTML: true} w.RawString(`["AUTH",`) diff --git a/envelopes_test.go b/envelopes_test.go index a2fc538..10732b1 100644 --- a/envelopes_test.go +++ b/envelopes_test.go @@ -3,7 +3,9 @@ package nostr import ( "testing" + "github.com/minio/simdjson-go" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEventEnvelopeEncodingAndDecoding(t *testing.T) { @@ -174,4 +176,155 @@ func TestParseMessage(t *testing.T) { } } +func TestParseMessageSIMD(t *testing.T) { + testCases := []struct { + Name string + Message []byte + ExpectedEnvelope Envelope + ExpectedErrorSubstring string + }{ + { + Name: "nil", + Message: nil, + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "parse failed", + }, + { + Name: "empty string", + Message: []byte(""), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "parse failed", + }, + { + Name: "invalid input", + Message: []byte("invalid input"), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "parse failed", + }, + { + Name: "invalid JSON", + Message: []byte("{not valid json}"), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "parse failed", + }, + { + Name: "invalid REQ", + Message: []byte(`["REQ","zzz", {"authors": [23]}]`), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "not string, but int", + }, + { + Name: "same invalid REQ from before, but now valid", + Message: []byte(`["REQ","zzz", {"kinds": [23]}]`), + ExpectedEnvelope: &ReqEnvelope{SubscriptionID: "zzz", Filters: Filters{{Kinds: []int{23}}}}, + }, + { + Name: "different invalid REQ", + Message: []byte(`["REQ","zzz", {"authors": "string"}]`), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "next item is not", + }, + { + Name: "yet another", + Message: []byte(`["REQ","zzz", {"unknownfield": "_"}]`), + ExpectedEnvelope: nil, + ExpectedErrorSubstring: "unexpected filter field 'unknownfield'", + }, + { + Name: "EVENT envelope with subscription id", + Message: []byte( + `["EVENT","_",{"kind":1,"id":"dc90c95f09947507c1044e8f48bcf6350aa6bff1507dd4acfc755b9239b5c962","pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","created_at":1644271588,"tags":[],"content":"now that https://blueskyweb.org/blog/2-7-2022-overview was announced we can stop working on nostr?","sig":"230e9d8f0ddaf7eb70b5f7741ccfa37e87a455c9a469282e3464e2052d3192cd63a167e196e381ef9d7e69e9ea43af2443b839974dc85d8aaab9efe1d9296524"}]`, + ), + ExpectedEnvelope: &EventEnvelope{SubscriptionID: ptr("_"), Event: Event{Kind: 1, ID: "dc90c95f09947507c1044e8f48bcf6350aa6bff1507dd4acfc755b9239b5c962", PubKey: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d", CreatedAt: 1644271588, Tags: Tags{}, Content: "now that https://blueskyweb.org/blog/2-7-2022-overview was announced we can stop working on nostr?", Sig: "230e9d8f0ddaf7eb70b5f7741ccfa37e87a455c9a469282e3464e2052d3192cd63a167e196e381ef9d7e69e9ea43af2443b839974dc85d8aaab9efe1d9296524"}}, + }, + { + Name: "EVENT envelope without subscription id", + Message: []byte( + `["EVENT",{"kind":1,"id":"dc90c95f09947507c1044e8f48bcf6350aa6bff1507dd4acfc755b9239b5c962","pubkey":"3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d","created_at":1644271588,"tags":[],"content":"now that https://blueskyweb.org/blog/2-7-2022-overview was announced we can stop working on nostr?","sig":"230e9d8f0ddaf7eb70b5f7741ccfa37e87a455c9a469282e3464e2052d3192cd63a167e196e381ef9d7e69e9ea43af2443b839974dc85d8aaab9efe1d9296524"}]`, + ), + ExpectedEnvelope: &EventEnvelope{Event: Event{Kind: 1, ID: "dc90c95f09947507c1044e8f48bcf6350aa6bff1507dd4acfc755b9239b5c962", PubKey: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d", CreatedAt: 1644271588, Tags: Tags{}, Content: "now that https://blueskyweb.org/blog/2-7-2022-overview was announced we can stop working on nostr?", Sig: "230e9d8f0ddaf7eb70b5f7741ccfa37e87a455c9a469282e3464e2052d3192cd63a167e196e381ef9d7e69e9ea43af2443b839974dc85d8aaab9efe1d9296524"}}, + }, + { + Name: "AUTH envelope with challenge", + Message: []byte(`["AUTH","challenge-string"]`), + ExpectedEnvelope: &AuthEnvelope{Challenge: ptr("challenge-string")}, + }, + { + Name: "AUTH envelope with event", + Message: []byte( + `["AUTH", {"kind":22242,"id":"9b86ca5d2a9b4aa09870e710438c2fd2fcdeca12a18b6f17ab9ebcdbc43f1d4a","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1740505646,"tags":[["relay","ws://localhost:7777","2"],["challenge","3027526784722639360"]],"content":"","sig":"eceb827c4bba1de0ab8ee43f3e98df71194f5bdde0af27b5cda38e5c4338b5f63d31961acb5e3c119fd00ecef8b469867d060b697dbaa6ecee1906b483bc307d"}]`, + ), + ExpectedEnvelope: &AuthEnvelope{Event: Event{Kind: 22242, ID: "9b86ca5d2a9b4aa09870e710438c2fd2fcdeca12a18b6f17ab9ebcdbc43f1d4a", PubKey: "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", CreatedAt: 1740505646, Tags: Tags{Tag{"relay", "ws://localhost:7777", "2"}, Tag{"challenge", "3027526784722639360"}}, Content: "", Sig: "eceb827c4bba1de0ab8ee43f3e98df71194f5bdde0af27b5cda38e5c4338b5f63d31961acb5e3c119fd00ecef8b469867d060b697dbaa6ecee1906b483bc307d"}}, + }, + { + Name: "NOTICE envelope", + Message: []byte(`["NOTICE","test notice message"]`), + ExpectedEnvelope: ptr(NoticeEnvelope("test notice message")), + }, + { + Name: "EOSE envelope", + Message: []byte(`["EOSE","subscription123"]`), + ExpectedEnvelope: ptr(EOSEEnvelope("subscription123")), + }, + { + Name: "CLOSE envelope", + Message: []byte(`["CLOSE","subscription123"]`), + ExpectedEnvelope: ptr(CloseEnvelope("subscription123")), + }, + { + Name: "CLOSED envelope", + Message: []byte(`["CLOSED","subscription123","reason: test closed"]`), + ExpectedEnvelope: &ClosedEnvelope{SubscriptionID: "subscription123", Reason: "reason: test closed"}, + }, + { + Name: "OK envelope", + Message: []byte(`["OK","3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefaaaaa",true,""]`), + ExpectedEnvelope: &OKEnvelope{EventID: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefaaaaa", OK: true, Reason: ""}, + }, + { + Name: "COUNT envelope with just count", + Message: []byte(`["COUNT","sub1",{"count":42}]`), + ExpectedEnvelope: &CountEnvelope{SubscriptionID: "sub1", Count: ptr(int64(42))}, + }, + { + Name: "COUNT envelope with count and hll", + Message: []byte(`["COUNT","sub1",{"count":42, "hll": "0100000101000000000000040000000001020000000002000000000200000003000002040000000101020001010000000000000007000004010000000200040000020400000000000102000002000004010000010000000301000102030002000301000300010000070000000001000004000102010000000400010002000000000103000100010001000001040100020001000000000000010000020000000000030100000001000400010000000000000901010100000000040000000b030000010100010000010000010000000003000000000000010003000100020000000000010000010100000100000104000200030001000300000001000101000102"}]`), + ExpectedEnvelope: &CountEnvelope{SubscriptionID: "sub1", Count: ptr(int64(42)), HyperLogLog: []byte{1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 2, 4, 0, 0, 0, 1, 1, 2, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4, 1, 0, 0, 0, 2, 0, 4, 0, 0, 2, 4, 0, 0, 0, 0, 0, 1, 2, 0, 0, 2, 0, 0, 4, 1, 0, 0, 1, 0, 0, 0, 3, 1, 0, 1, 2, 3, 0, 2, 0, 3, 1, 0, 3, 0, 1, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 4, 0, 1, 2, 1, 0, 0, 0, 4, 0, 1, 0, 2, 0, 0, 0, 0, 1, 3, 0, 1, 0, 1, 0, 1, 0, 0, 1, 4, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 9, 1, 1, 1, 0, 0, 0, 0, 4, 0, 0, 0, 11, 3, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 4, 0, 2, 0, 3, 0, 1, 0, 3, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2}}, + }, + { + Name: "REQ envelope", + Message: []byte(`["REQ","sub1", {"until": 999999, "kinds":[1]}]`), + ExpectedEnvelope: &ReqEnvelope{SubscriptionID: "sub1", Filters: Filters{{Kinds: []int{1}, Until: ptr(Timestamp(999999))}}}, + }, + { + Name: "bigger REQ envelope", + Message: []byte(`["REQ","sub1z\\\"zzz", {"authors":["9b86ca5d2a9b4aa09870e710438c2fd2fcdeca12a18b6f17ab9ebcdbc43f1d4a","8eee10b2ce1162b040fdcfdadb4f888c64aaf87531dab28cc0c09fbdea1b663e","0deadebefb3c1a760f036952abf675076343dd8424efdeaa0f1d9803a359ed46"],"since":1740425099,"limit":2,"#x":["<","as"]}, {"kinds": [2345, 112], "#plic": ["a"], "#ploc": ["blblb", "wuwuw"]}]`), + ExpectedEnvelope: &ReqEnvelope{SubscriptionID: "sub1z\\\"zzz", Filters: Filters{{Authors: []string{"9b86ca5d2a9b4aa09870e710438c2fd2fcdeca12a18b6f17ab9ebcdbc43f1d4a", "8eee10b2ce1162b040fdcfdadb4f888c64aaf87531dab28cc0c09fbdea1b663e", "0deadebefb3c1a760f036952abf675076343dd8424efdeaa0f1d9803a359ed46"}, Since: ptr(Timestamp(1740425099)), Limit: 2, Tags: TagMap{"x": []string{"<", "as"}}}, {Kinds: []int{2345, 112}, Tags: TagMap{"plic": []string{"a"}, "ploc": []string{"blblb", "wuwuw"}}}}}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + var pj *simdjson.ParsedJson + envelope, err := ParseMessageSIMD(testCase.Message, pj) + + if testCase.ExpectedErrorSubstring == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), testCase.ExpectedErrorSubstring) + return + } + + if testCase.ExpectedEnvelope == nil { + require.Nil(t, envelope, "expected nil but got %v", envelope) + return + } + + require.NotNil(t, envelope, "expected non-nil envelope but got nil") + require.Equal(t, testCase.ExpectedEnvelope.Label(), envelope.Label()) + require.Equal(t, testCase.ExpectedEnvelope.String(), envelope.String()) + }) + } +} + func ptr[S any](s S) *S { return &s } diff --git a/event_simdjson.go b/event_simdjson.go new file mode 100644 index 0000000..33ff197 --- /dev/null +++ b/event_simdjson.go @@ -0,0 +1,84 @@ +package nostr + +import ( + "fmt" + "slices" + + "github.com/minio/simdjson-go" +) + +var ( + attrId = []byte("id") + attrPubkey = []byte("pubkey") + attrCreatedAt = []byte("created_at") + attrKind = []byte("kind") + attrContent = []byte("content") + attrTags = []byte("tags") + attrSig = []byte("sig") +) + +func (event *Event) UnmarshalSIMD(iter *simdjson.Iter) error { + obj, err := iter.Object(nil) + if err != nil { + return fmt.Errorf("unexpected at event: %w", err) + } + + for { + name, t, err := obj.NextElementBytes(iter) + if err != nil { + return err + } else if t == simdjson.TypeNone { + break + } + + switch { + case slices.Equal(name, attrId): + event.ID, err = iter.String() + case slices.Equal(name, attrPubkey): + event.PubKey, err = iter.String() + case slices.Equal(name, attrContent): + event.Content, err = iter.String() + case slices.Equal(name, attrSig): + event.Sig, err = iter.String() + case slices.Equal(name, attrCreatedAt): + var ts uint64 + ts, err = iter.Uint() + event.CreatedAt = Timestamp(ts) + case slices.Equal(name, attrKind): + var kind uint64 + kind, err = iter.Uint() + event.Kind = int(kind) + case slices.Equal(name, attrTags): + var arr *simdjson.Array + arr, err = iter.Array(nil) + if err != nil { + return err + } + event.Tags = make(Tags, 0, 10) + titer := arr.Iter() + var subArr *simdjson.Array + for { + if t := titer.Advance(); t == simdjson.TypeNone { + break + } + subArr, err = titer.Array(subArr) + if err != nil { + return err + } + tag, err := subArr.AsString() + if err != nil { + return err + } + event.Tags = append(event.Tags, tag) + } + default: + return fmt.Errorf("unexpected event field '%s'", name) + } + + if err != nil { + return err + } + } + + return nil +} diff --git a/filter_simdjson.go b/filter_simdjson.go new file mode 100644 index 0000000..e46170d --- /dev/null +++ b/filter_simdjson.go @@ -0,0 +1,107 @@ +package nostr + +import ( + "fmt" + "slices" + + "github.com/minio/simdjson-go" +) + +var ( + attrIds = []byte("ids") + attrAuthors = []byte("authors") + attrKinds = []byte("kinds") + attrLimit = []byte("limit") + attrSince = []byte("since") + attrUntil = []byte("until") + attrSearch = []byte("search") +) + +func (filter *Filter) UnmarshalSIMD(iter *simdjson.Iter) error { + obj, err := iter.Object(nil) + if err != nil { + return fmt.Errorf("unexpected at filter: %w", err) + } + + var arr *simdjson.Array + for { + name, t, err := obj.NextElementBytes(iter) + if err != nil { + return err + } else if t == simdjson.TypeNone { + break + } + + switch { + case slices.Equal(name, attrIds): + if arr, err = iter.Array(arr); err == nil { + filter.IDs, err = arr.AsString() + } + case slices.Equal(name, attrAuthors): + if arr, err = iter.Array(arr); err == nil { + filter.Authors, err = arr.AsString() + } + case slices.Equal(name, attrKinds): + if arr, err = iter.Array(arr); err == nil { + i := arr.Iter() + filter.Kinds = make([]int, 0, 6) + for { + t := i.Advance() + if t == simdjson.TypeNone { + break + } + if kind, err := i.Uint(); err != nil { + return err + } else { + filter.Kinds = append(filter.Kinds, int(kind)) + } + } + } + case slices.Equal(name, attrSearch): + filter.Search, err = iter.String() + case slices.Equal(name, attrSince): + var tsu uint64 + tsu, err = iter.Uint() + ts := Timestamp(tsu) + filter.Since = &ts + case slices.Equal(name, attrUntil): + var tsu uint64 + tsu, err = iter.Uint() + ts := Timestamp(tsu) + filter.Until = &ts + case slices.Equal(name, attrLimit): + var limit uint64 + limit, err = iter.Uint() + filter.Limit = int(limit) + if limit == 0 { + filter.LimitZero = true + } + default: + if len(name) > 1 && name[0] == '#' { + if filter.Tags == nil { + filter.Tags = make(TagMap, 1) + } + + arr, err := iter.Array(arr) + if err != nil { + return err + } + vals, err := arr.AsString() + if err != nil { + return err + } + + filter.Tags[string(name[1:])] = vals + continue + } + + return fmt.Errorf("unexpected filter field '%s'", name) + } + + if err != nil { + return err + } + } + + return nil +} diff --git a/go.mod b/go.mod index 10aadaf..48fc039 100644 --- a/go.mod +++ b/go.mod @@ -59,9 +59,11 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/simdjson-go v0.4.5 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect @@ -78,7 +80,7 @@ require ( github.com/valyala/fasthttp v1.51.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/sys v0.30.0 // indirect google.golang.org/protobuf v1.36.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect diff --git a/go.sum b/go.sum index 035a801..3ffea0b 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,10 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM= @@ -165,6 +169,8 @@ github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBW github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A= +github.com/minio/simdjson-go v0.4.5/go.mod h1:eoNz0DcLQRyEDeaPr4Ru6JpjlZPzbA0IodxVJk8lO8E= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -280,6 +286,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=