From 2016f11dd1dac868230e4a45e26fdf4b9dc7eac3 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Sun, 8 Sep 2024 11:53:57 -0300 Subject: [PATCH] remove nson. it isn't worth keeping this here since it isn't being used and is unlikely to ever be. --- nson/README.md | 57 ----------- nson/nson.go | 241 ---------------------------------------------- nson/nson_test.go | 231 -------------------------------------------- 3 files changed, 529 deletions(-) delete mode 100644 nson/README.md delete mode 100644 nson/nson.go delete mode 100644 nson/nson_test.go diff --git a/nson/README.md b/nson/README.md deleted file mode 100644 index 96354e7..0000000 --- a/nson/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# NSON - -A crazy way to encode Nostr events into valid JSON that is also much faster to decode as long as they are actually -encoded using the strict NSON encoding and the decoder is prepared to read it using a NSON decoder. - -See https://github.com/nostr-protocol/nips/pull/515. - -Some benchmarks: - -``` -goos: linux -goarch: amd64 -pkg: github.com/nbd-wtf/go-nostr/nson -cpu: 13th Gen Intel(R) Core(TM) i7-13620H -BenchmarkNSONEncoding/easyjson.Marshal-16 18795 61397 ns/op -BenchmarkNSONEncoding/nson.Marshal-16 5985 205112 ns/op -BenchmarkNSONDecoding/easyjson.Unmarshal-16 14928 83890 ns/op -BenchmarkNSONDecoding/nson.Unmarshal-16 24982 50527 ns/op -BenchmarkNSONDecoding/easyjson.Unmarshal+sig-16 196 5898287 ns/op -BenchmarkNSONDecoding/nson.Unmarshal+sig-16 205 5802747 ns/op -PASS -ok github.com/nbd-wtf/go-nostr/nson 10.227s -``` - -It takes a little while more to encode (although it's probably possible to optimize that), but decodes at 10x the -speed of normal JSON. - -The performance gain is real, but negligible once you add hash validation and signature verification, so it should -be used wisely, mostly for situations in which the reader wouldn't care about the signature, e.g. reading from a -local database. - -## How it works - -It's explained better in the NIP proposal linked above, but the idea is that we encode field offset and sizes into -a special JSON attribute called `"nson"`, and then the reader can just pull the strings directly from the JSON blob -without having to parse the full JSON syntax. Also for fields of static size we don't even need that. This is only -possible because Nostr events have a static and strict format. - -## Update: comparison with `easyjson` - -Another comparison, using the `easyjson` library that is already built in `go-nostr`, shows that the performance gains -are only of 2x (the standard library JSON encoding is just too slow). - -``` -goos: linux -goarch: amd64 -pkg: github.com/nbd-wtf/go-nostr/nson -cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics -BenchmarkNSONEncoding/easyjson.Marshal-4 21511 54849 ns/op -BenchmarkNSONEncoding/nson.Marshal-4 4810 297624 ns/op -BenchmarkNSONDecoding/easyjson.Unmarshal-4 25196 46652 ns/op -BenchmarkNSONDecoding/nson.Unmarshal-4 61117 22933 ns/op -BenchmarkNSONDecoding/easyjson.Unmarshal+sig-4 303 4110988 ns/op -BenchmarkNSONDecoding/nson.Unmarshal+sig-4 296 3881435 ns/op -PASS -ok github.com/nbd-wtf/go-nostr/nson -``` diff --git a/nson/nson.go b/nson/nson.go deleted file mode 100644 index daa1f93..0000000 --- a/nson/nson.go +++ /dev/null @@ -1,241 +0,0 @@ -package nson - -import ( - "encoding/binary" - "encoding/hex" - "fmt" - "strconv" - "strings" - "unsafe" - - "github.com/nbd-wtf/go-nostr" -) - -/* - nson size - kind chars - content chars - number of tags (let's say it's two) - number of items on the first tag (let's say it's three) - number of chars on the first item - number of chars on the second item - number of chars on the third item - number of items on the second tag (let's say it's two) - number of chars on the first item - number of chars on the second item - "nson":"xxkkccccttnn111122223333nn11112222" -*/ - -const ( - ID_START = 7 - ID_END = 7 + 64 - PUBKEY_START = 83 - PUBKEY_END = 83 + 64 - SIG_START = 156 - SIG_END = 156 + 128 - CREATED_AT_START = 299 - CREATED_AT_END = 299 + 10 - - NSON_STRING_START = 318 // the actual json string for the "nson" field - NSON_VALUES_START = 318 + 2 // skipping the first byte which delimits the nson size - - NSON_MARKER_START = 309 // this is used just to determine if an event is nson or not - NSON_MARKER_END = 317 // it's just the `,"nson":` (including ,": garbage to reduce false positives) part -) - -var ErrNotNSON = fmt.Errorf("not nson") - -func UnmarshalBytes(data []byte, evt *nostr.Event) (err error) { - return Unmarshal(unsafe.String(unsafe.SliceData(data), len(data)), evt) -} - -// Unmarshal turns a NSON string into a nostr.Event struct. -func Unmarshal(data string, evt *nostr.Event) (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("failed to decode nson: %v", r) - } - }() - - // check if it's nson - if data[NSON_MARKER_START:NSON_MARKER_END] != ",\"nson\":" { - return ErrNotNSON - } - - // nson values - nsonSize, nsonDescriptors := parseDescriptors(data) - - // static fields - evt.ID = data[ID_START:ID_END] - evt.PubKey = data[PUBKEY_START:PUBKEY_END] - evt.Sig = data[SIG_START:SIG_END] - ts, _ := strconv.ParseUint(data[CREATED_AT_START:CREATED_AT_END], 10, 64) - evt.CreatedAt = nostr.Timestamp(ts) - - // dynamic fields - // kind - kindChars := int(nsonDescriptors[0]) - kindStart := NSON_VALUES_START + nsonSize + 9 // len(`","kind":`) - evt.Kind, _ = strconv.Atoi(data[kindStart : kindStart+kindChars]) - - // content - contentChars := int(binary.BigEndian.Uint16(nsonDescriptors[1:3])) - contentStart := kindStart + kindChars + 12 // len(`,"content":"`) - evt.Content, _ = strconv.Unquote(data[contentStart-1 : contentStart+contentChars+1]) - - // tags - nTags := int(nsonDescriptors[3]) - evt.Tags = make(nostr.Tags, nTags) - tagsStart := contentStart + contentChars + 9 // len(`","tags":`) - - nsonIndex := 3 - tagsIndex := tagsStart - for t := 0; t < nTags; t++ { - nsonIndex++ - tagsIndex += 1 // len(`[`) or len(`,`) - nItems := int(nsonDescriptors[nsonIndex]) - tag := make(nostr.Tag, nItems) - for n := 0; n < nItems; n++ { - nsonIndex++ - itemStart := tagsIndex + 2 // len(`["`) or len(`,"`) - itemChars := int(binary.BigEndian.Uint16(nsonDescriptors[nsonIndex:])) - nsonIndex++ - tag[n], _ = strconv.Unquote(data[itemStart-1 : itemStart+itemChars+1]) - tagsIndex = itemStart + itemChars + 1 // len(`"`) - } - tagsIndex += 1 // len(`]`) - evt.Tags[t] = tag - } - - return err -} - -func MarshalBytes(evt *nostr.Event) ([]byte, error) { - v, err := Marshal(evt) - return unsafe.Slice(unsafe.StringData(v), len(v)), err -} - -func Marshal(evt *nostr.Event) (string, error) { - // start building the nson descriptors (without the first byte that represents the nson size) - nsonBuf := make([]byte, 256) - - // build the tags - nTags := len(evt.Tags) - nsonBuf[3] = uint8(nTags) - nsonIndex := 3 // start here - - tagBuilder := strings.Builder{} - tagBuilder.Grow(1000) // a guess - tagBuilder.WriteString(`[`) - for t, tag := range evt.Tags { - nItems := len(tag) - nsonIndex++ - nsonBuf[nsonIndex] = uint8(nItems) - - tagBuilder.WriteString(`[`) - for i, item := range tag { - v := strconv.Quote(item) - nsonIndex++ - binary.BigEndian.PutUint16(nsonBuf[nsonIndex:], uint16(len(v)-2)) - nsonIndex++ - tagBuilder.WriteString(v) - if nItems > i+1 { - tagBuilder.WriteString(`,`) - } - } - tagBuilder.WriteString(`]`) - if nTags > t+1 { - tagBuilder.WriteString(`,`) - } - } - tagBuilder.WriteString(`]}`) - nsonBuf = nsonBuf[0 : nsonIndex+1] - - kind := strconv.Itoa(evt.Kind) - kindChars := len(kind) - nsonBuf[0] = uint8(kindChars) - - content := strconv.Quote(evt.Content) - contentChars := len(content) - 2 - binary.BigEndian.PutUint16(nsonBuf[1:3], uint16(contentChars)) - - // actually build the json - base := strings.Builder{} - base.Grow(NSON_VALUES_START + // everything up to "nson": - 2 + len(nsonBuf)*2 + // nson - 9 + kindChars + // kind and its label - 12 + contentChars + // content and its label - 9 + tagBuilder.Len() + // tags and its label - 2, // the end - ) - base.WriteString(`{"id":"` + evt.ID + `","pubkey":"` + evt.PubKey + `","sig":"` + evt.Sig + - `","created_at":` + strconv.FormatInt(int64(evt.CreatedAt), 10) + `,"nson":"`) - - nsonSizeBytes := len(nsonBuf) - if nsonSizeBytes > 255 { - return "", fmt.Errorf("can't encode to nson, there are too many tags or tag items") - } - base.WriteString(hex.EncodeToString([]byte{uint8(nsonSizeBytes)})) // nson size (bytes) - - base.WriteString(hex.EncodeToString(nsonBuf)) // nson descriptors - base.WriteString(`","kind":` + kind + `,"content":` + content + `,"tags":`) - base.WriteString(tagBuilder.String() /* includes the end */) - - return base.String(), nil -} - -func parseDescriptors(data string) (int, []byte) { - nsonSizeBytes, _ := hex.DecodeString(data[NSON_STRING_START:NSON_VALUES_START]) - size := int(nsonSizeBytes[0]) * 2 // number of bytes is given, we x2 because the string is in hex - values, _ := hex.DecodeString(data[NSON_VALUES_START : NSON_VALUES_START+size]) - return size, values -} - -// A nson.Event is basically a wrapper over the string that makes it easy to get each event property (except tags). -type Event struct { - data string - - descriptorsSize int - descriptors []byte -} - -func New(nsonText string) Event { - return Event{data: nsonText} -} - -func (ne *Event) parseDescriptors() { - if ne.descriptors == nil { - ne.descriptorsSize, ne.descriptors = parseDescriptors(ne.data) - } -} - -func (ne Event) GetID() string { return ne.data[ID_START:ID_END] } -func (ne Event) GetPubkey() string { return ne.data[PUBKEY_START:PUBKEY_END] } -func (ne Event) GetSig() string { return ne.data[SIG_START:SIG_END] } -func (ne Event) GetCreatedAt() nostr.Timestamp { - ts, _ := strconv.ParseUint(ne.data[CREATED_AT_START:CREATED_AT_END], 10, 64) - return nostr.Timestamp(ts) -} - -func (ne *Event) GetKind() int { - ne.parseDescriptors() - - kindChars := int(ne.descriptors[0]) - kindStart := NSON_VALUES_START + ne.descriptorsSize + 9 // len(`","kind":`) - kind, _ := strconv.Atoi(ne.data[kindStart : kindStart+kindChars]) - - return kind -} - -func (ne *Event) GetContent() string { - ne.parseDescriptors() - - kindChars := int(ne.descriptors[0]) - kindStart := NSON_VALUES_START + ne.descriptorsSize + 9 // len(`","kind":`) - - contentChars := int(binary.BigEndian.Uint16(ne.descriptors[1:3])) - contentStart := kindStart + kindChars + 12 // len(`,"content":"`) - content, _ := strconv.Unquote(`"` + ne.data[contentStart:contentStart+contentChars] + `"`) - - return content -} diff --git a/nson/nson_test.go b/nson/nson_test.go deleted file mode 100644 index b224f3a..0000000 --- a/nson/nson_test.go +++ /dev/null @@ -1,231 +0,0 @@ -package nson - -import ( - "encoding/json" - "testing" - - "github.com/mailru/easyjson" - "github.com/nbd-wtf/go-nostr" - "github.com/nbd-wtf/go-nostr/test_common" -) - -func TestBasicNsonParse(t *testing.T) { - for _, jevt := range nsonTestEvents { - evt := &nostr.Event{} - if err := Unmarshal(jevt, evt); err != nil { - t.Fatalf("error unmarshalling nson: %s", err) - } - checkParsedCorrectly(t, evt, jevt) - } -} - -func TestNsonPartialGet(t *testing.T) { - for _, jevt := range nsonTestEvents { - evt := &nostr.Event{} - if err := Unmarshal(jevt, evt); err != nil { - t.Fatalf("error unmarshalling nson: %s", err) - } - - wrapper := New(jevt) - - if id := wrapper.GetID(); id != evt.ID { - t.Fatalf("partial id wrong. got %v, expected %v", id, evt.ID) - } - if pubkey := wrapper.GetPubkey(); pubkey != evt.PubKey { - t.Fatalf("partial pubkey wrong. got %v, expected %v", pubkey, evt.PubKey) - } - if sig := wrapper.GetSig(); sig != evt.Sig { - t.Fatalf("partial sig wrong. got %v, expected %v", sig, evt.Sig) - } - if createdAt := wrapper.GetCreatedAt(); createdAt != evt.CreatedAt { - t.Fatalf("partial created_at wrong. got %v, expected %v", createdAt, evt.CreatedAt) - } - if kind := wrapper.GetKind(); kind != evt.Kind { - t.Fatalf("partial kind wrong. got %v, expected %v", kind, evt.Kind) - } - if content := wrapper.GetContent(); content != evt.Content { - t.Fatalf("partial content wrong. got %v, expected %v", content, evt.Content) - } - } -} - -func TestNsonEncode(t *testing.T) { - for _, jevt := range test_common.NormalEvents { - pevt := &nostr.Event{} - if err := json.Unmarshal([]byte(jevt), pevt); err != nil { - t.Fatalf("failed to decode normal json: %s", err) - } - nevt, err := Marshal(pevt) - if err != nil { - t.Fatalf("failed to encode nson: %s", err) - } - - evt := &nostr.Event{} - if err := Unmarshal(nevt, evt); err != nil { - t.Fatalf("error unmarshalling nson: %s", err) - } - checkParsedCorrectly(t, pevt, jevt) - checkParsedCorrectly(t, evt, jevt) - } -} - -func checkParsedCorrectly(t *testing.T, evt *nostr.Event, jevt string) (isBad bool) { - var canonical nostr.Event - err := json.Unmarshal([]byte(jevt), &canonical) - if err != nil { - t.Fatalf("error unmarshalling normal json: %s", err) - } - - if evt.ID != canonical.ID { - t.Fatalf("id is wrong: %s != %s", evt.ID, canonical.ID) - isBad = true - } - if evt.PubKey != canonical.PubKey { - t.Fatalf("pubkey is wrong: %s != %s", evt.PubKey, canonical.PubKey) - isBad = true - } - if evt.Sig != canonical.Sig { - t.Fatalf("sig is wrong: %s != %s", evt.Sig, canonical.Sig) - isBad = true - } - if evt.Content != canonical.Content { - t.Fatalf("content is wrong: %s != %s", evt.Content, canonical.Content) - isBad = true - } - if evt.Kind != canonical.Kind { - t.Fatalf("kind is wrong: %d != %d", evt.Kind, canonical.Kind) - isBad = true - } - if evt.CreatedAt != canonical.CreatedAt { - t.Fatalf("created_at is wrong: %v != %v", evt.CreatedAt, canonical.CreatedAt) - isBad = true - } - if len(evt.Tags) != len(canonical.Tags) { - t.Fatalf("tag number is wrong: %v != %v", len(evt.Tags), len(canonical.Tags)) - isBad = true - } - for i := range evt.Tags { - if len(evt.Tags[i]) != len(canonical.Tags[i]) { - t.Fatalf("tag[%d] length is wrong: `%v` != `%v`", i, len(evt.Tags[i]), len(canonical.Tags[i])) - isBad = true - } - for j := range evt.Tags[i] { - if evt.Tags[i][j] != canonical.Tags[i][j] { - t.Fatalf("tag[%d][%d] is wrong: `%s` != `%s`", i, j, evt.Tags[i][j], canonical.Tags[i][j]) - isBad = true - } - } - } - - return isBad -} - -var nsonTestEvents = []string{ - `{"id":"192eaf31bd20476bbe9265a3667cfef6410dfd563c02a64cb15d6fa8efec0ed6","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"5b9051596a5ba0619fd5fd7d2766b8aeb0cc398f1d1a0804f4b4ed884482025b3d4888e4c892f2fc437415bfc121482a990fad30f5cd9e333e55364052f99bbc","created_at":1688505641,"nson":"0401000500","kind":1,"content":"hello","tags":[]}`, - `{"id":"921ada34fe581b506975c641f2d1a3fb4f491f1d30c2490452e8524776895ebf","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"1f15a39e93a13f14f783eb127b2977e5dc5d207070dfa280fe45879b6b142ec1943ec921ab4268e69a43704d5641b45d18bf3789037c4842e062cd347a8a7ee1","created_at":1688553190,"nson":"12010006020200060005040005004000120006","kind":1,"content":"maçã","tags":[["entity","fruit"],["owner","79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","wss://リレー.jp","person"]]}`, - `{"id":"06212bae3cfc917d4b1239a3bad4fdba1e0e1ff09fbd2ee7b6da15d5fd859f58","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"47199a3a4184528d2c6cbb94df03b9793ea65b4578154ff5edce794d03ee2408cd3ca699b39cc11e791656e98b510194330d3dc215389c5648eddf33b8362444","created_at":1688572619,"nson":"0401000400","kind":1,"content":"x\ny","tags":[]}`, - `{"id":"ec9345e2af4225aada296964fa6025a1666dcac8dba154f5591a81f7dee1f84a","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"49f4b9edd7eff9e127b70077daff9a66da8c1ad974e5e6f47c094e8cc0c553071ff61c07b69d3db80c25f36248237ba6021038f5eb6b569ce79e3b024e8e358d","created_at":1688572819,"nson":"0401000400","kind":1,"content":"x\ty","tags":[]}`, -} - -var ( - EncodedEventEasyJson []byte - EncodedEventNSON string - DecodedEvent *nostr.Event -) - -func BenchmarkNSONEncoding(b *testing.B) { - events := make([]*nostr.Event, len(test_common.NormalEvents)) - for i, jevt := range test_common.NormalEvents { - evt := &nostr.Event{} - json.Unmarshal([]byte(jevt), evt) - events[i] = evt - } - - b.ResetTimer() - - b.Run("easyjson.Marshal", func(b *testing.B) { - var encodedEvent []byte - for i := 0; i < b.N; i++ { - for _, evt := range events { - encodedEvent, _ = easyjson.Marshal(evt) - } - } - EncodedEventEasyJson = encodedEvent - }) - - b.Run("nson.Marshal", func(b *testing.B) { - var encodedEvent string - for i := 0; i < b.N; i++ { - for _, evt := range events { - encodedEvent, _ = Marshal(evt) - } - } - EncodedEventNSON = encodedEvent - }) -} - -func BenchmarkNSONDecoding(b *testing.B) { - events := make([]string, len(test_common.NormalEvents)) - for i, jevt := range test_common.NormalEvents { - evt := &nostr.Event{} - json.Unmarshal([]byte(jevt), evt) - nevt, _ := Marshal(evt) - events[i] = nevt - } - - b.ResetTimer() - - b.Run("easyjson.Unmarshal", func(b *testing.B) { - evt := &nostr.Event{} - for i := 0; i < b.N; i++ { - for _, nevt := range events { - err := easyjson.Unmarshal([]byte(nevt), evt) - if err != nil { - b.Fatalf("failed to unmarshal: %s", err) - } - } - } - DecodedEvent = evt - }) - - b.Run("nson.Unmarshal", func(b *testing.B) { - evt := &nostr.Event{} - for i := 0; i < b.N; i++ { - for _, nevt := range events { - err := Unmarshal(nevt, evt) - if err != nil { - b.Fatalf("failed to unmarshal: %s", err) - } - } - } - DecodedEvent = evt - }) - - b.Run("easyjson.Unmarshal+sig", func(b *testing.B) { - evt := &nostr.Event{} - for i := 0; i < b.N; i++ { - for _, nevt := range events { - err := easyjson.Unmarshal([]byte(nevt), evt) - if err != nil { - b.Fatalf("failed to unmarshal: %s", err) - } - evt.CheckSignature() - } - } - DecodedEvent = evt - }) - - b.Run("nson.Unmarshal+sig", func(b *testing.B) { - evt := &nostr.Event{} - for i := 0; i < b.N; i++ { - for _, nevt := range events { - err := Unmarshal(nevt, evt) - if err != nil { - b.Fatalf("failed to unmarshal: %s", err) - } - evt.CheckSignature() - } - } - DecodedEvent = evt - }) -}