highperfocused/go-nostr
1
0
Fork 0
mirror of https://github.com/nbd-wtf/go-nostr.git synced 2025-05-03 07:10:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

binary encoding and some changes to nson benchmarks.

Browse Source
This commit is contained in:
fiatjaf 2023-11-02 15:28:01 -03:00
parent 1789d43d51
commit 4c72e16f3e
No known key found for this signature in database
GPG Key ID: BAD43C4BE5C1A3A1
7 changed files with 484 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
binary/README.md Normal file
View File

@ -0,0 +1,18 @@
# The simplest binary encoding for Nostr events
Some benchmarks:
goos: linux
goarch: amd64
pkg: github.com/nbd-wtf/go-nostr/binary
cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics
BenchmarkBinaryEncoding/easyjson.Marshal-4 24488 53274 ns/op 35191 B/op 102 allocs/op
BenchmarkBinaryEncoding/binary.Marshal-4 5066 218284 ns/op 1282116 B/op 88 allocs/op
BenchmarkBinaryEncoding/binary.MarshalBinary-4 5743 191603 ns/op 1277763 B/op 37 allocs/op
BenchmarkBinaryDecoding/easyjson.Unmarshal-4 32701 38647 ns/op 45832 B/op 124 allocs/op
BenchmarkBinaryDecoding/binary.Unmarshal-4 85705 14249 ns/op 25488 B/op 141 allocs/op
BenchmarkBinaryDecoding/binary.UnmarshalBinary-4 213438 5451 ns/op 16784 B/op 39 allocs/op
BenchmarkBinaryDecoding/easyjson.Unmarshal+sig-4 307 3971993 ns/op 131639 B/op 404 allocs/op
BenchmarkBinaryDecoding/binary.Unmarshal+sig-4 310 3924042 ns/op 111277 B/op 421 allocs/op
PASS
ok github.com/nbd-wtf/go-nostr/binary 11.444s

76
binary/binary.go Normal file
View File

@ -0,0 +1,76 @@
package binary
import (
"encoding/binary"
"fmt"
"github.com/nbd-wtf/go-nostr"
)
func UnmarshalBinary(data []byte, evt *Event) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("failed to decode leaner: %v", r)
}
}()
copy(evt.ID[:], data[0:32])
copy(evt.PubKey[:], data[32:64])
copy(evt.Sig[:], data[64:128])
evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
evt.Kind = binary.BigEndian.Uint16(data[132:134])
contentLength := int(binary.BigEndian.Uint16(data[134:136]))
evt.Content = string(data[136 : 136+contentLength])
curr := 136 + contentLength
ntags := int(data[curr])
evt.Tags = make(nostr.Tags, ntags)
for t := range evt.Tags {
curr = curr + 1
nItems := int(data[curr])
tag := make(nostr.Tag, nItems)
for i := range tag {
curr = curr + 1
itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
itemStart := curr + 2
itemEnd := itemStart + itemSize
item := string(data[itemStart:itemEnd])
tag[i] = item
curr = itemEnd
}
evt.Tags[t] = tag
}
return err
}
func MarshalBinary(evt *Event) []byte {
content := []byte(evt.Content)
buf := make([]byte, 32+32+64+4+2+2+len(content)+65536 /* blergh */)
copy(buf[0:32], evt.ID[:])
copy(buf[32:64], evt.PubKey[:])
copy(buf[64:128], evt.Sig[:])
binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
binary.BigEndian.PutUint16(buf[132:134], evt.Kind)
binary.BigEndian.PutUint16(buf[134:136], uint16(len(content)))
copy(buf[136:], content)
curr := 136 + len(content)
buf[curr] = uint8(len(evt.Tags))
for _, tag := range evt.Tags {
curr++
buf[curr] = uint8(len(tag))
for _, item := range tag {
curr++
itemb := []byte(item)
itemSize := len(itemb)
binary.BigEndian.PutUint16(buf[curr:curr+2], uint16(itemSize))
itemEnd := curr + 2 + itemSize
copy(buf[curr+2:itemEnd], itemb)
curr = itemEnd
}
}
buf = buf[0 : curr+1]
return buf
}

239
binary/binary_test.go Normal file
View File

File diff suppressed because one or more lines are too long

44
binary/event.go Normal file
View File

@ -0,0 +1,44 @@
package binary
import (
"encoding/hex"
"github.com/nbd-wtf/go-nostr"
)
type Event struct {
PubKey [32]byte
Sig [64]byte
ID [32]byte
Kind uint16
CreatedAt nostr.Timestamp
Content string
Tags nostr.Tags
}
func BinaryEvent(evt *nostr.Event) *Event {
bevt := Event{
Tags: evt.Tags,
Content: evt.Content,
Kind: uint16(evt.Kind),
CreatedAt: evt.CreatedAt,
}
hex.Decode(bevt.ID[:], []byte(evt.ID))
hex.Decode(bevt.PubKey[:], []byte(evt.PubKey))
hex.Decode(bevt.Sig[:], []byte(evt.Sig))
return &bevt
}
func (bevt *Event) ToNormalEvent() *nostr.Event {
return &nostr.Event{
Tags: bevt.Tags,
Content: bevt.Content,
Kind: int(bevt.Kind),
CreatedAt: bevt.CreatedAt,
ID: hex.EncodeToString(bevt.ID[:]),
PubKey: hex.EncodeToString(bevt.PubKey[:]),
Sig: hex.EncodeToString(bevt.Sig[:]),
}
}

79
binary/hybrid.go Normal file
View File

@ -0,0 +1,79 @@
package binary
import (
"encoding/binary"
"encoding/hex"
"fmt"
"github.com/nbd-wtf/go-nostr"
)
func Unmarshal(data []byte, evt *nostr.Event) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("failed to decode leaner: %v", r)
}
}()
evt.ID = hex.EncodeToString(data[0:32])
evt.PubKey = hex.EncodeToString(data[32:64])
evt.Sig = hex.EncodeToString(data[64:128])
evt.CreatedAt = nostr.Timestamp(binary.BigEndian.Uint32(data[128:132]))
evt.Kind = int(binary.BigEndian.Uint16(data[132:134]))
contentLength := int(binary.BigEndian.Uint16(data[134:136]))
evt.Content = string(data[136 : 136+contentLength])
curr := 136 + contentLength
ntags := int(data[curr])
evt.Tags = make(nostr.Tags, ntags)
for t := range evt.Tags {
curr = curr + 1
nItems := int(data[curr])
tag := make(nostr.Tag, nItems)
for i := range tag {
curr = curr + 1
itemSize := int(binary.BigEndian.Uint16(data[curr : curr+2]))
itemStart := curr + 2
itemEnd := itemStart + itemSize
item := string(data[itemStart:itemEnd])
tag[i] = item
curr = itemEnd
}
evt.Tags[t] = tag
}
return err
}
func Marshal(evt *nostr.Event) ([]byte, error) {
content := []byte(evt.Content)
buf := make([]byte, 32+32+64+4+2+2+len(content)+65536 /* blergh */)
hex.Decode(buf[0:32], []byte(evt.ID))
hex.Decode(buf[32:64], []byte(evt.PubKey))
hex.Decode(buf[64:128], []byte(evt.Sig))
binary.BigEndian.PutUint32(buf[128:132], uint32(evt.CreatedAt))
binary.BigEndian.PutUint16(buf[132:134], uint16(evt.Kind))
binary.BigEndian.PutUint16(buf[134:136], uint16(len(content)))
copy(buf[136:], content)
curr := 136 + len(content)
buf[curr] = uint8(len(evt.Tags))
for _, tag := range evt.Tags {
curr++
buf[curr] = uint8(len(tag))
for _, item := range tag {
curr++
itemb := []byte(item)
itemSize := len(itemb)
binary.BigEndian.PutUint16(buf[curr:curr+2], uint16(itemSize))
itemEnd := curr + 2 + itemSize
copy(buf[curr+2:itemEnd], itemb)
curr = itemEnd
}
}
buf = buf[0 : curr+1]
return buf, nil
}

20
nson/README.md
View File

@ -35,3 +35,23 @@ It's explained better in the NIP proposal linked above, but the idea is that we
a special JSON attribute called `"nson"`, and then the reader can just pull the strings directly from the JSON blob
without having to parse the full JSON syntax. Also for fields of static size we don't even need that. This is only
possible because Nostr events have a static and strict format.
## Update: comparison with `easyjson`
Another comparison, using the `easyjson` library that is already built in `go-nostr`, shows that the performance gains
are only of 2x (the standard library JSON encoding is just too slow).
```
goos: linux
goarch: amd64
pkg: github.com/nbd-wtf/go-nostr/nson
cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics
BenchmarkNSONEncoding/easyjson.Marshal-4 21511 54849 ns/op
BenchmarkNSONEncoding/nson.Marshal-4 4810 297624 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal-4 25196 46652 ns/op
BenchmarkNSONDecoding/nson.Unmarshal-4 61117 22933 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal+sig-4 303 4110988 ns/op
BenchmarkNSONDecoding/nson.Unmarshal+sig-4 296 3881435 ns/op
PASS
ok github.com/nbd-wtf/go-nostr/nson
```

37
nson/nson_test.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"testing"
"github.com/mailru/easyjson"
"github.com/nbd-wtf/go-nostr"
)
@ -153,10 +154,10 @@ func BenchmarkNSONEncoding(b *testing.B) {
events[i] = evt
}
b.Run("json.Marshal", func(b *testing.B) {
b.Run("easyjson.Marshal", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, evt := range events {
json.Marshal(evt)
easyjson.Marshal(evt)
}
}
})
@ -168,32 +169,22 @@ func BenchmarkNSONEncoding(b *testing.B) {
}
}
})
b.Run("nson.Marshal to bytes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, evt := range events {
MarshalBytes(evt)
}
}
})
}
func BenchmarkNSONDecoding(b *testing.B) {
events := make([]string, len(normalEvents))
eventsB := make([][]byte, len(normalEvents))
for i, jevt := range normalEvents {
evt := &nostr.Event{}
json.Unmarshal([]byte(jevt), evt)
nevt, _ := Marshal(evt)
events[i] = nevt
eventsB[i] = []byte(nevt)
}
b.Run("json.Unmarshal", func(b *testing.B) {
b.Run("easyjson.Unmarshal", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, nevt := range events {
evt := &nostr.Event{}
err := json.Unmarshal([]byte(nevt), evt)
err := easyjson.Unmarshal([]byte(nevt), evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
@ -213,23 +204,11 @@ func BenchmarkNSONDecoding(b *testing.B) {
}
})
b.Run("nson.Unmarshal from bytes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, nevt := range eventsB {
evt := &nostr.Event{}
err := UnmarshalBytes(nevt, evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
}
}
})
b.Run("json.Unmarshal + sig verification", func(b *testing.B) {
b.Run("easyjson.Unmarshal+sig", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, nevt := range events {
evt := &nostr.Event{}
err := json.Unmarshal([]byte(nevt), evt)
err := easyjson.Unmarshal([]byte(nevt), evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
@ -238,7 +217,7 @@ func BenchmarkNSONDecoding(b *testing.B) {
}
})
b.Run("nson.Unmarshal + sig verification", func(b *testing.B) {
b.Run("nson.Unmarshal+sig", func(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, nevt := range events {
evt := &nostr.Event{}