highperfocused/go-nostr
1
0
Fork 0
mirror of https://github.com/nbd-wtf/go-nostr.git synced 2025-05-05 16:20:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

remove nson.

Browse Source 
it isn't worth keeping this here since it isn't being used and is unlikely to ever be.
This commit is contained in:
fiatjaf 2024-09-08 11:53:57 -03:00
parent b7c79c9c9c
commit 2016f11dd1
3 changed files with 0 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
nson/README.md
View File

@ -1,57 +0,0 @@
# NSON
A crazy way to encode Nostr events into valid JSON that is also much faster to decode as long as they are actually
encoded using the strict NSON encoding and the decoder is prepared to read it using a NSON decoder.
See https://github.com/nostr-protocol/nips/pull/515.
Some benchmarks:
```
goos: linux
goarch: amd64
pkg: github.com/nbd-wtf/go-nostr/nson
cpu: 13th Gen Intel(R) Core(TM) i7-13620H
BenchmarkNSONEncoding/easyjson.Marshal-16 18795 61397 ns/op
BenchmarkNSONEncoding/nson.Marshal-16 5985 205112 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal-16 14928 83890 ns/op
BenchmarkNSONDecoding/nson.Unmarshal-16 24982 50527 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal+sig-16 196 5898287 ns/op
BenchmarkNSONDecoding/nson.Unmarshal+sig-16 205 5802747 ns/op
PASS
ok github.com/nbd-wtf/go-nostr/nson 10.227s
```
It takes a little while more to encode (although it's probably possible to optimize that), but decodes at 10x the
speed of normal JSON.
The performance gain is real, but negligible once you add hash validation and signature verification, so it should
be used wisely, mostly for situations in which the reader wouldn't care about the signature, e.g. reading from a
local database.
## How it works
It's explained better in the NIP proposal linked above, but the idea is that we encode field offset and sizes into
a special JSON attribute called `"nson"`, and then the reader can just pull the strings directly from the JSON blob
without having to parse the full JSON syntax. Also for fields of static size we don't even need that. This is only
possible because Nostr events have a static and strict format.
## Update: comparison with `easyjson`
Another comparison, using the `easyjson` library that is already built in `go-nostr`, shows that the performance gains
are only of 2x (the standard library JSON encoding is just too slow).
```
goos: linux
goarch: amd64
pkg: github.com/nbd-wtf/go-nostr/nson
cpu: AMD Ryzen 3 3200G with Radeon Vega Graphics
BenchmarkNSONEncoding/easyjson.Marshal-4 21511 54849 ns/op
BenchmarkNSONEncoding/nson.Marshal-4 4810 297624 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal-4 25196 46652 ns/op
BenchmarkNSONDecoding/nson.Unmarshal-4 61117 22933 ns/op
BenchmarkNSONDecoding/easyjson.Unmarshal+sig-4 303 4110988 ns/op
BenchmarkNSONDecoding/nson.Unmarshal+sig-4 296 3881435 ns/op
PASS
ok github.com/nbd-wtf/go-nostr/nson
```

241
nson/nson.go
View File

@ -1,241 +0,0 @@
package nson
import (
"encoding/binary"
"encoding/hex"
"fmt"
"strconv"
"strings"
"unsafe"
"github.com/nbd-wtf/go-nostr"
)
/*
nson size
kind chars
content chars
number of tags (let's say it's two)
number of items on the first tag (let's say it's three)
number of chars on the first item
number of chars on the second item
number of chars on the third item
number of items on the second tag (let's say it's two)
number of chars on the first item
number of chars on the second item
"nson":"xxkkccccttnn111122223333nn11112222"
*/
const (
ID_START = 7
ID_END = 7 + 64
PUBKEY_START = 83
PUBKEY_END = 83 + 64
SIG_START = 156
SIG_END = 156 + 128
CREATED_AT_START = 299
CREATED_AT_END = 299 + 10
NSON_STRING_START = 318 // the actual json string for the "nson" field
NSON_VALUES_START = 318 + 2 // skipping the first byte which delimits the nson size
NSON_MARKER_START = 309 // this is used just to determine if an event is nson or not
NSON_MARKER_END = 317 // it's just the `,"nson":` (including ,": garbage to reduce false positives) part
)
var ErrNotNSON = fmt.Errorf("not nson")
func UnmarshalBytes(data []byte, evt *nostr.Event) (err error) {
return Unmarshal(unsafe.String(unsafe.SliceData(data), len(data)), evt)
}
// Unmarshal turns a NSON string into a nostr.Event struct.
func Unmarshal(data string, evt *nostr.Event) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("failed to decode nson: %v", r)
}
}()
// check if it's nson
if data[NSON_MARKER_START:NSON_MARKER_END] != ",\"nson\":" {
return ErrNotNSON
}
// nson values
nsonSize, nsonDescriptors := parseDescriptors(data)
// static fields
evt.ID = data[ID_START:ID_END]
evt.PubKey = data[PUBKEY_START:PUBKEY_END]
evt.Sig = data[SIG_START:SIG_END]
ts, _ := strconv.ParseUint(data[CREATED_AT_START:CREATED_AT_END], 10, 64)
evt.CreatedAt = nostr.Timestamp(ts)
// dynamic fields
// kind
kindChars := int(nsonDescriptors[0])
kindStart := NSON_VALUES_START + nsonSize + 9 // len(`","kind":`)
evt.Kind, _ = strconv.Atoi(data[kindStart : kindStart+kindChars])
// content
contentChars := int(binary.BigEndian.Uint16(nsonDescriptors[1:3]))
contentStart := kindStart + kindChars + 12 // len(`,"content":"`)
evt.Content, _ = strconv.Unquote(data[contentStart-1 : contentStart+contentChars+1])
// tags
nTags := int(nsonDescriptors[3])
evt.Tags = make(nostr.Tags, nTags)
tagsStart := contentStart + contentChars + 9 // len(`","tags":`)
nsonIndex := 3
tagsIndex := tagsStart
for t := 0; t < nTags; t++ {
nsonIndex++
tagsIndex += 1 // len(`[`) or len(`,`)
nItems := int(nsonDescriptors[nsonIndex])
tag := make(nostr.Tag, nItems)
for n := 0; n < nItems; n++ {
nsonIndex++
itemStart := tagsIndex + 2 // len(`["`) or len(`,"`)
itemChars := int(binary.BigEndian.Uint16(nsonDescriptors[nsonIndex:]))
nsonIndex++
tag[n], _ = strconv.Unquote(data[itemStart-1 : itemStart+itemChars+1])
tagsIndex = itemStart + itemChars + 1 // len(`"`)
}
tagsIndex += 1 // len(`]`)
evt.Tags[t] = tag
}
return err
}
func MarshalBytes(evt *nostr.Event) ([]byte, error) {
v, err := Marshal(evt)
return unsafe.Slice(unsafe.StringData(v), len(v)), err
}
func Marshal(evt *nostr.Event) (string, error) {
// start building the nson descriptors (without the first byte that represents the nson size)
nsonBuf := make([]byte, 256)
// build the tags
nTags := len(evt.Tags)
nsonBuf[3] = uint8(nTags)
nsonIndex := 3 // start here
tagBuilder := strings.Builder{}
tagBuilder.Grow(1000) // a guess
tagBuilder.WriteString(`[`)
for t, tag := range evt.Tags {
nItems := len(tag)
nsonIndex++
nsonBuf[nsonIndex] = uint8(nItems)
tagBuilder.WriteString(`[`)
for i, item := range tag {
v := strconv.Quote(item)
nsonIndex++
binary.BigEndian.PutUint16(nsonBuf[nsonIndex:], uint16(len(v)-2))
nsonIndex++
tagBuilder.WriteString(v)
if nItems > i+1 {
tagBuilder.WriteString(`,`)
}
}
tagBuilder.WriteString(`]`)
if nTags > t+1 {
tagBuilder.WriteString(`,`)
}
}
tagBuilder.WriteString(`]}`)
nsonBuf = nsonBuf[0 : nsonIndex+1]
kind := strconv.Itoa(evt.Kind)
kindChars := len(kind)
nsonBuf[0] = uint8(kindChars)
content := strconv.Quote(evt.Content)
contentChars := len(content) - 2
binary.BigEndian.PutUint16(nsonBuf[1:3], uint16(contentChars))
// actually build the json
base := strings.Builder{}
base.Grow(NSON_VALUES_START + // everything up to "nson":
2 + len(nsonBuf)*2 + // nson
9 + kindChars + // kind and its label
12 + contentChars + // content and its label
9 + tagBuilder.Len() + // tags and its label
2, // the end
)
base.WriteString(`{"id":"` + evt.ID + `","pubkey":"` + evt.PubKey + `","sig":"` + evt.Sig +
`","created_at":` + strconv.FormatInt(int64(evt.CreatedAt), 10) + `,"nson":"`)
nsonSizeBytes := len(nsonBuf)
if nsonSizeBytes > 255 {
return "", fmt.Errorf("can't encode to nson, there are too many tags or tag items")
}
base.WriteString(hex.EncodeToString([]byte{uint8(nsonSizeBytes)})) // nson size (bytes)
base.WriteString(hex.EncodeToString(nsonBuf)) // nson descriptors
base.WriteString(`","kind":` + kind + `,"content":` + content + `,"tags":`)
base.WriteString(tagBuilder.String() /* includes the end */)
return base.String(), nil
}
func parseDescriptors(data string) (int, []byte) {
nsonSizeBytes, _ := hex.DecodeString(data[NSON_STRING_START:NSON_VALUES_START])
size := int(nsonSizeBytes[0]) * 2 // number of bytes is given, we x2 because the string is in hex
values, _ := hex.DecodeString(data[NSON_VALUES_START : NSON_VALUES_START+size])
return size, values
}
// A nson.Event is basically a wrapper over the string that makes it easy to get each event property (except tags).
type Event struct {
data string
descriptorsSize int
descriptors []byte
}
func New(nsonText string) Event {
return Event{data: nsonText}
}
func (ne *Event) parseDescriptors() {
if ne.descriptors == nil {
ne.descriptorsSize, ne.descriptors = parseDescriptors(ne.data)
}
}
func (ne Event) GetID() string { return ne.data[ID_START:ID_END] }
func (ne Event) GetPubkey() string { return ne.data[PUBKEY_START:PUBKEY_END] }
func (ne Event) GetSig() string { return ne.data[SIG_START:SIG_END] }
func (ne Event) GetCreatedAt() nostr.Timestamp {
ts, _ := strconv.ParseUint(ne.data[CREATED_AT_START:CREATED_AT_END], 10, 64)
return nostr.Timestamp(ts)
}
func (ne *Event) GetKind() int {
ne.parseDescriptors()
kindChars := int(ne.descriptors[0])
kindStart := NSON_VALUES_START + ne.descriptorsSize + 9 // len(`","kind":`)
kind, _ := strconv.Atoi(ne.data[kindStart : kindStart+kindChars])
return kind
}
func (ne *Event) GetContent() string {
ne.parseDescriptors()
kindChars := int(ne.descriptors[0])
kindStart := NSON_VALUES_START + ne.descriptorsSize + 9 // len(`","kind":`)
contentChars := int(binary.BigEndian.Uint16(ne.descriptors[1:3]))
contentStart := kindStart + kindChars + 12 // len(`,"content":"`)
content, _ := strconv.Unquote(`"` + ne.data[contentStart:contentStart+contentChars] + `"`)
return content
}

231
nson/nson_test.go
View File

@ -1,231 +0,0 @@
package nson
import (
"encoding/json"
"testing"
"github.com/mailru/easyjson"
"github.com/nbd-wtf/go-nostr"
"github.com/nbd-wtf/go-nostr/test_common"
)
func TestBasicNsonParse(t *testing.T) {
for _, jevt := range nsonTestEvents {
evt := &nostr.Event{}
if err := Unmarshal(jevt, evt); err != nil {
t.Fatalf("error unmarshalling nson: %s", err)
}
checkParsedCorrectly(t, evt, jevt)
}
}
func TestNsonPartialGet(t *testing.T) {
for _, jevt := range nsonTestEvents {
evt := &nostr.Event{}
if err := Unmarshal(jevt, evt); err != nil {
t.Fatalf("error unmarshalling nson: %s", err)
}
wrapper := New(jevt)
if id := wrapper.GetID(); id != evt.ID {
t.Fatalf("partial id wrong. got %v, expected %v", id, evt.ID)
}
if pubkey := wrapper.GetPubkey(); pubkey != evt.PubKey {
t.Fatalf("partial pubkey wrong. got %v, expected %v", pubkey, evt.PubKey)
}
if sig := wrapper.GetSig(); sig != evt.Sig {
t.Fatalf("partial sig wrong. got %v, expected %v", sig, evt.Sig)
}
if createdAt := wrapper.GetCreatedAt(); createdAt != evt.CreatedAt {
t.Fatalf("partial created_at wrong. got %v, expected %v", createdAt, evt.CreatedAt)
}
if kind := wrapper.GetKind(); kind != evt.Kind {
t.Fatalf("partial kind wrong. got %v, expected %v", kind, evt.Kind)
}
if content := wrapper.GetContent(); content != evt.Content {
t.Fatalf("partial content wrong. got %v, expected %v", content, evt.Content)
}
}
}
func TestNsonEncode(t *testing.T) {
for _, jevt := range test_common.NormalEvents {
pevt := &nostr.Event{}
if err := json.Unmarshal([]byte(jevt), pevt); err != nil {
t.Fatalf("failed to decode normal json: %s", err)
}
nevt, err := Marshal(pevt)
if err != nil {
t.Fatalf("failed to encode nson: %s", err)
}
evt := &nostr.Event{}
if err := Unmarshal(nevt, evt); err != nil {
t.Fatalf("error unmarshalling nson: %s", err)
}
checkParsedCorrectly(t, pevt, jevt)
checkParsedCorrectly(t, evt, jevt)
}
}
func checkParsedCorrectly(t *testing.T, evt *nostr.Event, jevt string) (isBad bool) {
var canonical nostr.Event
err := json.Unmarshal([]byte(jevt), &canonical)
if err != nil {
t.Fatalf("error unmarshalling normal json: %s", err)
}
if evt.ID != canonical.ID {
t.Fatalf("id is wrong: %s != %s", evt.ID, canonical.ID)
isBad = true
}
if evt.PubKey != canonical.PubKey {
t.Fatalf("pubkey is wrong: %s != %s", evt.PubKey, canonical.PubKey)
isBad = true
}
if evt.Sig != canonical.Sig {
t.Fatalf("sig is wrong: %s != %s", evt.Sig, canonical.Sig)
isBad = true
}
if evt.Content != canonical.Content {
t.Fatalf("content is wrong: %s != %s", evt.Content, canonical.Content)
isBad = true
}
if evt.Kind != canonical.Kind {
t.Fatalf("kind is wrong: %d != %d", evt.Kind, canonical.Kind)
isBad = true
}
if evt.CreatedAt != canonical.CreatedAt {
t.Fatalf("created_at is wrong: %v != %v", evt.CreatedAt, canonical.CreatedAt)
isBad = true
}
if len(evt.Tags) != len(canonical.Tags) {
t.Fatalf("tag number is wrong: %v != %v", len(evt.Tags), len(canonical.Tags))
isBad = true
}
for i := range evt.Tags {
if len(evt.Tags[i]) != len(canonical.Tags[i]) {
t.Fatalf("tag[%d] length is wrong: `%v` != `%v`", i, len(evt.Tags[i]), len(canonical.Tags[i]))
isBad = true
}
for j := range evt.Tags[i] {
if evt.Tags[i][j] != canonical.Tags[i][j] {
t.Fatalf("tag[%d][%d] is wrong: `%s` != `%s`", i, j, evt.Tags[i][j], canonical.Tags[i][j])
isBad = true
}
}
}
return isBad
}
var nsonTestEvents = []string{
`{"id":"192eaf31bd20476bbe9265a3667cfef6410dfd563c02a64cb15d6fa8efec0ed6","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"5b9051596a5ba0619fd5fd7d2766b8aeb0cc398f1d1a0804f4b4ed884482025b3d4888e4c892f2fc437415bfc121482a990fad30f5cd9e333e55364052f99bbc","created_at":1688505641,"nson":"0401000500","kind":1,"content":"hello","tags":[]}`,
`{"id":"921ada34fe581b506975c641f2d1a3fb4f491f1d30c2490452e8524776895ebf","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"1f15a39e93a13f14f783eb127b2977e5dc5d207070dfa280fe45879b6b142ec1943ec921ab4268e69a43704d5641b45d18bf3789037c4842e062cd347a8a7ee1","created_at":1688553190,"nson":"12010006020200060005040005004000120006","kind":1,"content":"maçã","tags":[["entity","fruit"],["owner","79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","wss://リレー.jp","person"]]}`,
`{"id":"06212bae3cfc917d4b1239a3bad4fdba1e0e1ff09fbd2ee7b6da15d5fd859f58","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"47199a3a4184528d2c6cbb94df03b9793ea65b4578154ff5edce794d03ee2408cd3ca699b39cc11e791656e98b510194330d3dc215389c5648eddf33b8362444","created_at":1688572619,"nson":"0401000400","kind":1,"content":"x\ny","tags":[]}`,
`{"id":"ec9345e2af4225aada296964fa6025a1666dcac8dba154f5591a81f7dee1f84a","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","sig":"49f4b9edd7eff9e127b70077daff9a66da8c1ad974e5e6f47c094e8cc0c553071ff61c07b69d3db80c25f36248237ba6021038f5eb6b569ce79e3b024e8e358d","created_at":1688572819,"nson":"0401000400","kind":1,"content":"x\ty","tags":[]}`,
}
var (
EncodedEventEasyJson []byte
EncodedEventNSON string
DecodedEvent *nostr.Event
)
func BenchmarkNSONEncoding(b *testing.B) {
events := make([]*nostr.Event, len(test_common.NormalEvents))
for i, jevt := range test_common.NormalEvents {
evt := &nostr.Event{}
json.Unmarshal([]byte(jevt), evt)
events[i] = evt
}
b.ResetTimer()
b.Run("easyjson.Marshal", func(b *testing.B) {
var encodedEvent []byte
for i := 0; i < b.N; i++ {
for _, evt := range events {
encodedEvent, _ = easyjson.Marshal(evt)
}
}
EncodedEventEasyJson = encodedEvent
})
b.Run("nson.Marshal", func(b *testing.B) {
var encodedEvent string
for i := 0; i < b.N; i++ {
for _, evt := range events {
encodedEvent, _ = Marshal(evt)
}
}
EncodedEventNSON = encodedEvent
})
}
func BenchmarkNSONDecoding(b *testing.B) {
events := make([]string, len(test_common.NormalEvents))
for i, jevt := range test_common.NormalEvents {
evt := &nostr.Event{}
json.Unmarshal([]byte(jevt), evt)
nevt, _ := Marshal(evt)
events[i] = nevt
}
b.ResetTimer()
b.Run("easyjson.Unmarshal", func(b *testing.B) {
evt := &nostr.Event{}
for i := 0; i < b.N; i++ {
for _, nevt := range events {
err := easyjson.Unmarshal([]byte(nevt), evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
}
}
DecodedEvent = evt
})
b.Run("nson.Unmarshal", func(b *testing.B) {
evt := &nostr.Event{}
for i := 0; i < b.N; i++ {
for _, nevt := range events {
err := Unmarshal(nevt, evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
}
}
DecodedEvent = evt
})
b.Run("easyjson.Unmarshal+sig", func(b *testing.B) {
evt := &nostr.Event{}
for i := 0; i < b.N; i++ {
for _, nevt := range events {
err := easyjson.Unmarshal([]byte(nevt), evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
evt.CheckSignature()
}
}
DecodedEvent = evt
})
b.Run("nson.Unmarshal+sig", func(b *testing.B) {
evt := &nostr.Event{}
for i := 0; i < b.N; i++ {
for _, nevt := range events {
err := Unmarshal(nevt, evt)
if err != nil {
b.Fatalf("failed to unmarshal: %s", err)
}
evt.CheckSignature()
}
}
DecodedEvent = evt
})
}