From 1fd99675582b6a57c63ead39c6db139129c87485 Mon Sep 17 00:00:00 2001 From: Blake Mizerany Date: Wed, 11 Dec 2024 17:50:29 -0800 Subject: [PATCH] grammar: introduce new grammar package This package provides a way to convert JSON schemas to equivalent EBNF. It is intended to be a replacement to llama.cpp's schema_to_grammar. This is still an early version and does not yet support all JSON schema features. The to-do list includes: - minumum/maximum constraints on integer types - minLength/maxLength constraints on string types - defs and refs --- grammar/bench_test.go | 22 ++ grammar/grammar.go | 227 +++++++++++++++++++ grammar/grammar_test.go | 75 +++++++ grammar/internal/diff/diff.go | 261 ++++++++++++++++++++++ grammar/internal/diff/diff_test.go | 44 ++++ grammar/internal/diff/testdata/allnew.txt | 13 ++ grammar/internal/diff/testdata/allold.txt | 13 ++ grammar/internal/diff/testdata/basic.txt | 35 +++ grammar/internal/diff/testdata/dups.txt | 40 ++++ grammar/internal/diff/testdata/end.txt | 38 ++++ grammar/internal/diff/testdata/eof.txt | 9 + grammar/internal/diff/testdata/eof1.txt | 18 ++ grammar/internal/diff/testdata/eof2.txt | 18 ++ grammar/internal/diff/testdata/long.txt | 62 +++++ grammar/internal/diff/testdata/same.txt | 5 + grammar/internal/diff/testdata/start.txt | 34 +++ grammar/internal/diff/testdata/triv.txt | 40 ++++ grammar/jsonschema/decode.go | 171 ++++++++++++++ grammar/jsonschema/decode_test.go | 104 +++++++++ grammar/testdata/schemas.txt | 76 +++++++ llm/server.go | 13 +- 21 files changed, 1314 insertions(+), 4 deletions(-) create mode 100644 grammar/bench_test.go create mode 100644 grammar/grammar.go create mode 100644 grammar/grammar_test.go create mode 100644 grammar/internal/diff/diff.go create mode 100644 grammar/internal/diff/diff_test.go create mode 100644 grammar/internal/diff/testdata/allnew.txt create mode 100644 grammar/internal/diff/testdata/allold.txt create mode 100644 grammar/internal/diff/testdata/basic.txt create mode 100644 grammar/internal/diff/testdata/dups.txt create mode 100644 grammar/internal/diff/testdata/end.txt create mode 100644 grammar/internal/diff/testdata/eof.txt create mode 100644 grammar/internal/diff/testdata/eof1.txt create mode 100644 grammar/internal/diff/testdata/eof2.txt create mode 100644 grammar/internal/diff/testdata/long.txt create mode 100644 grammar/internal/diff/testdata/same.txt create mode 100644 grammar/internal/diff/testdata/start.txt create mode 100644 grammar/internal/diff/testdata/triv.txt create mode 100644 grammar/jsonschema/decode.go create mode 100644 grammar/jsonschema/decode_test.go create mode 100644 grammar/testdata/schemas.txt diff --git a/grammar/bench_test.go b/grammar/bench_test.go new file mode 100644 index 000000000..115f0d0b4 --- /dev/null +++ b/grammar/bench_test.go @@ -0,0 +1,22 @@ +//go:build go1.24 + +package grammar + +import "testing" + +func BenchmarkFromSchema(b *testing.B) { + for tt := range testCases(b) { + b.Run("", func(b *testing.B) { + s := []byte(tt.schema) + + b.ReportAllocs() + for b.Loop() { + _, err := FromSchema(nil, s) + if err != nil { + b.Fatalf("GrammarFromSchema: %v", err) + } + } + }) + return + } +} diff --git a/grammar/grammar.go b/grammar/grammar.go new file mode 100644 index 000000000..e6828c0fb --- /dev/null +++ b/grammar/grammar.go @@ -0,0 +1,227 @@ +package grammar + +import ( + "bytes" + "encoding/json" + "fmt" + "iter" + "strconv" + + "github.com/ollama/ollama/grammar/jsonschema" +) + +const jsonTerms = ` +# Unicode +# +# Unicode characters can be specified directly in the grammar, for example +# hiragana ::= [ぁ-ゟ], or with escapes: 8-bit (\xXX), 16-bit (\uXXXX) or 32-bit +# (\UXXXXXXXX). +unicode ::= \x{hex}{2} | \u{hex}{4} | \U{hex}{8} + +# JSON grammar from RFC 7159 +null ::= "null" +object ::= "{" (kv ("," kv)*)? "}" +array ::= "[" (value ("," value)*)? "]" +kv ::= string ":" value +integer ::= "0" | [1-9] [0-9]* +number ::= "-"? integer frac? exp? +frac ::= "." [0-9]+ +exp ::= ("e" | "E") ("+" | "-") [0-9]+ +string ::= "\"" char* "\"" +escape ::= ["/" | "b" | "f" | "n" | "r" | "t" | unicode] +char ::= [^"\\] | escape +space ::= (" " | "\t" | "\n" | "\r")* +hex ::= [0-9] | [a-f] | [A-F] +boolean ::= "true" | "false" +value ::= object | array | string | number | boolean | "null" + +# User-defined +` + +// FromSchema generates a grammar from a JSON schema. +func FromSchema(buf []byte, jsonSchema []byte) ([]byte, error) { + var s *jsonschema.Schema + if err := json.Unmarshal(jsonSchema, &s); err != nil { + return nil, err + } + + var g builder + + // "root" is the only rule that is guaranteed to exist, so we start + // with its length for padding, and then adjust it as we go. + g.pad = len("root") + for id := range dependencies("root", s) { + g.pad = max(g.pad, len(id)) + } + + g.b.WriteString(jsonTerms) + + ids := make(map[*jsonschema.Schema]string) + for id, s := range dependencies("root", s) { + ids[s] = id + g.define(id) + if err := fromSchema(&g, ids, s); err != nil { + return nil, err + } + } + g.define("root") + if err := fromSchema(&g, ids, s); err != nil { + return nil, err + } + g.define("") // finalize the last rule + return g.b.Bytes(), nil +} + +func fromSchema(g *builder, ids map[*jsonschema.Schema]string, s *jsonschema.Schema) error { + switch typ := s.EffectiveType(); typ { + case "array": + if len(s.PrefixItems) == 0 && s.Items == nil { + g.u("array") + } else { + g.q("[") + for i, s := range s.PrefixItems { + if i > 0 { + g.q(",") + } + g.u(ids[s]) + } + if s.Items != nil { + g.u("(") + if len(s.PrefixItems) > 0 { + g.q(",") + } + g.u(ids[s.Items]) + g.u(")*") + } + g.q("]") + } + case "object": + if len(s.Properties) == 0 { + g.u("object") + } else { + g.q("{") + for i, p := range s.Properties { + name := ids[p] + if i > 0 { + g.q(",") + } + g.q(p.Name) + g.q(":") + g.u(name) + } + g.q("}") + } + case "number": + buildConstrainedNumber(g, s) + case "string": + if len(s.Enum) == 0 { + g.u("string") + } else { + g.u("(") + for i, e := range s.Enum { + if i > 0 { + g.q("|") + } + g.q(string(e)) + } + g.u(")") + } + case "boolean", "value", "null", "integer": + g.u(typ) + default: + return fmt.Errorf("%s: unsupported type %q", s.Name, typ) + } + return nil +} + +// dependencies returns a sequence of all child dependencies of the schema in +// post-order. +// +// The first value is the id/pointer to the dependency, and the second value +// is the schema. +func dependencies(id string, s *jsonschema.Schema) iter.Seq2[string, *jsonschema.Schema] { + return func(yield func(string, *jsonschema.Schema) bool) { + for i, p := range s.Properties { + id := fmt.Sprintf("%s_%d", id, i) + for did, d := range dependencies(id, p) { + if !yield(did, d) { + return + } + } + if !yield(id, p) { + return + } + } + for i, p := range s.PrefixItems { + id := fmt.Sprintf("tuple_%d", i) + for did, d := range dependencies(id, p) { + id := fmt.Sprintf("%s_%s", id, did) + if !yield(id, d) { + return + } + } + if !yield(id, p) { + return + } + } + if s.Items != nil { + id := fmt.Sprintf("%s_tuple_%d", id, len(s.PrefixItems)) + for did, d := range dependencies(id, s.Items) { + if !yield(did, d) { + return + } + } + if !yield(id, s.Items) { + return + } + } + } +} + +type builder struct { + b bytes.Buffer + pad int + rules int + items int +} + +// define terminates the current rule, if any, and then either starts a new +// rule or does nothing else if the name is empty. +func (b *builder) define(name string) { + if b.rules > 0 { + b.b.WriteString(";\n") + } + if name == "" { + return + } + fmt.Fprintf(&b.b, "% -*s", b.pad, name) + b.b.WriteString(" ::=") + b.rules++ + b.items = 0 +} + +// quote appends a terminal to the current rule. +func (b *builder) q(s string) { + if b.items > 0 { + b.b.WriteString(" ") + } + b.b.WriteString(" ") + b.b.WriteString(strconv.Quote(s)) +} + +// u appends a non-terminal to the current rule. +func (b *builder) u(s string) { + if b.items > 0 { + b.b.WriteString(" ") + } + b.b.WriteString(" ") + b.b.WriteString(s) +} + +func buildConstrainedNumber(b *builder, s *jsonschema.Schema) { + if s.Minimum == 0 && s.Maximum == 0 { + b.u("TODO") + } else { + b.u("number") + } +} diff --git a/grammar/grammar_test.go b/grammar/grammar_test.go new file mode 100644 index 000000000..5ba315aa9 --- /dev/null +++ b/grammar/grammar_test.go @@ -0,0 +1,75 @@ +package grammar + +import ( + "bufio" + "cmp" + "iter" + "strings" + "testing" + + _ "embed" + + "github.com/ollama/ollama/grammar/internal/diff" +) + +func TestFromSchema(t *testing.T) { + for tt := range testCases(t) { + t.Run(tt.name, func(t *testing.T) { + g, err := FromSchema(nil, []byte(tt.schema)) + if err != nil { + t.Fatalf("FromSchema: %v", err) + } + got := string(g) + got = strings.TrimPrefix(got, jsonTerms) + if got != tt.want { + t.Logf("schema:\n%s", tt.schema) + t.Fatal(string(diff.Diff("got", []byte(got), "want", []byte(tt.want)))) + } + }) + } +} + +type testCase struct { + name string + schema string + want string +} + +//go:embed testdata/schemas.txt +var tests string + +func testCases(t testing.TB) iter.Seq[testCase] { + t.Helper() + return func(yield func(testCase) bool) { + t.Helper() + sc := bufio.NewScanner(strings.NewReader(tests)) + name := "" + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + name = "" + continue + } + if line[0] == '#' { + name = cmp.Or(name, strings.TrimSpace(line[1:])) + continue + } + s := sc.Text() + g := "" + for sc.Scan() { + line = strings.TrimSpace(sc.Text()) + if line == "" || line[0] == '#' { + break + } + g += sc.Text() + "\n" + } + if !yield(testCase{name, s, g}) { + return + } + name = strings.TrimSpace(strings.TrimPrefix(line, "#")) + } + if err := sc.Err(); err != nil { + t.Fatalf("error reading tests: %v", err) + } + } +} diff --git a/grammar/internal/diff/diff.go b/grammar/internal/diff/diff.go new file mode 100644 index 000000000..28aae49bc --- /dev/null +++ b/grammar/internal/diff/diff.go @@ -0,0 +1,261 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package diff + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +// A pair is a pair of values tracked for both the x and y side of a diff. +// It is typically a pair of line indexes. +type pair struct{ x, y int } + +// Diff returns an anchored diff of the two texts old and new +// in the “unified diff” format. If old and new are identical, +// Diff returns a nil slice (no output). +// +// Unix diff implementations typically look for a diff with +// the smallest number of lines inserted and removed, +// which can in the worst case take time quadratic in the +// number of lines in the texts. As a result, many implementations +// either can be made to run for a long time or cut off the search +// after a predetermined amount of work. +// +// In contrast, this implementation looks for a diff with the +// smallest number of “unique” lines inserted and removed, +// where unique means a line that appears just once in both old and new. +// We call this an “anchored diff” because the unique lines anchor +// the chosen matching regions. An anchored diff is usually clearer +// than a standard diff, because the algorithm does not try to +// reuse unrelated blank lines or closing braces. +// The algorithm also guarantees to run in O(n log n) time +// instead of the standard O(n²) time. +// +// Some systems call this approach a “patience diff,” named for +// the “patience sorting” algorithm, itself named for a solitaire card game. +// We avoid that name for two reasons. First, the name has been used +// for a few different variants of the algorithm, so it is imprecise. +// Second, the name is frequently interpreted as meaning that you have +// to wait longer (to be patient) for the diff, meaning that it is a slower algorithm, +// when in fact the algorithm is faster than the standard one. +func Diff(oldName string, old []byte, newName string, new []byte) []byte { + if bytes.Equal(old, new) { + return nil + } + x := lines(old) + y := lines(new) + + // Print diff header. + var out bytes.Buffer + fmt.Fprintf(&out, "diff %s %s\n", oldName, newName) + fmt.Fprintf(&out, "--- %s\n", oldName) + fmt.Fprintf(&out, "+++ %s\n", newName) + + // Loop over matches to consider, + // expanding each match to include surrounding lines, + // and then printing diff chunks. + // To avoid setup/teardown cases outside the loop, + // tgs returns a leading {0,0} and trailing {len(x), len(y)} pair + // in the sequence of matches. + var ( + done pair // printed up to x[:done.x] and y[:done.y] + chunk pair // start lines of current chunk + count pair // number of lines from each side in current chunk + ctext []string // lines for current chunk + ) + for _, m := range tgs(x, y) { + if m.x < done.x { + // Already handled scanning forward from earlier match. + continue + } + + // Expand matching lines as far as possible, + // establishing that x[start.x:end.x] == y[start.y:end.y]. + // Note that on the first (or last) iteration we may (or definitely do) + // have an empty match: start.x==end.x and start.y==end.y. + start := m + for start.x > done.x && start.y > done.y && x[start.x-1] == y[start.y-1] { + start.x-- + start.y-- + } + end := m + for end.x < len(x) && end.y < len(y) && x[end.x] == y[end.y] { + end.x++ + end.y++ + } + + // Emit the mismatched lines before start into this chunk. + // (No effect on first sentinel iteration, when start = {0,0}.) + for _, s := range x[done.x:start.x] { + ctext = append(ctext, "-"+s) + count.x++ + } + for _, s := range y[done.y:start.y] { + ctext = append(ctext, "+"+s) + count.y++ + } + + // If we're not at EOF and have too few common lines, + // the chunk includes all the common lines and continues. + const C = 3 // number of context lines + if (end.x < len(x) || end.y < len(y)) && + (end.x-start.x < C || (len(ctext) > 0 && end.x-start.x < 2*C)) { + for _, s := range x[start.x:end.x] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = end + continue + } + + // End chunk with common lines for context. + if len(ctext) > 0 { + n := end.x - start.x + if n > C { + n = C + } + for _, s := range x[start.x : start.x+n] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = pair{start.x + n, start.y + n} + + // Format and emit chunk. + // Convert line numbers to 1-indexed. + // Special case: empty file shows up as 0,0 not 1,0. + if count.x > 0 { + chunk.x++ + } + if count.y > 0 { + chunk.y++ + } + fmt.Fprintf(&out, "@@ -%d,%d +%d,%d @@\n", chunk.x, count.x, chunk.y, count.y) + for _, s := range ctext { + out.WriteString(s) + } + count.x = 0 + count.y = 0 + ctext = ctext[:0] + } + + // If we reached EOF, we're done. + if end.x >= len(x) && end.y >= len(y) { + break + } + + // Otherwise start a new chunk. + chunk = pair{end.x - C, end.y - C} + for _, s := range x[chunk.x:end.x] { + ctext = append(ctext, " "+s) + count.x++ + count.y++ + } + done = end + } + + return out.Bytes() +} + +// lines returns the lines in the file x, including newlines. +// If the file does not end in a newline, one is supplied +// along with a warning about the missing newline. +func lines(x []byte) []string { + l := strings.SplitAfter(string(x), "\n") + if l[len(l)-1] == "" { + l = l[:len(l)-1] + } else { + // Treat last line as having a message about the missing newline attached, + // using the same text as BSD/GNU diff (including the leading backslash). + l[len(l)-1] += "\n\\ No newline at end of file\n" + } + return l +} + +// tgs returns the pairs of indexes of the longest common subsequence +// of unique lines in x and y, where a unique line is one that appears +// once in x and once in y. +// +// The longest common subsequence algorithm is as described in +// Thomas G. Szymanski, “A Special Case of the Maximal Common +// Subsequence Problem,” Princeton TR #170 (January 1975), +// available at https://research.swtch.com/tgs170.pdf. +func tgs(x, y []string) []pair { + // Count the number of times each string appears in a and b. + // We only care about 0, 1, many, counted as 0, -1, -2 + // for the x side and 0, -4, -8 for the y side. + // Using negative numbers now lets us distinguish positive line numbers later. + m := make(map[string]int) + for _, s := range x { + if c := m[s]; c > -2 { + m[s] = c - 1 + } + } + for _, s := range y { + if c := m[s]; c > -8 { + m[s] = c - 4 + } + } + + // Now unique strings can be identified by m[s] = -1+-4. + // + // Gather the indexes of those strings in x and y, building: + // xi[i] = increasing indexes of unique strings in x. + // yi[i] = increasing indexes of unique strings in y. + // inv[i] = index j such that x[xi[i]] = y[yi[j]]. + var xi, yi, inv []int + for i, s := range y { + if m[s] == -1+-4 { + m[s] = len(yi) + yi = append(yi, i) + } + } + for i, s := range x { + if j, ok := m[s]; ok && j >= 0 { + xi = append(xi, i) + inv = append(inv, j) + } + } + + // Apply Algorithm A from Szymanski's paper. + // In those terms, A = J = inv and B = [0, n). + // We add sentinel pairs {0,0}, and {len(x),len(y)} + // to the returned sequence, to help the processing loop. + J := inv + n := len(xi) + T := make([]int, n) + L := make([]int, n) + for i := range T { + T[i] = n + 1 + } + for i := range n { + k := sort.Search(n, func(k int) bool { + return T[k] >= J[i] + }) + T[k] = J[i] + L[i] = k + 1 + } + k := 0 + for _, v := range L { + if k < v { + k = v + } + } + seq := make([]pair, 2+k) + seq[1+k] = pair{len(x), len(y)} // sentinel at end + lastj := n + for i := n - 1; i >= 0; i-- { + if L[i] == k && J[i] < lastj { + seq[k] = pair{xi[i], yi[J[i]]} + k-- + } + } + seq[0] = pair{0, 0} // sentinel at start + return seq +} diff --git a/grammar/internal/diff/diff_test.go b/grammar/internal/diff/diff_test.go new file mode 100644 index 000000000..ee15b0089 --- /dev/null +++ b/grammar/internal/diff/diff_test.go @@ -0,0 +1,44 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package diff + +import ( + "bytes" + "path/filepath" + "testing" + + "golang.org/x/tools/txtar" +) + +func clean(text []byte) []byte { + text = bytes.ReplaceAll(text, []byte("$\n"), []byte("\n")) + text = bytes.TrimSuffix(text, []byte("^D\n")) + return text +} + +func Test(t *testing.T) { + files, _ := filepath.Glob("testdata/*.txt") + if len(files) == 0 { + t.Fatalf("no testdata") + } + + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + a, err := txtar.ParseFile(file) + if err != nil { + t.Fatal(err) + } + if len(a.Files) != 3 || a.Files[2].Name != "diff" { + t.Fatalf("%s: want three files, third named \"diff\"", file) + } + diffs := Diff(a.Files[0].Name, clean(a.Files[0].Data), a.Files[1].Name, clean(a.Files[1].Data)) + want := clean(a.Files[2].Data) + if !bytes.Equal(diffs, want) { + t.Fatalf("%s: have:\n%s\nwant:\n%s\n%s", file, + diffs, want, Diff("have", diffs, "want", want)) + } + }) + } +} diff --git a/grammar/internal/diff/testdata/allnew.txt b/grammar/internal/diff/testdata/allnew.txt new file mode 100644 index 000000000..887564927 --- /dev/null +++ b/grammar/internal/diff/testdata/allnew.txt @@ -0,0 +1,13 @@ +-- old -- +-- new -- +a +b +c +-- diff -- +diff old new +--- old ++++ new +@@ -0,0 +1,3 @@ ++a ++b ++c diff --git a/grammar/internal/diff/testdata/allold.txt b/grammar/internal/diff/testdata/allold.txt new file mode 100644 index 000000000..bcc9ac0ee --- /dev/null +++ b/grammar/internal/diff/testdata/allold.txt @@ -0,0 +1,13 @@ +-- old -- +a +b +c +-- new -- +-- diff -- +diff old new +--- old ++++ new +@@ -1,3 +0,0 @@ +-a +-b +-c diff --git a/grammar/internal/diff/testdata/basic.txt b/grammar/internal/diff/testdata/basic.txt new file mode 100644 index 000000000..d2565b5d6 --- /dev/null +++ b/grammar/internal/diff/testdata/basic.txt @@ -0,0 +1,35 @@ +Example from Hunt and McIlroy, “An Algorithm for Differential File Comparison.” +https://www.cs.dartmouth.edu/~doug/diff.pdf + +-- old -- +a +b +c +d +e +f +g +-- new -- +w +a +b +x +y +z +e +-- diff -- +diff old new +--- old ++++ new +@@ -1,7 +1,7 @@ ++w + a + b +-c +-d ++x ++y ++z + e +-f +-g diff --git a/grammar/internal/diff/testdata/dups.txt b/grammar/internal/diff/testdata/dups.txt new file mode 100644 index 000000000..d10524d0d --- /dev/null +++ b/grammar/internal/diff/testdata/dups.txt @@ -0,0 +1,40 @@ +-- old -- +a + +b + +c + +d + +e + +f +-- new -- +a + +B + +C + +d + +e + +f +-- diff -- +diff old new +--- old ++++ new +@@ -1,8 +1,8 @@ + a + $ +-b +- +-c ++B ++ ++C + $ + d + $ diff --git a/grammar/internal/diff/testdata/end.txt b/grammar/internal/diff/testdata/end.txt new file mode 100644 index 000000000..158637c13 --- /dev/null +++ b/grammar/internal/diff/testdata/end.txt @@ -0,0 +1,38 @@ +-- old -- +1 +2 +3 +4 +5 +6 +7 +eight +nine +ten +eleven +-- new -- +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +-- diff -- +diff old new +--- old ++++ new +@@ -5,7 +5,6 @@ + 5 + 6 + 7 +-eight +-nine +-ten +-eleven ++8 ++9 ++10 diff --git a/grammar/internal/diff/testdata/eof.txt b/grammar/internal/diff/testdata/eof.txt new file mode 100644 index 000000000..5dc145c4d --- /dev/null +++ b/grammar/internal/diff/testdata/eof.txt @@ -0,0 +1,9 @@ +-- old -- +a +b +c^D +-- new -- +a +b +c^D +-- diff -- diff --git a/grammar/internal/diff/testdata/eof1.txt b/grammar/internal/diff/testdata/eof1.txt new file mode 100644 index 000000000..1ebf621e9 --- /dev/null +++ b/grammar/internal/diff/testdata/eof1.txt @@ -0,0 +1,18 @@ +-- old -- +a +b +c +-- new -- +a +b +c^D +-- diff -- +diff old new +--- old ++++ new +@@ -1,3 +1,3 @@ + a + b +-c ++c +\ No newline at end of file diff --git a/grammar/internal/diff/testdata/eof2.txt b/grammar/internal/diff/testdata/eof2.txt new file mode 100644 index 000000000..047705e68 --- /dev/null +++ b/grammar/internal/diff/testdata/eof2.txt @@ -0,0 +1,18 @@ +-- old -- +a +b +c^D +-- new -- +a +b +c +-- diff -- +diff old new +--- old ++++ new +@@ -1,3 +1,3 @@ + a + b +-c +\ No newline at end of file ++c diff --git a/grammar/internal/diff/testdata/long.txt b/grammar/internal/diff/testdata/long.txt new file mode 100644 index 000000000..3fc99f71d --- /dev/null +++ b/grammar/internal/diff/testdata/long.txt @@ -0,0 +1,62 @@ +-- old -- +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +14½ +15 +16 +17 +18 +19 +20 +-- new -- +1 +2 +3 +4 +5 +6 +8 +9 +10 +11 +12 +13 +14 +17 +18 +19 +20 +-- diff -- +diff old new +--- old ++++ new +@@ -4,7 +4,6 @@ + 4 + 5 + 6 +-7 + 8 + 9 + 10 +@@ -12,9 +11,6 @@ + 12 + 13 + 14 +-14½ +-15 +-16 + 17 + 18 + 19 diff --git a/grammar/internal/diff/testdata/same.txt b/grammar/internal/diff/testdata/same.txt new file mode 100644 index 000000000..86b1100d8 --- /dev/null +++ b/grammar/internal/diff/testdata/same.txt @@ -0,0 +1,5 @@ +-- old -- +hello world +-- new -- +hello world +-- diff -- diff --git a/grammar/internal/diff/testdata/start.txt b/grammar/internal/diff/testdata/start.txt new file mode 100644 index 000000000..217b2fdc9 --- /dev/null +++ b/grammar/internal/diff/testdata/start.txt @@ -0,0 +1,34 @@ +-- old -- +e +pi +4 +5 +6 +7 +8 +9 +10 +-- new -- +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +-- diff -- +diff old new +--- old ++++ new +@@ -1,5 +1,6 @@ +-e +-pi ++1 ++2 ++3 + 4 + 5 + 6 diff --git a/grammar/internal/diff/testdata/triv.txt b/grammar/internal/diff/testdata/triv.txt new file mode 100644 index 000000000..ab5759fcb --- /dev/null +++ b/grammar/internal/diff/testdata/triv.txt @@ -0,0 +1,40 @@ +Another example from Hunt and McIlroy, +“An Algorithm for Differential File Comparison.” +https://www.cs.dartmouth.edu/~doug/diff.pdf + +Anchored diff gives up on finding anything, +since there are no unique lines. + +-- old -- +a +b +c +a +b +b +a +-- new -- +c +a +b +a +b +c +-- diff -- +diff old new +--- old ++++ new +@@ -1,7 +1,6 @@ +-a +-b +-c +-a +-b +-b +-a ++c ++a ++b ++a ++b ++c diff --git a/grammar/jsonschema/decode.go b/grammar/jsonschema/decode.go new file mode 100644 index 000000000..b6d51be2f --- /dev/null +++ b/grammar/jsonschema/decode.go @@ -0,0 +1,171 @@ +package jsonschema + +import ( + "bytes" + "encoding/json" + "errors" +) + +// Schema holds a JSON schema. +type Schema struct { + // Name is the name of the property. For the parent/root property, this + // is "root". For child properties, this is the name of the property. + Name string `json:"-"` + + // Type is the type of the property. + // + // TODO: Union types (e.g. make this a []string). + Type string + + // PrefixItems is a list of schemas for each item in a tuple. By + // default, the tuple is "closed." unless Items is set to true or a + // valid Schema. + PrefixItems []*Schema + + // Items is the schema for each item in a list. + // + // If it is missing, or its JSON value is "null" or "false", it is nil. + // If the JSON value is "true", it is set to the empty Schema. If the + // JSON value is an object, it will be decoded as a Schema. + Items *Schema + + // MinItems specifies the minimum number of items allowed in a list. + MinItems int + + // MaxItems specifies the maximum number of items allowed in a list. + MaxItems int + + // Properties is the schema for each property of an object. + Properties []*Schema + + // Format is the format of the property. This is used to validate the + // property against a specific format. + // + // It is the callers responsibility to validate the property against + // the format. + Format string + + // Minimum specifies the minimum value for numeric properties. + Minimum float64 + + // Maximum specifies the maximum value for numeric properties. + Maximum float64 + + // Enum is a list of valid values for the property. + Enum []json.RawMessage +} + +func (s *Schema) UnmarshalJSON(data []byte) error { + type S Schema + w := struct { + Properties props + Items items + *S + }{ + S: (*S)(s), + } + if err := json.Unmarshal(data, &w); err != nil { + return err + } + if w.Items.set { + s.Items = &w.Items.Schema + } + s.Properties = w.Properties + return nil +} + +type items struct { + Schema + set bool +} + +func (s *items) UnmarshalJSON(data []byte) error { + switch b := data[0]; b { + case 't': + *s = items{set: true} + case '{': + type I items + if err := json.Unmarshal(data, (*I)(s)); err != nil { + return err + } + s.set = true + case 'n', 'f': + default: + return errors.New("invalid Items") + } + return nil +} + +// EffectiveType returns the effective type of the schema. If the Type field is +// not empty, it is returned; otherwise: +// +// - If the schema has both Properties and Items, it returns an empty string. +// - If the schema has Properties, it returns "object". +// - If the schema has Items, it returns "array". +// - If the schema has neither Properties nor Items, it returns "value". +// +// The returned string is never empty. +func (d *Schema) EffectiveType() string { + if d.Type == "" { + if len(d.Properties) > 0 { + return "object" + } + if len(d.PrefixItems) > 0 || d.Items != nil { + return "array" + } + return "value" + } + return d.Type +} + +// props is an ordered list of properties. The order of the properties +// is the order in which they were defined in the schema. +type props []*Schema + +var _ json.Unmarshaler = (*props)(nil) + +func (v *props) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + return nil + } + if data[0] != '{' { + return errors.New("expected object") + } + + d := json.NewDecoder(bytes.NewReader(data)) + + // TODO(bmizerany): Consider DisallowUnknownFields. Currently, we, like + // llama.cpp, ignore unknown fields, which could be lead to unexpected + // behavior for clients of this package, since they may not be aware + // that "additionalFields", "itemsPrefix", etc, are being ignored. + // + // For now, just do what llama.cpp does. + + t, err := d.Token() + if err != nil { + return err + } + if t != json.Delim('{') { + return errors.New("expected object") + } + for d.More() { + // Use the first token (map key) as the property name, then + // decode the rest of the object fields into a Schema and + // append. + t, err := d.Token() + if err != nil { + return err + } + if t == json.Delim('}') { + return nil + } + s := &Schema{ + Name: t.(string), + } + if err := d.Decode(s); err != nil { + return err + } + *v = append(*v, s) + } + return nil +} diff --git a/grammar/jsonschema/decode_test.go b/grammar/jsonschema/decode_test.go new file mode 100644 index 000000000..2bfdb8b4d --- /dev/null +++ b/grammar/jsonschema/decode_test.go @@ -0,0 +1,104 @@ +package jsonschema + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +const testSchemaBasic = ` +{ + "properties": { + "tupleClosedEmpty": { "prefixItems": [] }, + "tupleClosedMissing": { "prefixItems": [{}] }, + "tupleClosedNull": { "prefixItems": [{}], "items": null }, + "tupleClosedFalse": { "prefixItems": [{}], "items": false }, + "tupleOpenTrue": { "prefixItems": [{}], "items": true }, + "tupleOpenEmpty": { "prefixItems": [{}], "items": {} }, + "tupleOpenTyped": { "prefixItems": [{}], "items": {"type": "boolean"} }, + "tupleOpenMax": { "prefixItems": [{}], "items": true, "maxItems": 3}, + + "array": { "items": {"type": "number"} }, + + "null": { "type": "null" }, + "string": { "type": "string" }, + "boolean": { "type": "boolean" } + } +} +` + +func TestSchemaUnmarshal(t *testing.T) { + var got *Schema + if err := json.Unmarshal([]byte(testSchemaBasic), &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + want := &Schema{ + Properties: []*Schema{ + {Name: "tupleClosedEmpty", PrefixItems: []*Schema{}, Items: nil}, + {Name: "tupleClosedMissing", PrefixItems: []*Schema{{}}, Items: nil}, + {Name: "tupleClosedNull", PrefixItems: []*Schema{{}}, Items: nil}, + {Name: "tupleClosedFalse", PrefixItems: []*Schema{{}}, Items: nil}, + + {Name: "tupleOpenTrue", PrefixItems: []*Schema{{}}, Items: &Schema{}}, + {Name: "tupleOpenEmpty", PrefixItems: []*Schema{{}}, Items: &Schema{}}, + {Name: "tupleOpenTyped", PrefixItems: []*Schema{{}}, Items: &Schema{Type: "boolean"}}, + {Name: "tupleOpenMax", PrefixItems: []*Schema{{}}, Items: &Schema{}, MaxItems: 3}, + + {Name: "array", Items: &Schema{Type: "number"}}, + + {Name: "null", Type: "null"}, + {Name: "string", Type: "string"}, + {Name: "boolean", Type: "boolean"}, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("(-want, +got)\n%s", diff) + } +} + +func TestEffectiveType(t *testing.T) { + const schema = ` + {"properties": { + "o": {"type": "object"}, + "a": {"type": "array"}, + "n": {"type": "number"}, + "s": {"type": "string"}, + "z": {"type": "null"}, + "b": {"type": "boolean"}, + + "t0": {"prefixItems": [{}], "items": {"type": "number"}}, + "t1": {"items": {"type": "number"}, "maxItems": 3}, + + "v": {"maxItems": 3} + }} + ` + + var s *Schema + if err := json.Unmarshal([]byte(schema), &s); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + + var got []string + for _, p := range s.Properties { + got = append(got, p.EffectiveType()) + } + + want := strings.Fields(` + object + array + number + string + null + boolean + array + array + value + `) + if !reflect.DeepEqual(want, got) { + t.Errorf("\ngot:\n\t%v\nwant:\n\t%v", got, want) + } +} diff --git a/grammar/testdata/schemas.txt b/grammar/testdata/schemas.txt new file mode 100644 index 000000000..6cb24c9ed --- /dev/null +++ b/grammar/testdata/schemas.txt @@ -0,0 +1,76 @@ +# This file holds tests for JSON schema to EBNF grammar conversions. +# +# The format is a JSON schema, followed by the expected EBNF grammar. Each test +# MAY be preceded by a comment that describes the test (e.g. the test name), followed by +# the JSON schema and the expected EBNF grammar. If no comment is present, the test +# name the tests number in the file (e.g. "#0", "#1", etc.) +# +# Blank lines signify the end or start of a new test. Comments can be added +# anywhere in the file, but they must be preceded by a '#' character and start at +# the beginning of the line. + +# default +{} +root ::= value; + +{"properties": {}} +root ::= value; + +# array +{"properties": {"a": {"type": "array", "items": {"type": "string"}}}} +root_0_tuple_0 ::= string; +root_0 ::= "[" ( root_0_tuple_0 )* "]"; +root ::= "{" "a" ":" root_0 "}"; + +# array with nested array +{"type": "array", "items": {"type": "array", "items": {"type": "string"}}} +root_tuple_0_tuple_0 ::= string; +root_tuple_0 ::= "[" ( root_tuple_0_tuple_0 )* "]"; +root ::= "[" ( root_tuple_0 )* "]"; + +# object +{"properties": {"e": {}}} +root_0 ::= value; +root ::= "{" "e" ":" root_0 "}"; + +# object with nested object +{"properties": {"o": {"type": "object", "properties": {"e": {}}}}} +root_0_0 ::= value; +root_0 ::= "{" "e" ":" root_0_0 "}"; +root ::= "{" "o" ":" root_0 "}"; + +# boolean +{"type": "boolean"} +root ::= boolean; + +# number +{"properties": {"n": {"type": "number", "minimum": 123, "maximum": 4567}}} +root_0 ::= number; +root ::= "{" "n" ":" root_0 "}"; + +# string +{"type": "string"} +root ::= string; + +# string with enum +{"type": "string", "enum": ["a", "b", "c"]} +root ::= ( "\"a\"" "|" "\"b\"" "|" "\"c\"" ); + +# spaces in key +{"properties": {"a b": {}}} +root_0 ::= value; +root ::= "{" "a b" ":" root_0 "}"; + +# issue7978 +{ "type": "object", "properties": { "steps": { "type": "array", "items": { "type": "object", "properties": { "explanation": { "type": "string" }, "output": { "type": "string" } }, "required": [ "explanation", "output" ], "additionalProperties": false } }, "final_answer": { "type": "string" } }, "required": [ "steps", "final_answer" ], "additionalProperties": false } +root_0_tuple_0_0 ::= string; +root_0_tuple_0_1 ::= string; +root_0_tuple_0 ::= "{" "explanation" ":" root_0_tuple_0_0 "," "output" ":" root_0_tuple_0_1 "}"; +root_0 ::= "[" ( root_0_tuple_0 )* "]"; +root_1 ::= string; +root ::= "{" "steps" ":" root_0 "," "final_answer" ":" root_1 "}"; + +# !! # special characters in key +# !! {"properties": {"a!b": {}}} +# !! !invalid character '!' in key +# !! diff --git a/llm/server.go b/llm/server.go index adc11aaea..e6b8f91d9 100644 --- a/llm/server.go +++ b/llm/server.go @@ -29,6 +29,7 @@ import ( "github.com/ollama/ollama/envconfig" "github.com/ollama/ollama/format" "github.com/ollama/ollama/fs/ggml" + "github.com/ollama/ollama/grammar" "github.com/ollama/ollama/llama" "github.com/ollama/ollama/model" ) @@ -700,9 +701,9 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu } // User provided a JSON schema - g := llama.SchemaToGrammar(req.Format) - if g == nil { - return fmt.Errorf("invalid JSON schema in format") + g, err := grammar.FromSchema(nil, req.Format) + if err != nil { + return fmt.Errorf("invalid JSON schema in format: %w", err) } req.Grammar = string(g) } @@ -713,6 +714,11 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu req.Options = &opts } + if req.Options == nil { + opts := api.DefaultOptions() + req.Options = &opts + } + if err := s.sem.Acquire(ctx, 1); err != nil { if errors.Is(err, context.Canceled) { slog.Info("aborting completion request due to client closing the connection") @@ -727,7 +733,6 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu if req.Options.NumPredict < 0 || req.Options.NumPredict > 10*s.options.NumCtx { req.Options.NumPredict = 10 * s.options.NumCtx } - // Make sure the server is ready status, err := s.getServerStatusRetry(ctx) if err != nil {