From 4e986a823ca47eb16f563d15a6fe4cc393a00715 Mon Sep 17 00:00:00 2001 From: Josh Yan Date: Thu, 27 Jun 2024 10:59:15 -0700 Subject: [PATCH 1/4] unquote, trimp space --- parser/parser.go | 9 ++++++++- parser/parser_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/parser/parser.go b/parser/parser.go index 686a1e695..fa60ebc0f 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -125,6 +125,7 @@ func ParseFile(r io.Reader) (*File, error) { // pass case stateValue: s, ok := unquote(b.String()) + if !ok || isSpace(r) { if _, err := b.WriteRune(r); err != nil { return nil, err @@ -158,7 +159,13 @@ func ParseFile(r io.Reader) (*File, error) { case stateComment, stateNil: // pass; nothing to flush case stateValue: - s, ok := unquote(b.String()) + var s string + var ok bool + if cmd.Name == "model" { + s, ok = unquote(strings.TrimSpace(b.String())) + } else { + s, ok = unquote(b.String()) + } if !ok { return nil, io.ErrUnexpectedEOF } diff --git a/parser/parser_test.go b/parser/parser_test.go index 7123e53bf..35556515d 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -48,6 +48,26 @@ func TestParseFileFrom(t *testing.T) { expected []Command err error }{ + { + "FROM \"FOO BAR \"", + []Command{{Name: "model", Args: "FOO BAR "}}, + nil, + }, + { + "FROM \"FOO BAR\"\nPARAMETER param1 value1", + []Command{{Name: "model", Args: "FOO BAR"}, {Name: "param1", Args: "value1"}}, + nil, + }, + { + "FROM FOOO BAR ", + []Command{{Name: "model", Args: "FOOO BAR"}}, + nil, + }, + { + "FROM /what/is/the path ", + []Command{{Name: "model", Args: "/what/is/the path"}}, + nil, + }, { "FROM foo", []Command{{Name: "model", Args: "foo"}}, @@ -86,6 +106,11 @@ func TestParseFileFrom(t *testing.T) { []Command{{Name: "param1", Args: "value1"}, {Name: "model", Args: "foo"}}, nil, }, + { + "PARAMETER what the \nFROM lemons make lemonade ", + []Command{{Name: "what", Args: "the "}, {Name: "model", Args: "lemons make lemonade"}}, + nil, + }, } for _, c := range cases { From 9bd00041fa1c82881299f34a5950f9edc2a7e66c Mon Sep 17 00:00:00 2001 From: Josh Yan Date: Thu, 27 Jun 2024 11:18:38 -0700 Subject: [PATCH 2/4] trim all params --- parser/parser.go | 11 ++--------- parser/parser_test.go | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index fa60ebc0f..7f566da4e 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -124,8 +124,7 @@ func ParseFile(r io.Reader) (*File, error) { case stateComment, stateNil: // pass case stateValue: - s, ok := unquote(b.String()) - + s, ok := unquote(strings.TrimSpace(b.String())) if !ok || isSpace(r) { if _, err := b.WriteRune(r); err != nil { return nil, err @@ -159,13 +158,7 @@ func ParseFile(r io.Reader) (*File, error) { case stateComment, stateNil: // pass; nothing to flush case stateValue: - var s string - var ok bool - if cmd.Name == "model" { - s, ok = unquote(strings.TrimSpace(b.String())) - } else { - s, ok = unquote(b.String()) - } + s, ok := unquote(strings.TrimSpace(b.String())) if !ok { return nil, io.ErrUnexpectedEOF } diff --git a/parser/parser_test.go b/parser/parser_test.go index 35556515d..3dc592239 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -108,7 +108,7 @@ func TestParseFileFrom(t *testing.T) { }, { "PARAMETER what the \nFROM lemons make lemonade ", - []Command{{Name: "what", Args: "the "}, {Name: "model", Args: "lemons make lemonade"}}, + []Command{{Name: "what", Args: "the"}, {Name: "model", Args: "lemons make lemonade"}}, nil, }, } @@ -424,7 +424,7 @@ func TestParseFileParameters(t *testing.T) { "mirostat_eta 1.0": {"mirostat_eta", "1.0"}, "penalize_newline true": {"penalize_newline", "true"}, "stop ### User:": {"stop", "### User:"}, - "stop ### User: ": {"stop", "### User: "}, + "stop ### User: ": {"stop", "### User:"}, "stop \"### User:\"": {"stop", "### User:"}, "stop \"### User: \"": {"stop", "### User: "}, "stop \"\"\"### User:\"\"\"": {"stop", "### User:"}, From 26e4e66faff20a94bb8fee9ec2bc3e17a07fb19e Mon Sep 17 00:00:00 2001 From: Josh Yan Date: Mon, 1 Jul 2024 09:43:49 -0700 Subject: [PATCH 3/4] updated parsefile test --- parser/parser_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 3dc592239..171bd4206 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -22,7 +22,13 @@ ADAPTER adapter1 LICENSE MIT PARAMETER param1 value1 PARAMETER param2 value2 -TEMPLATE template1 +TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ .Response }}<|eot_id|>""" ` reader := strings.NewReader(input) @@ -36,7 +42,7 @@ TEMPLATE template1 {Name: "license", Args: "MIT"}, {Name: "param1", Args: "value1"}, {Name: "param2", Args: "value2"}, - {Name: "template", Args: "template1"}, + {Name: "template", Args: "{{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|>"}, } assert.Equal(t, expectedCommands, modelfile.Commands) From 7e571f95f0306f90e4f754e34df96ebc36f93626 Mon Sep 17 00:00:00 2001 From: Josh Yan Date: Mon, 1 Jul 2024 11:07:48 -0700 Subject: [PATCH 4/4] trimspace test case --- parser/parser_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/parser/parser_test.go b/parser/parser_test.go index 171bd4206..2b5c4c888 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -48,6 +48,39 @@ TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|> assert.Equal(t, expectedCommands, modelfile.Commands) } +func TestParseFileTrimSpace(t *testing.T) { + input := ` +FROM " model 1" +ADAPTER adapter3 +LICENSE "MIT " +PARAMETER param1 value1 +PARAMETER param2 value2 +TEMPLATE """ {{ if .System }}<|start_header_id|>system<|end_header_id|> + +{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|> + +{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|> + +{{ .Response }}<|eot_id|> """ +` + + reader := strings.NewReader(input) + + modelfile, err := ParseFile(reader) + require.NoError(t, err) + + expectedCommands := []Command{ + {Name: "model", Args: " model 1"}, + {Name: "adapter", Args: "adapter3"}, + {Name: "license", Args: "MIT "}, + {Name: "param1", Args: "value1"}, + {Name: "param2", Args: "value2"}, + {Name: "template", Args: " {{ if .System }}<|start_header_id|>system<|end_header_id|>\n\n{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>\n\n{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>\n\n{{ .Response }}<|eot_id|> "}, + } + + assert.Equal(t, expectedCommands, modelfile.Commands) +} + func TestParseFileFrom(t *testing.T) { var cases = []struct { input string