diff options
Diffstat (limited to 'src/pkg/encoding/json')
| -rw-r--r-- | src/pkg/encoding/json/decode.go | 16 | ||||
| -rw-r--r-- | src/pkg/encoding/json/decode_test.go | 42 | ||||
| -rw-r--r-- | src/pkg/encoding/json/encode.go | 41 | ||||
| -rw-r--r-- | src/pkg/encoding/json/encode_test.go | 31 | ||||
| -rw-r--r-- | src/pkg/encoding/json/example_test.go | 32 | ||||
| -rw-r--r-- | src/pkg/encoding/json/fold.go | 143 | ||||
| -rw-r--r-- | src/pkg/encoding/json/fold_test.go | 116 | ||||
| -rw-r--r-- | src/pkg/encoding/json/indent.go | 5 | ||||
| -rw-r--r-- | src/pkg/encoding/json/scanner_test.go | 22 | ||||
| -rw-r--r-- | src/pkg/encoding/json/stream.go | 5 |
10 files changed, 410 insertions, 43 deletions
diff --git a/src/pkg/encoding/json/decode.go b/src/pkg/encoding/json/decode.go index 458fb39ec..af1c908ad 100644 --- a/src/pkg/encoding/json/decode.go +++ b/src/pkg/encoding/json/decode.go @@ -8,6 +8,7 @@ package json import ( + "bytes" "encoding" "encoding/base64" "errors" @@ -15,7 +16,6 @@ import ( "reflect" "runtime" "strconv" - "strings" "unicode" "unicode/utf16" "unicode/utf8" @@ -54,6 +54,11 @@ import ( // If no more serious errors are encountered, Unmarshal returns // an UnmarshalTypeError describing the earliest such error. // +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// // When unmarshaling quoted strings, invalid UTF-8 or // invalid UTF-16 surrogate pairs are not treated as an error. // Instead, they are replaced by the Unicode replacement @@ -500,11 +505,11 @@ func (d *decodeState) object(v reflect.Value) { d.error(errPhase) } - // Read string key. + // Read key. start := d.off - 1 op = d.scanWhile(scanContinue) item := d.data[start : d.off-1] - key, ok := unquote(item) + key, ok := unquoteBytes(item) if !ok { d.error(errPhase) } @@ -526,11 +531,11 @@ func (d *decodeState) object(v reflect.Value) { fields := cachedTypeFields(v.Type()) for i := range fields { ff := &fields[i] - if ff.name == key { + if bytes.Equal(ff.nameBytes, key) { f = ff break } - if f == nil && strings.EqualFold(ff.name, key) { + if f == nil && ff.equalFold(ff.nameBytes, key) { f = ff } } @@ -561,6 +566,7 @@ func (d *decodeState) object(v reflect.Value) { if destring { d.value(reflect.ValueOf(&d.tempstr)) d.literalStore([]byte(d.tempstr), subv, true) + d.tempstr = "" // Zero scratch space for successive values. } else { d.value(subv) } diff --git a/src/pkg/encoding/json/decode_test.go b/src/pkg/encoding/json/decode_test.go index 22c5f89f7..238a87fd6 100644 --- a/src/pkg/encoding/json/decode_test.go +++ b/src/pkg/encoding/json/decode_test.go @@ -1060,6 +1060,21 @@ func TestEmptyString(t *testing.T) { } } +// Test that the returned error is non-nil when trying to unmarshal null string into int, for successive ,string option +// Issue 7046 +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + } + data := []byte(`{"A": "1", "B": null}`) + var s T + err := Unmarshal(data, &s) + if err == nil { + t.Fatalf("expected error; got %v", s) + } +} + func intp(x int) *int { p := new(int) *p = x @@ -1110,8 +1125,8 @@ func TestInterfaceSet(t *testing.T) { // Issue 2540 func TestUnmarshalNulls(t *testing.T) { jsonData := []byte(`{ - "Bool" : null, - "Int" : null, + "Bool" : null, + "Int" : null, "Int8" : null, "Int16" : null, "Int32" : null, @@ -1316,3 +1331,26 @@ func TestPrefilled(t *testing.T) { } } } + +var invalidUnmarshalTests = []struct { + v interface{} + want string +}{ + {nil, "json: Unmarshal(nil)"}, + {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {(*int)(nil), "json: Unmarshal(nil *int)"}, +} + +func TestInvalidUnmarshal(t *testing.T) { + buf := []byte(`{"a":"1"}`) + for _, tt := range invalidUnmarshalTests { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Errorf("Unmarshal expecting error, got nil") + continue + } + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } + } +} diff --git a/src/pkg/encoding/json/encode.go b/src/pkg/encoding/json/encode.go index 7d6c71d7a..741ddd89c 100644 --- a/src/pkg/encoding/json/encode.go +++ b/src/pkg/encoding/json/encode.go @@ -44,6 +44,7 @@ import ( // if an invalid UTF-8 sequence is encountered. // The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" // to keep some browsers from misinterpreting JSON output as HTML. +// Ampersand "&" is also escaped to "\u0026" for the same reason. // // Array and slice values encode as JSON arrays, except that // []byte encodes as a base64-encoded string, and a nil slice @@ -241,24 +242,15 @@ type encodeState struct { scratch [64]byte } -// TODO(bradfitz): use a sync.Cache here -var encodeStatePool = make(chan *encodeState, 8) +var encodeStatePool sync.Pool func newEncodeState() *encodeState { - select { - case e := <-encodeStatePool: + if v := encodeStatePool.Get(); v != nil { + e := v.(*encodeState) e.Reset() return e - default: - return new(encodeState) - } -} - -func putEncodeState(e *encodeState) { - select { - case encodeStatePool <- e: - default: } + return new(encodeState) } func (e *encodeState) marshal(v interface{}) (err error) { @@ -813,7 +805,7 @@ func (e *encodeState) string(s string) (int, error) { e.WriteByte('r') default: // This encodes bytes < 0x20 except for \n and \r, - // as well as < and >. The latter are escaped because they + // as well as <, > and &. The latter are escaped because they // can lead to security holes when user-controlled strings // are rendered into JSON and served to some browsers. e.WriteString(`\u00`) @@ -936,6 +928,9 @@ func (e *encodeState) stringBytes(s []byte) (int, error) { // A field represents a single field found in a struct. type field struct { name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + tag bool index []int typ reflect.Type @@ -943,6 +938,12 @@ type field struct { quoted bool } +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + // byName sorts field by name, breaking ties with depth, // then breaking ties with "name came from json tag", then // breaking ties with index sequence. @@ -1042,8 +1043,14 @@ func typeFields(t reflect.Type) []field { if name == "" { name = sf.Name } - fields = append(fields, field{name, tagged, index, ft, - opts.Contains("omitempty"), opts.Contains("string")}) + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. @@ -1057,7 +1064,7 @@ func typeFields(t reflect.Type) []field { // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { - next = append(next, field{name: ft.Name(), index: index, typ: ft}) + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) } } } diff --git a/src/pkg/encoding/json/encode_test.go b/src/pkg/encoding/json/encode_test.go index 9395db7cb..2e89a78eb 100644 --- a/src/pkg/encoding/json/encode_test.go +++ b/src/pkg/encoding/json/encode_test.go @@ -25,13 +25,30 @@ type Optionals struct { Mr map[string]interface{} `json:"mr"` Mo map[string]interface{} `json:",omitempty"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitempty"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitempty"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitempty"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitempty"` } var optionalsExpected = `{ "sr": "", "omitempty": 0, "slr": null, - "mr": {} + "mr": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "sto": {} }` func TestOmitEmpty(t *testing.T) { @@ -76,7 +93,7 @@ func TestStringTag(t *testing.T) { // Verify that it round-trips. var s2 StringTag - err = NewDecoder(bytes.NewBuffer(got)).Decode(&s2) + err = NewDecoder(bytes.NewReader(got)).Decode(&s2) if err != nil { t.Fatalf("Decode: %v", err) } @@ -425,3 +442,13 @@ func TestIssue6458(t *testing.T) { t.Errorf("Marshal(x) = %#q; want %#q", b, want) } } + +func TestHTMLEscape(t *testing.T) { + var b, want bytes.Buffer + m := `{"M":"<html>foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `</html>"}` + want.Write([]byte(`{"M":"\u003chtml\u003efoo \u0026\u2028 \u2029\u003c/html\u003e"}`)) + HTMLEscape(&b, []byte(m)) + if !bytes.Equal(b.Bytes(), want.Bytes()) { + t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) + } +} diff --git a/src/pkg/encoding/json/example_test.go b/src/pkg/encoding/json/example_test.go index ea0bc149c..ca4e5ae68 100644 --- a/src/pkg/encoding/json/example_test.go +++ b/src/pkg/encoding/json/example_test.go @@ -5,6 +5,7 @@ package json_test import ( + "bytes" "encoding/json" "fmt" "io" @@ -127,3 +128,34 @@ func ExampleRawMessage() { // YCbCr &{255 0 -10} // RGB &{98 218 255} } + +func ExampleIndent() { + type Road struct { + Name string + Number int + } + roads := []Road{ + {"Diamond Fork", 29}, + {"Sheep Creek", 51}, + } + + b, err := json.Marshal(roads) + if err != nil { + log.Fatal(err) + } + + var out bytes.Buffer + json.Indent(&out, b, "=", "\t") + out.WriteTo(os.Stdout) + // Output: + // [ + // = { + // = "Name": "Diamond Fork", + // = "Number": 29 + // = }, + // = { + // = "Name": "Sheep Creek", + // = "Number": 51 + // = } + // =] +} diff --git a/src/pkg/encoding/json/fold.go b/src/pkg/encoding/json/fold.go new file mode 100644 index 000000000..d6f77c93e --- /dev/null +++ b/src/pkg/encoding/json/fold.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "unicode/utf8" +) + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} diff --git a/src/pkg/encoding/json/fold_test.go b/src/pkg/encoding/json/fold_test.go new file mode 100644 index 000000000..9fb94646a --- /dev/null +++ b/src/pkg/encoding/json/fold_test.go @@ -0,0 +1,116 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "strings" + "testing" + "unicode/utf8" +) + +var foldTests = []struct { + fn func(s, t []byte) bool + s, t string + want bool +}{ + {equalFoldRight, "", "", true}, + {equalFoldRight, "a", "a", true}, + {equalFoldRight, "", "a", false}, + {equalFoldRight, "a", "", false}, + {equalFoldRight, "a", "A", true}, + {equalFoldRight, "AB", "ab", true}, + {equalFoldRight, "AB", "ac", false}, + {equalFoldRight, "sbkKc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKK", false}, + {equalFoldRight, "e", "é", false}, + {equalFoldRight, "s", "S", true}, + + {simpleLetterEqualFold, "", "", true}, + {simpleLetterEqualFold, "abc", "abc", true}, + {simpleLetterEqualFold, "abc", "ABC", true}, + {simpleLetterEqualFold, "abc", "ABCD", false}, + {simpleLetterEqualFold, "abc", "xxx", false}, + + {asciiEqualFold, "a_B", "A_b", true}, + {asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent +} + +func TestFold(t *testing.T) { + for i, tt := range foldTests { + if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want { + t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want) + } + truth := strings.EqualFold(tt.s, tt.t) + if truth != tt.want { + t.Errorf("strings.EqualFold doesn't agree with case %d", i) + } + } +} + +func TestFoldAgainstUnicode(t *testing.T) { + const bufSize = 5 + buf1 := make([]byte, 0, bufSize) + buf2 := make([]byte, 0, bufSize) + var runes []rune + for i := 0x20; i <= 0x7f; i++ { + runes = append(runes, rune(i)) + } + runes = append(runes, kelvin, smallLongEss) + + funcs := []struct { + name string + fold func(s, t []byte) bool + letter bool // must be ASCII letter + simple bool // must be simple ASCII letter (not 'S' or 'K') + }{ + { + name: "equalFoldRight", + fold: equalFoldRight, + }, + { + name: "asciiEqualFold", + fold: asciiEqualFold, + simple: true, + }, + { + name: "simpleLetterEqualFold", + fold: simpleLetterEqualFold, + simple: true, + letter: true, + }, + } + + for _, ff := range funcs { + for _, r := range runes { + if r >= utf8.RuneSelf { + continue + } + if ff.letter && !isASCIILetter(byte(r)) { + continue + } + if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') { + continue + } + for _, r2 := range runes { + buf1 := append(buf1[:0], 'x') + buf2 := append(buf2[:0], 'x') + buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] + buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] + buf1 = append(buf1, 'x') + buf2 = append(buf2, 'x') + want := bytes.EqualFold(buf1, buf2) + if got := ff.fold(buf1, buf2); got != want { + t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) + } + } + } + } +} + +func isASCIILetter(b byte) bool { + return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') +} diff --git a/src/pkg/encoding/json/indent.go b/src/pkg/encoding/json/indent.go index 11ef709cc..e1bacafd6 100644 --- a/src/pkg/encoding/json/indent.go +++ b/src/pkg/encoding/json/indent.go @@ -69,8 +69,9 @@ func newline(dst *bytes.Buffer, prefix, indent string, depth int) { // Each element in a JSON object or array begins on a new, // indented line beginning with prefix followed by one or more // copies of indent according to the indentation nesting. -// The data appended to dst has no trailing newline, to make it easier -// to embed inside other formatted JSON data. +// The data appended to dst does not begin with the prefix nor +// any indentation, and has no trailing newline, to make it +// easier to embed inside other formatted JSON data. func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { origLen := dst.Len() var scan scanner diff --git a/src/pkg/encoding/json/scanner_test.go b/src/pkg/encoding/json/scanner_test.go index 90e45ff03..788034290 100644 --- a/src/pkg/encoding/json/scanner_test.go +++ b/src/pkg/encoding/json/scanner_test.go @@ -239,23 +239,16 @@ func trim(b []byte) []byte { var jsonBig []byte -const ( - big = 10000 - small = 100 -) - func initBig() { - n := big + n := 10000 if testing.Short() { - n = small + n = 100 } - if len(jsonBig) != n { - b, err := Marshal(genValue(n)) - if err != nil { - panic(err) - } - jsonBig = b + b, err := Marshal(genValue(n)) + if err != nil { + panic(err) } + jsonBig = b } func genValue(n int) interface{} { @@ -296,6 +289,9 @@ func genArray(n int) []interface{} { if f > n { f = n } + if f < 1 { + f = 1 + } x := make([]interface{}, f) for i := range x { x[i] = genValue(((i+1)*n)/f - (i*n)/f) diff --git a/src/pkg/encoding/json/stream.go b/src/pkg/encoding/json/stream.go index 1928abadb..1cb289fd8 100644 --- a/src/pkg/encoding/json/stream.go +++ b/src/pkg/encoding/json/stream.go @@ -148,7 +148,8 @@ func NewEncoder(w io.Writer) *Encoder { return &Encoder{w: w} } -// Encode writes the JSON encoding of v to the stream. +// Encode writes the JSON encoding of v to the stream, +// followed by a newline character. // // See the documentation for Marshal for details about the // conversion of Go values to JSON. @@ -173,7 +174,7 @@ func (enc *Encoder) Encode(v interface{}) error { if _, err = enc.w.Write(e.Bytes()); err != nil { enc.err = err } - putEncodeState(e) + encodeStatePool.Put(e) return err } |