diff options
Diffstat (limited to 'src/pkg/encoding')
39 files changed, 967 insertions, 122 deletions
diff --git a/src/pkg/encoding/ascii85/ascii85.go b/src/pkg/encoding/ascii85/ascii85.go index e2afc5871..60da304b5 100644 --- a/src/pkg/encoding/ascii85/ascii85.go +++ b/src/pkg/encoding/ascii85/ascii85.go @@ -281,6 +281,18 @@ func (d *decoder) Read(p []byte) (n int, err error) { d.nbuf = copy(d.buf[0:], d.buf[nsrc:d.nbuf]) continue // copy out and return } + if ndst == 0 && d.err == nil { + // Special case: input buffer is mostly filled with non-data bytes. + // Filter out such bytes to make room for more input. + off := 0 + for i := 0; i < d.nbuf; i++ { + if d.buf[i] > ' ' { + d.buf[off] = d.buf[i] + off++ + } + } + d.nbuf = off + } } // Out of input, out of decoded output. Check errors. diff --git a/src/pkg/encoding/ascii85/ascii85_test.go b/src/pkg/encoding/ascii85/ascii85_test.go index 42cf7e80e..aad199b4f 100644 --- a/src/pkg/encoding/ascii85/ascii85_test.go +++ b/src/pkg/encoding/ascii85/ascii85_test.go @@ -8,6 +8,7 @@ import ( "bytes" "io" "io/ioutil" + "strings" "testing" ) @@ -16,6 +17,11 @@ type testpair struct { } var pairs = []testpair{ + // Encode returns 0 when len(src) is 0 + { + "", + "", + }, // Wikipedia example { "Man is distinguished, not only by his reason, but by this singular passion from " + @@ -110,7 +116,7 @@ func TestDecode(t *testing.T) { func TestDecoder(t *testing.T) { for _, p := range pairs { - decoder := NewDecoder(bytes.NewBufferString(p.encoded)) + decoder := NewDecoder(strings.NewReader(p.encoded)) dbuf, err := ioutil.ReadAll(decoder) if err != nil { t.Fatal("Read failed", err) @@ -125,7 +131,7 @@ func TestDecoder(t *testing.T) { func TestDecoderBuffering(t *testing.T) { for bs := 1; bs <= 12; bs++ { - decoder := NewDecoder(bytes.NewBufferString(bigtest.encoded)) + decoder := NewDecoder(strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int for total = 0; total < len(bigtest.decoded); { @@ -191,3 +197,14 @@ func TestBig(t *testing.T) { t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) } } + +func TestDecoderInternalWhitespace(t *testing.T) { + s := strings.Repeat(" ", 2048) + "z" + decoded, err := ioutil.ReadAll(NewDecoder(strings.NewReader(s))) + if err != nil { + t.Errorf("Decode gave error %v", err) + } + if want := []byte("\000\000\000\000"); !bytes.Equal(want, decoded) { + t.Errorf("Decode failed: got %v, want %v", decoded, want) + } +} diff --git a/src/pkg/encoding/asn1/asn1.go b/src/pkg/encoding/asn1/asn1.go index 992356c26..ec7f91c1b 100644 --- a/src/pkg/encoding/asn1/asn1.go +++ b/src/pkg/encoding/asn1/asn1.go @@ -23,6 +23,7 @@ import ( "fmt" "math/big" "reflect" + "strconv" "time" ) @@ -197,6 +198,19 @@ func (oi ObjectIdentifier) Equal(other ObjectIdentifier) bool { return true } +func (oi ObjectIdentifier) String() string { + var s string + + for i, v := range oi { + if i > 0 { + s += "." + } + s += strconv.Itoa(v) + } + + return s +} + // parseObjectIdentifier parses an OBJECT IDENTIFIER from the given bytes and // returns it. An object identifier is a sequence of variable length integers // that are assigned in a hierarchy. @@ -451,11 +465,17 @@ func parseSequenceOf(bytes []byte, sliceType reflect.Type, elemType reflect.Type if err != nil { return } - // We pretend that GENERAL STRINGs are PRINTABLE STRINGs so - // that a sequence of them can be parsed into a []string. - if t.tag == tagGeneralString { + switch t.tag { + case tagIA5String, tagGeneralString, tagT61String, tagUTF8String: + // We pretend that various other string types are + // PRINTABLE STRINGs so that a sequence of them can be + // parsed into a []string. t.tag = tagPrintableString + case tagGeneralizedTime, tagUTCTime: + // Likewise, both time types are treated the same. + t.tag = tagUTCTime } + if t.class != classUniversal || t.isCompound != compoundType || t.tag != expectedTag { err = StructuralError{"sequence tag mismatch"} return @@ -632,6 +652,10 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam universalTag = tagGeneralizedTime } + if params.set { + universalTag = tagSet + } + expectedClass := classUniversal expectedTag := universalTag @@ -852,13 +876,20 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // // The following tags on struct fields have special meaning to Unmarshal: // -// optional marks the field as ASN.1 OPTIONAL -// [explicit] tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC -// default:x sets the default value for optional integer fields +// application specifies that a APPLICATION tag is used +// default:x sets the default value for optional integer fields +// explicit specifies that an additional, explicit tag wraps the implicit one +// optional marks the field as ASN.1 OPTIONAL +// set causes a SET, rather than a SEQUENCE type to be expected +// tag:x specifies the ASN.1 tag number; implies ASN.1 CONTEXT SPECIFIC // // If the type of the first field of a structure is RawContent then the raw // ASN1 contents of the struct will be stored in it. // +// If the type name of a slice element ends with "SET" then it's treated as if +// the "set" tag was set on it. This can be used with nested slices where a +// struct tag cannot be given. +// // Other ASN.1 types are not supported; if it encounters them, // Unmarshal returns a parse error. func Unmarshal(b []byte, val interface{}) (rest []byte, err error) { diff --git a/src/pkg/encoding/asn1/asn1_test.go b/src/pkg/encoding/asn1/asn1_test.go index f68804ebf..b553f78e0 100644 --- a/src/pkg/encoding/asn1/asn1_test.go +++ b/src/pkg/encoding/asn1/asn1_test.go @@ -6,6 +6,7 @@ package asn1 import ( "bytes" + "fmt" "math/big" "reflect" "testing" @@ -171,6 +172,12 @@ func TestBitStringAt(t *testing.T) { if bs.At(9) != 1 { t.Error("#4: Failed") } + if bs.At(-1) != 0 { + t.Error("#5: Failed") + } + if bs.At(17) != 0 { + t.Error("#6: Failed") + } } type bitStringRightAlignTest struct { @@ -225,6 +232,10 @@ func TestObjectIdentifier(t *testing.T) { } } } + + if s := ObjectIdentifier([]int{1, 2, 3, 4}).String(); s != "1.2.3.4" { + t.Errorf("bad ObjectIdentifier.String(). Got %s, want 1.2.3.4", s) + } } type timeTest struct { @@ -238,6 +249,7 @@ var utcTestData = []timeTest{ {"910506164540+0730", true, time.Date(1991, 05, 06, 16, 45, 40, 0, time.FixedZone("", 7*60*60+30*60))}, {"910506234540Z", true, time.Date(1991, 05, 06, 23, 45, 40, 0, time.UTC)}, {"9105062345Z", true, time.Date(1991, 05, 06, 23, 45, 0, 0, time.UTC)}, + {"5105062345Z", true, time.Date(1951, 05, 06, 23, 45, 0, 0, time.UTC)}, {"a10506234540Z", false, time.Time{}}, {"91a506234540Z", false, time.Time{}}, {"9105a6234540Z", false, time.Time{}}, @@ -389,6 +401,10 @@ type TestBigInt struct { X *big.Int } +type TestSet struct { + Ints []int `asn1:"set"` +} + var unmarshalTestData = []struct { in []byte out interface{} @@ -408,6 +424,7 @@ var unmarshalTestData = []struct { {[]byte{0x01, 0x01, 0xff}, newBool(true)}, {[]byte{0x30, 0x0b, 0x13, 0x03, 0x66, 0x6f, 0x6f, 0x02, 0x01, 0x22, 0x02, 0x01, 0x33}, &TestElementsAfterString{"foo", 0x22, 0x33}}, {[]byte{0x30, 0x05, 0x02, 0x03, 0x12, 0x34, 0x56}, &TestBigInt{big.NewInt(0x123456)}}, + {[]byte{0x30, 0x0b, 0x31, 0x09, 0x02, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x01, 0x03}, &TestSet{Ints: []int{1, 2, 3}}}, } func TestUnmarshal(t *testing.T) { @@ -509,6 +526,38 @@ func TestRawStructs(t *testing.T) { } } +type oiEqualTest struct { + first ObjectIdentifier + second ObjectIdentifier + same bool +} + +var oiEqualTests = []oiEqualTest{ + { + ObjectIdentifier{1, 2, 3}, + ObjectIdentifier{1, 2, 3}, + true, + }, + { + ObjectIdentifier{1}, + ObjectIdentifier{1, 2, 3}, + false, + }, + { + ObjectIdentifier{1, 2, 3}, + ObjectIdentifier{10, 11, 12}, + false, + }, +} + +func TestObjectIdentifierEqual(t *testing.T) { + for _, o := range oiEqualTests { + if s := o.first.Equal(o.second); s != o.same { + t.Errorf("ObjectIdentifier.Equal: got: %t want: %t", s, o.same) + } + } +} + var derEncodedSelfSignedCert = Certificate{ TBSCertificate: TBSCertificate{ Version: 0, @@ -737,3 +786,29 @@ var derEncodedPaypalNULCertBytes = []byte{ 0xc8, 0x64, 0x8c, 0xb5, 0x50, 0x23, 0x82, 0x6f, 0xdb, 0xb8, 0x22, 0x1c, 0x43, 0x96, 0x07, 0xa8, 0xbb, } + +var stringSliceTestData = [][]string{ + {"foo", "bar"}, + {"foo", "\\bar"}, + {"foo", "\"bar\""}, + {"foo", "åäö"}, +} + +func TestStringSlice(t *testing.T) { + for _, test := range stringSliceTestData { + bs, err := Marshal(test) + if err != nil { + t.Error(err) + } + + var res []string + _, err = Unmarshal(bs, &res) + if err != nil { + t.Error(err) + } + + if fmt.Sprintf("%v", res) != fmt.Sprintf("%v", test) { + t.Errorf("incorrect marshal/unmarshal; %v != %v", res, test) + } + } +} diff --git a/src/pkg/encoding/asn1/marshal.go b/src/pkg/encoding/asn1/marshal.go index ed17e41a5..e26fe59b3 100644 --- a/src/pkg/encoding/asn1/marshal.go +++ b/src/pkg/encoding/asn1/marshal.go @@ -295,8 +295,23 @@ func marshalTwoDigits(out *forkableWriter, v int) (err error) { return out.WriteByte(byte('0' + v%10)) } +func marshalFourDigits(out *forkableWriter, v int) (err error) { + var bytes [4]byte + for i := range bytes { + bytes[3-i] = '0' + byte(v%10) + v /= 10 + } + _, err = out.Write(bytes[:]) + return +} + +func outsideUTCRange(t time.Time) bool { + year := t.Year() + return year < 1950 || year >= 2050 +} + func marshalUTCTime(out *forkableWriter, t time.Time) (err error) { - year, month, day := t.Date() + year := t.Year() switch { case 1950 <= year && year < 2000: @@ -310,6 +325,24 @@ func marshalUTCTime(out *forkableWriter, t time.Time) (err error) { return } + return marshalTimeCommon(out, t) +} + +func marshalGeneralizedTime(out *forkableWriter, t time.Time) (err error) { + year := t.Year() + if year < 0 || year > 9999 { + return StructuralError{"cannot represent time as GeneralizedTime"} + } + if err = marshalFourDigits(out, year); err != nil { + return + } + + return marshalTimeCommon(out, t) +} + +func marshalTimeCommon(out *forkableWriter, t time.Time) (err error) { + _, month, day := t.Date() + err = marshalTwoDigits(out, int(month)) if err != nil { return @@ -378,7 +411,12 @@ func stripTagAndLength(in []byte) []byte { func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameters) (err error) { switch value.Type() { case timeType: - return marshalUTCTime(out, value.Interface().(time.Time)) + t := value.Interface().(time.Time) + if outsideUTCRange(t) { + return marshalGeneralizedTime(out, t) + } else { + return marshalUTCTime(out, t) + } case bitStringType: return marshalBitString(out, value.Interface().(BitString)) case objectIdentifierType: @@ -504,7 +542,8 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) return StructuralError{"explicit string type given to non-string member"} } - if tag == tagPrintableString { + switch tag { + case tagPrintableString: if params.stringType == 0 { // This is a string without an explicit string type. We'll use // a PrintableString if the character set in the string is @@ -521,6 +560,10 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) } else { tag = params.stringType } + case tagUTCTime: + if outsideUTCRange(v.Interface().(time.Time)) { + tag = tagGeneralizedTime + } } if params.set { @@ -568,6 +611,14 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) } // Marshal returns the ASN.1 encoding of val. +// +// In addition to the struct tags recognised by Unmarshal, the following can be +// used: +// +// ia5: causes strings to be marshaled as ASN.1, IA5 strings +// omitempty: causes empty slices to be skipped +// printable: causes strings to be marshaled as ASN.1, PrintableString strings. +// utf8: causes strings to be marshaled as ASN.1, UTF8 strings func Marshal(val interface{}) ([]byte, error) { var out bytes.Buffer v := reflect.ValueOf(val) diff --git a/src/pkg/encoding/asn1/marshal_test.go b/src/pkg/encoding/asn1/marshal_test.go index 763c86da2..a15acbed0 100644 --- a/src/pkg/encoding/asn1/marshal_test.go +++ b/src/pkg/encoding/asn1/marshal_test.go @@ -67,6 +67,14 @@ type marshalTest struct { out string // hex encoded } +func farFuture() time.Time { + t, err := time.Parse(time.RFC3339, "2100-04-05T12:01:01Z") + if err != nil { + panic(err) + } + return t +} + var marshalTests = []marshalTest{ {10, "02010a"}, {127, "02017f"}, @@ -83,6 +91,7 @@ var marshalTests = []marshalTest{ {time.Unix(0, 0).UTC(), "170d3730303130313030303030305a"}, {time.Unix(1258325776, 0).UTC(), "170d3039313131353232353631365a"}, {time.Unix(1258325776, 0).In(PST), "17113039313131353134353631362d30383030"}, + {farFuture(), "180f32313030303430353132303130315a"}, {BitString{[]byte{0x80}, 1}, "03020780"}, {BitString{[]byte{0x81, 0xf0}, 12}, "03030481f0"}, {ObjectIdentifier([]int{1, 2, 3, 4}), "06032a0304"}, diff --git a/src/pkg/encoding/base32/base32.go b/src/pkg/encoding/base32/base32.go index fe17b7322..d770de391 100644 --- a/src/pkg/encoding/base32/base32.go +++ b/src/pkg/encoding/base32/base32.go @@ -179,13 +179,11 @@ func (e *encoder) Write(p []byte) (n int, err error) { nn := len(e.out) / 8 * 5 if nn > len(p) { nn = len(p) + nn -= nn % 5 } - nn -= nn % 5 - if nn > 0 { - e.enc.Encode(e.out[0:], p[0:nn]) - if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { - return n, e.err - } + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/5*8]); e.err != nil { + return n, e.err } n += nn p = p[nn:] @@ -268,7 +266,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // 7, 5 and 2 are not valid padding lengths, and so 1, 3 and 6 are not // valid dlen values. See RFC 4648 Section 6 "Base 32 Encoding" listing // the five valid padding lengths, and Section 9 "Illustrations and - // Examples" for an illustration for how the the 1st, 3rd and 6th base32 + // Examples" for an illustration for how the 1st, 3rd and 6th base32 // src bytes do not yield enough information to decode a dst byte. if dlen == 1 || dlen == 3 || dlen == 6 { return n, false, CorruptInputError(olen - len(src) - 1) diff --git a/src/pkg/encoding/base32/base32_test.go b/src/pkg/encoding/base32/base32_test.go index 63298d1c9..f56b996fa 100644 --- a/src/pkg/encoding/base32/base32_test.go +++ b/src/pkg/encoding/base32/base32_test.go @@ -108,7 +108,7 @@ func TestDecode(t *testing.T) { func TestDecoder(t *testing.T) { for _, p := range pairs { - decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + decoder := NewDecoder(StdEncoding, strings.NewReader(p.encoded)) dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) count, err := decoder.Read(dbuf) if err != nil && err != io.EOF { @@ -125,7 +125,7 @@ func TestDecoder(t *testing.T) { func TestDecoderBuffering(t *testing.T) { for bs := 1; bs <= 12; bs++ { - decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int for total = 0; total < len(bigtest.decoded); { @@ -267,13 +267,13 @@ LNEBUWIIDFON2CA3DBMJXXE5LNFY== ====` encodedShort := strings.Replace(encoded, "\n", "", -1) - dec := NewDecoder(StdEncoding, bytes.NewBufferString(encoded)) + dec := NewDecoder(StdEncoding, strings.NewReader(encoded)) res1, err := ioutil.ReadAll(dec) if err != nil { t.Errorf("ReadAll failed: %v", err) } - dec = NewDecoder(StdEncoding, bytes.NewBufferString(encodedShort)) + dec = NewDecoder(StdEncoding, strings.NewReader(encodedShort)) var res2 []byte res2, err = ioutil.ReadAll(dec) if err != nil { diff --git a/src/pkg/encoding/base64/base64.go b/src/pkg/encoding/base64/base64.go index 85e398fd0..e38c26d0e 100644 --- a/src/pkg/encoding/base64/base64.go +++ b/src/pkg/encoding/base64/base64.go @@ -159,13 +159,11 @@ func (e *encoder) Write(p []byte) (n int, err error) { nn := len(e.out) / 4 * 3 if nn > len(p) { nn = len(p) + nn -= nn % 3 } - nn -= nn % 3 - if nn > 0 { - e.enc.Encode(e.out[0:], p[0:nn]) - if _, e.err = e.w.Write(e.out[0 : nn/3*4]); e.err != nil { - return n, e.err - } + e.enc.Encode(e.out[0:], p[0:nn]) + if _, e.err = e.w.Write(e.out[0 : nn/3*4]); e.err != nil { + return n, e.err } n += nn p = p[nn:] @@ -226,21 +224,33 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { var dbuf [4]byte dlen := 4 - for j := 0; j < 4; { + for j := range dbuf { if len(src) == 0 { return n, false, CorruptInputError(olen - len(src) - j) } in := src[0] src = src[1:] - if in == '=' && j >= 2 && len(src) < 4 { + if in == '=' { // We've reached the end and there's padding - if len(src)+j < 4-1 { - // not enough padding - return n, false, CorruptInputError(olen) - } - if len(src) > 0 && src[0] != '=' { + switch j { + case 0, 1: // incorrect padding return n, false, CorruptInputError(olen - len(src) - 1) + case 2: + // "==" is expected, the first "=" is already consumed. + if len(src) == 0 { + // not enough padding + return n, false, CorruptInputError(olen) + } + if src[0] != '=' { + // incorrect padding + return n, false, CorruptInputError(olen - len(src) - 1) + } + src = src[1:] + } + if len(src) > 0 { + // trailing garbage + err = CorruptInputError(olen - len(src)) } dlen, end = j, true break @@ -249,7 +259,6 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { if dbuf[j] == 0xFF { return n, false, CorruptInputError(olen - len(src) - 1) } - j++ } // Pack 4x 6-bit source blocks into 3 byte destination @@ -268,7 +277,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { n += dlen - 1 } - return n, end, nil + return n, end, err } // Decode decodes src using the encoding enc. It writes at most diff --git a/src/pkg/encoding/base64/base64_test.go b/src/pkg/encoding/base64/base64_test.go index 579591a88..a075194e0 100644 --- a/src/pkg/encoding/base64/base64_test.go +++ b/src/pkg/encoding/base64/base64_test.go @@ -9,6 +9,7 @@ import ( "errors" "io" "io/ioutil" + "reflect" "strings" "testing" "time" @@ -113,7 +114,7 @@ func TestDecode(t *testing.T) { func TestDecoder(t *testing.T) { for _, p := range pairs { - decoder := NewDecoder(StdEncoding, bytes.NewBufferString(p.encoded)) + decoder := NewDecoder(StdEncoding, strings.NewReader(p.encoded)) dbuf := make([]byte, StdEncoding.DecodedLen(len(p.encoded))) count, err := decoder.Read(dbuf) if err != nil && err != io.EOF { @@ -130,7 +131,7 @@ func TestDecoder(t *testing.T) { func TestDecoderBuffering(t *testing.T) { for bs := 1; bs <= 12; bs++ { - decoder := NewDecoder(StdEncoding, bytes.NewBufferString(bigtest.encoded)) + decoder := NewDecoder(StdEncoding, strings.NewReader(bigtest.encoded)) buf := make([]byte, len(bigtest.decoded)+12) var total int for total = 0; total < len(bigtest.decoded); { @@ -149,9 +150,13 @@ func TestDecodeCorrupt(t *testing.T) { }{ {"", -1}, {"!!!!", 0}, + {"====", 0}, {"x===", 1}, + {"=AAA", 0}, + {"A=AA", 1}, {"AA=A", 2}, - {"AAA=AAAA", 3}, + {"AA==A", 4}, + {"AAA=AAAA", 4}, {"AAAAA", 4}, {"AAAAAA", 4}, {"A=", 1}, @@ -161,6 +166,7 @@ func TestDecodeCorrupt(t *testing.T) { {"AAA=", -1}, {"AAAA", -1}, {"AAAAAA=", 7}, + {"YWJjZA=====", 8}, } for _, tc := range testCases { dbuf := make([]byte, StdEncoding.DecodedLen(len(tc.input))) @@ -308,13 +314,13 @@ bqbPb06551Y4 ` encodedShort := strings.Replace(encoded, "\n", "", -1) - dec := NewDecoder(StdEncoding, bytes.NewBufferString(encoded)) + dec := NewDecoder(StdEncoding, strings.NewReader(encoded)) res1, err := ioutil.ReadAll(dec) if err != nil { t.Errorf("ReadAll failed: %v", err) } - dec = NewDecoder(StdEncoding, bytes.NewBufferString(encodedShort)) + dec = NewDecoder(StdEncoding, strings.NewReader(encodedShort)) var res2 []byte res2, err = ioutil.ReadAll(dec) if err != nil { @@ -325,3 +331,14 @@ bqbPb06551Y4 t.Error("Decoded results not equal") } } + +func TestDecoderIssue7733(t *testing.T) { + s, err := StdEncoding.DecodeString("YWJjZA=====") + want := CorruptInputError(8) + if !reflect.DeepEqual(want, err) { + t.Errorf("Error = %v; want CorruptInputError(8)", err) + } + if string(s) != "abcd" { + t.Errorf("DecodeString = %q; want abcd", s) + } +} diff --git a/src/pkg/encoding/binary/binary.go b/src/pkg/encoding/binary/binary.go index f3466b9af..a5694876a 100644 --- a/src/pkg/encoding/binary/binary.go +++ b/src/pkg/encoding/binary/binary.go @@ -133,6 +133,7 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. +// When reading into a struct, all non-blank fields must be exported. func Read(r io.Reader, order ByteOrder, data interface{}) error { // Fast path for basic types and slices. if n := intDataSize(data); n != 0 { diff --git a/src/pkg/encoding/binary/binary_test.go b/src/pkg/encoding/binary/binary_test.go index fdfee7d87..c80c90383 100644 --- a/src/pkg/encoding/binary/binary_test.go +++ b/src/pkg/encoding/binary/binary_test.go @@ -111,7 +111,7 @@ func checkResult(t *testing.T, dir string, order ByteOrder, err error, have, wan func testRead(t *testing.T, order ByteOrder, b []byte, s1 interface{}) { var s2 Struct - err := Read(bytes.NewBuffer(b), order, &s2) + err := Read(bytes.NewReader(b), order, &s2) checkResult(t, "Read", order, err, s2, s1) } @@ -131,7 +131,7 @@ func TestBigEndianPtrWrite(t *testing.T) { testWrite(t, BigEndian, big, &s) } func TestReadSlice(t *testing.T) { slice := make([]int32, 2) - err := Read(bytes.NewBuffer(src), BigEndian, slice) + err := Read(bytes.NewReader(src), BigEndian, slice) checkResult(t, "ReadSlice", BigEndian, err, slice, res) } @@ -265,6 +265,30 @@ func TestBlankFields(t *testing.T) { } } +// An attempt to read into a struct with an unexported field will +// panic. This is probably not the best choice, but at this point +// anything else would be an API change. + +type Unexported struct { + a int32 +} + +func TestUnexportedRead(t *testing.T) { + var buf bytes.Buffer + u1 := Unexported{a: 1} + if err := Write(&buf, LittleEndian, &u1); err != nil { + t.Fatal(err) + } + + defer func() { + if recover() == nil { + t.Fatal("did not panic") + } + }() + var u2 Unexported + Read(&buf, LittleEndian, &u2) +} + type byteSliceReader struct { remain []byte } diff --git a/src/pkg/encoding/binary/varint_test.go b/src/pkg/encoding/binary/varint_test.go index 9476bd5fb..ca411ecbd 100644 --- a/src/pkg/encoding/binary/varint_test.go +++ b/src/pkg/encoding/binary/varint_test.go @@ -35,7 +35,7 @@ func testVarint(t *testing.T, x int64) { t.Errorf("Varint(%d): got n = %d; want %d", x, m, n) } - y, err := ReadVarint(bytes.NewBuffer(buf)) + y, err := ReadVarint(bytes.NewReader(buf)) if err != nil { t.Errorf("ReadVarint(%d): %s", x, err) } @@ -55,7 +55,7 @@ func testUvarint(t *testing.T, x uint64) { t.Errorf("Uvarint(%d): got n = %d; want %d", x, m, n) } - y, err := ReadUvarint(bytes.NewBuffer(buf)) + y, err := ReadUvarint(bytes.NewReader(buf)) if err != nil { t.Errorf("ReadUvarint(%d): %s", x, err) } @@ -114,7 +114,7 @@ func TestBufferTooSmall(t *testing.T) { t.Errorf("Uvarint(%v): got x = %d, n = %d", buf, x, n) } - x, err := ReadUvarint(bytes.NewBuffer(buf)) + x, err := ReadUvarint(bytes.NewReader(buf)) if x != 0 || err != io.EOF { t.Errorf("ReadUvarint(%v): got x = %d, err = %s", buf, x, err) } @@ -127,7 +127,7 @@ func testOverflow(t *testing.T, buf []byte, n0 int, err0 error) { t.Errorf("Uvarint(%v): got x = %d, n = %d; want 0, %d", buf, x, n, n0) } - x, err := ReadUvarint(bytes.NewBuffer(buf)) + x, err := ReadUvarint(bytes.NewReader(buf)) if x != 0 || err != err0 { t.Errorf("ReadUvarint(%v): got x = %d, err = %s; want 0, %s", buf, x, err, err0) } diff --git a/src/pkg/encoding/csv/reader.go b/src/pkg/encoding/csv/reader.go index b328dcc37..d9432954a 100644 --- a/src/pkg/encoding/csv/reader.go +++ b/src/pkg/encoding/csv/reader.go @@ -193,12 +193,6 @@ func (r *Reader) readRune() (rune, error) { return r1, err } -// unreadRune puts the last rune read from r back. -func (r *Reader) unreadRune() { - r.r.UnreadRune() - r.column-- -} - // skip reads runes up to and including the rune delim or until error. func (r *Reader) skip(delim rune) error { for { diff --git a/src/pkg/encoding/csv/writer_test.go b/src/pkg/encoding/csv/writer_test.go index 03ca6b093..22b740c07 100644 --- a/src/pkg/encoding/csv/writer_test.go +++ b/src/pkg/encoding/csv/writer_test.go @@ -26,6 +26,8 @@ var writeTests = []struct { {Input: [][]string{{"abc"}, {"def"}}, Output: "abc\ndef\n"}, {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\ndef\"\n"}, {Input: [][]string{{"abc\ndef"}}, Output: "\"abc\r\ndef\"\r\n", UseCRLF: true}, + {Input: [][]string{{"abc\rdef"}}, Output: "\"abcdef\"\r\n", UseCRLF: true}, + {Input: [][]string{{"abc\rdef"}}, Output: "\"abc\rdef\"\n", UseCRLF: false}, } func TestWrite(t *testing.T) { diff --git a/src/pkg/encoding/gob/codec_test.go b/src/pkg/encoding/gob/codec_test.go index b40f78360..fa57f3761 100644 --- a/src/pkg/encoding/gob/codec_test.go +++ b/src/pkg/encoding/gob/codec_test.go @@ -1364,11 +1364,7 @@ type DT struct { S []string } -func TestDebugStruct(t *testing.T) { - if debugFunc == nil { - return - } - Register(OnTheFly{}) +func newDT() DT { var dt DT dt.A = 17 dt.B = "hello" @@ -1379,6 +1375,15 @@ func TestDebugStruct(t *testing.T) { dt.M = map[string]int{"one": 1, "two": 2} dt.T = [3]int{11, 22, 33} dt.S = []string{"hi", "joe"} + return dt +} + +func TestDebugStruct(t *testing.T) { + if debugFunc == nil { + return + } + Register(OnTheFly{}) + dt := newDT() b := new(bytes.Buffer) err := NewEncoder(b).Encode(dt) if err != nil { @@ -1458,3 +1463,44 @@ func testFuzz(t *testing.T, seed int64, n int, input ...interface{}) { } } } + +// TestFuzzOneByte tries to decode corrupted input sequences +// and checks that no panic occurs. +func TestFuzzOneByte(t *testing.T) { + buf := new(bytes.Buffer) + Register(OnTheFly{}) + dt := newDT() + if err := NewEncoder(buf).Encode(dt); err != nil { + t.Fatal(err) + } + s := buf.String() + + indices := make([]int, 0, len(s)) + for i := 0; i < len(s); i++ { + switch i { + case 14, 167, 231, 265: // a slice length, corruptions are not handled yet. + continue + } + indices = append(indices, i) + } + if testing.Short() { + indices = []int{1, 111, 178} // known fixed panics + } + for _, i := range indices { + for j := 0; j < 256; j += 3 { + b := []byte(s) + b[i] ^= byte(j) + var e DT + func() { + defer func() { + if p := recover(); p != nil { + t.Errorf("crash for b[%d] ^= 0x%x", i, j) + panic(p) + } + }() + err := NewDecoder(bytes.NewReader(b)).Decode(&e) + _ = err + }() + } + } +} diff --git a/src/pkg/encoding/gob/decode.go b/src/pkg/encoding/gob/decode.go index 3e76f4c90..d8513148e 100644 --- a/src/pkg/encoding/gob/decode.go +++ b/src/pkg/encoding/gob/decode.go @@ -654,21 +654,20 @@ func (dec *Decoder) ignoreMap(state *decoderState, keyOp, elemOp decOp) { // decodeSlice decodes a slice and stores the slice header through p. // Slices are encoded as an unsigned length followed by the elements. -func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p uintptr, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl error) { +func (dec *Decoder) decodeSlice(atyp reflect.Type, state *decoderState, p unsafe.Pointer, elemOp decOp, elemWid uintptr, indir, elemIndir int, ovfl error) { nr := state.decodeUint() n := int(nr) if indir > 0 { - up := unsafe.Pointer(p) - if *(*unsafe.Pointer)(up) == nil { + if *(*unsafe.Pointer)(p) == nil { // Allocate the slice header. - *(*unsafe.Pointer)(up) = unsafe.Pointer(new([]unsafe.Pointer)) + *(*unsafe.Pointer)(p) = unsafe.Pointer(new([]unsafe.Pointer)) } - p = *(*uintptr)(up) + p = *(*unsafe.Pointer)(p) } // Allocate storage for the slice elements, that is, the underlying array, // if the existing slice does not have the capacity. // Always write a header at p. - hdrp := (*reflect.SliceHeader)(unsafe.Pointer(p)) + hdrp := (*reflect.SliceHeader)(p) if hdrp.Cap < n { hdrp.Data = reflect.MakeSlice(atyp, n, n).Pointer() hdrp.Cap = n @@ -686,7 +685,7 @@ func (dec *Decoder) ignoreSlice(state *decoderState, elemOp decOp) { // but first it checks that the assignment will succeed. func setInterfaceValue(ivalue reflect.Value, value reflect.Value) { if !value.Type().AssignableTo(ivalue.Type()) { - errorf("cannot assign value of type %s to %s", value.Type(), ivalue.Type()) + errorf("%s is not assignable to type %s", value.Type(), ivalue.Type()) } ivalue.Set(value) } @@ -702,6 +701,9 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p un if nr < 0 || nr > 1<<31 { // zero is permissible for anonymous types errorf("invalid type name length %d", nr) } + if nr > uint64(state.b.Len()) { + errorf("invalid type name length %d: exceeds input size", nr) + } b := make([]byte, nr) state.b.Read(b) name := string(b) @@ -887,7 +889,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg elemOp, elemIndir := dec.decOpFor(elemId, t.Elem(), name, inProgress) ovfl := overflow(name) op = func(i *decInstr, state *decoderState, p unsafe.Pointer) { - state.dec.decodeSlice(t, state, uintptr(p), *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) + state.dec.decodeSlice(t, state, p, *elemOp, t.Elem().Size(), i.indir, elemIndir, ovfl) } case reflect.Struct: @@ -1238,7 +1240,8 @@ func (dec *Decoder) decodeValue(wireId typeId, val reflect.Value) { } engine := *enginePtr if st := base; st.Kind() == reflect.Struct && ut.externalDec == 0 { - if engine.numInstr == 0 && st.NumField() > 0 && len(dec.wireType[wireId].StructT.Field) > 0 { + if engine.numInstr == 0 && st.NumField() > 0 && + dec.wireType[wireId] != nil && len(dec.wireType[wireId].StructT.Field) > 0 { name := base.Name() errorf("type mismatch: no fields matched compiling decoder for %s", name) } diff --git a/src/pkg/encoding/gob/decoder.go b/src/pkg/encoding/gob/decoder.go index 04f706ca5..3a769ec12 100644 --- a/src/pkg/encoding/gob/decoder.go +++ b/src/pkg/encoding/gob/decoder.go @@ -183,11 +183,13 @@ func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { return -1 } -// Decode reads the next value from the connection and stores +// Decode reads the next value from the input stream and stores // it in the data represented by the empty interface value. // If e is nil, the value will be discarded. Otherwise, // the value underlying e must be a pointer to the // correct type for the next data item received. +// If the input is at EOF, Decode returns io.EOF and +// does not modify e. func (dec *Decoder) Decode(e interface{}) error { if e == nil { return dec.DecodeValue(reflect.Value{}) @@ -202,10 +204,12 @@ func (dec *Decoder) Decode(e interface{}) error { return dec.DecodeValue(value) } -// DecodeValue reads the next value from the connection. +// DecodeValue reads the next value from the input stream. // If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value. // Otherwise, it stores the value into v. In that case, v must represent // a non-nil pointer to data or be an assignable reflect.Value (v.CanSet()) +// If the input is at EOF, DecodeValue returns io.EOF and +// does not modify e. func (dec *Decoder) DecodeValue(v reflect.Value) error { if v.IsValid() { if v.Kind() == reflect.Ptr && !v.IsNil() { diff --git a/src/pkg/encoding/gob/encode.go b/src/pkg/encoding/gob/encode.go index d158b6442..7831c02d1 100644 --- a/src/pkg/encoding/gob/encode.go +++ b/src/pkg/encoding/gob/encode.go @@ -491,7 +491,7 @@ func isZero(val reflect.Value) bool { return !val.Bool() case reflect.Complex64, reflect.Complex128: return val.Complex() == 0 - case reflect.Chan, reflect.Func, reflect.Ptr: + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Ptr: return val.IsNil() case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return val.Int() == 0 diff --git a/src/pkg/encoding/gob/encoder_test.go b/src/pkg/encoding/gob/encoder_test.go index 4ecf51d12..6445ce100 100644 --- a/src/pkg/encoding/gob/encoder_test.go +++ b/src/pkg/encoding/gob/encoder_test.go @@ -129,6 +129,8 @@ func TestBadData(t *testing.T) { corruptDataCheck("", io.EOF, t) corruptDataCheck("\x7Fhi", io.ErrUnexpectedEOF, t) corruptDataCheck("\x03now is the time for all good men", errBadType, t) + // issue 6323. + corruptDataCheck("\x04\x24foo", errRange, t) } // Types not supported at top level by the Encoder. @@ -630,7 +632,7 @@ func TestSliceReusesMemory(t *testing.T) { // Used to crash: negative count in recvMessage. func TestBadCount(t *testing.T) { b := []byte{0xfb, 0xa5, 0x82, 0x2f, 0xca, 0x1} - if err := NewDecoder(bytes.NewBuffer(b)).Decode(nil); err == nil { + if err := NewDecoder(bytes.NewReader(b)).Decode(nil); err == nil { t.Error("expected error from bad count") } else if err.Error() != errBadCount.Error() { t.Error("expected bad count error; got", err) diff --git a/src/pkg/encoding/gob/gobencdec_test.go b/src/pkg/encoding/gob/gobencdec_test.go index 0193e2b67..157b7723a 100644 --- a/src/pkg/encoding/gob/gobencdec_test.go +++ b/src/pkg/encoding/gob/gobencdec_test.go @@ -705,13 +705,14 @@ func TestGobEncoderExtraIndirect(t *testing.T) { } // Another bug: this caused a crash with the new Go1 Time type. -// We throw in a gob-encoding array, to test another case of isZero - +// We throw in a gob-encoding array, to test another case of isZero, +// and a struct containing an nil interface, to test a third. type isZeroBug struct { T time.Time S string I int A isZeroBugArray + F isZeroBugInterface } type isZeroBugArray [2]uint8 @@ -731,8 +732,20 @@ func (a *isZeroBugArray) GobDecode(data []byte) error { return nil } +type isZeroBugInterface struct { + I interface{} +} + +func (i isZeroBugInterface) GobEncode() (b []byte, e error) { + return []byte{}, nil +} + +func (i *isZeroBugInterface) GobDecode(data []byte) error { + return nil +} + func TestGobEncodeIsZero(t *testing.T) { - x := isZeroBug{time.Now(), "hello", -55, isZeroBugArray{1, 2}} + x := isZeroBug{time.Now(), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} b := new(bytes.Buffer) enc := NewEncoder(b) err := enc.Encode(x) diff --git a/src/pkg/encoding/hex/hex.go b/src/pkg/encoding/hex/hex.go index 167d00e03..d1fc7024a 100644 --- a/src/pkg/encoding/hex/hex.go +++ b/src/pkg/encoding/hex/hex.go @@ -146,6 +146,9 @@ func (h *dumper) Write(data []byte) (n int, err error) { h.buf[12] = ' ' h.buf[13] = ' ' _, err = h.w.Write(h.buf[4:]) + if err != nil { + return + } } Encode(h.buf[:], data[i:i+1]) h.buf[2] = ' ' diff --git a/src/pkg/encoding/hex/hex_test.go b/src/pkg/encoding/hex/hex_test.go index 356f590f0..b969636cd 100644 --- a/src/pkg/encoding/hex/hex_test.go +++ b/src/pkg/encoding/hex/hex_test.go @@ -38,7 +38,10 @@ func TestEncode(t *testing.T) { } func TestDecode(t *testing.T) { - for i, test := range encDecTests { + // Case for decoding uppercase hex characters, since + // Encode always uses lowercase. + decTests := append(encDecTests, encDecTest{"F8F9FAFBFCFDFEFF", []byte{0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}}) + for i, test := range decTests { dst := make([]byte, DecodedLen(len(test.enc))) n, err := Decode(dst, []byte(test.enc)) if err != nil { @@ -79,6 +82,7 @@ type errTest struct { var errTests = []errTest{ {"0", "encoding/hex: odd length hex string"}, {"0g", "encoding/hex: invalid byte: U+0067 'g'"}, + {"00gg", "encoding/hex: invalid byte: U+0067 'g'"}, {"0\x01", "encoding/hex: invalid byte: U+0001"}, } diff --git a/src/pkg/encoding/json/decode.go b/src/pkg/encoding/json/decode.go index 458fb39ec..af1c908ad 100644 --- a/src/pkg/encoding/json/decode.go +++ b/src/pkg/encoding/json/decode.go @@ -8,6 +8,7 @@ package json import ( + "bytes" "encoding" "encoding/base64" "errors" @@ -15,7 +16,6 @@ import ( "reflect" "runtime" "strconv" - "strings" "unicode" "unicode/utf16" "unicode/utf8" @@ -54,6 +54,11 @@ import ( // If no more serious errors are encountered, Unmarshal returns // an UnmarshalTypeError describing the earliest such error. // +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// // When unmarshaling quoted strings, invalid UTF-8 or // invalid UTF-16 surrogate pairs are not treated as an error. // Instead, they are replaced by the Unicode replacement @@ -500,11 +505,11 @@ func (d *decodeState) object(v reflect.Value) { d.error(errPhase) } - // Read string key. + // Read key. start := d.off - 1 op = d.scanWhile(scanContinue) item := d.data[start : d.off-1] - key, ok := unquote(item) + key, ok := unquoteBytes(item) if !ok { d.error(errPhase) } @@ -526,11 +531,11 @@ func (d *decodeState) object(v reflect.Value) { fields := cachedTypeFields(v.Type()) for i := range fields { ff := &fields[i] - if ff.name == key { + if bytes.Equal(ff.nameBytes, key) { f = ff break } - if f == nil && strings.EqualFold(ff.name, key) { + if f == nil && ff.equalFold(ff.nameBytes, key) { f = ff } } @@ -561,6 +566,7 @@ func (d *decodeState) object(v reflect.Value) { if destring { d.value(reflect.ValueOf(&d.tempstr)) d.literalStore([]byte(d.tempstr), subv, true) + d.tempstr = "" // Zero scratch space for successive values. } else { d.value(subv) } diff --git a/src/pkg/encoding/json/decode_test.go b/src/pkg/encoding/json/decode_test.go index 22c5f89f7..238a87fd6 100644 --- a/src/pkg/encoding/json/decode_test.go +++ b/src/pkg/encoding/json/decode_test.go @@ -1060,6 +1060,21 @@ func TestEmptyString(t *testing.T) { } } +// Test that the returned error is non-nil when trying to unmarshal null string into int, for successive ,string option +// Issue 7046 +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + } + data := []byte(`{"A": "1", "B": null}`) + var s T + err := Unmarshal(data, &s) + if err == nil { + t.Fatalf("expected error; got %v", s) + } +} + func intp(x int) *int { p := new(int) *p = x @@ -1110,8 +1125,8 @@ func TestInterfaceSet(t *testing.T) { // Issue 2540 func TestUnmarshalNulls(t *testing.T) { jsonData := []byte(`{ - "Bool" : null, - "Int" : null, + "Bool" : null, + "Int" : null, "Int8" : null, "Int16" : null, "Int32" : null, @@ -1316,3 +1331,26 @@ func TestPrefilled(t *testing.T) { } } } + +var invalidUnmarshalTests = []struct { + v interface{} + want string +}{ + {nil, "json: Unmarshal(nil)"}, + {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {(*int)(nil), "json: Unmarshal(nil *int)"}, +} + +func TestInvalidUnmarshal(t *testing.T) { + buf := []byte(`{"a":"1"}`) + for _, tt := range invalidUnmarshalTests { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Errorf("Unmarshal expecting error, got nil") + continue + } + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } + } +} diff --git a/src/pkg/encoding/json/encode.go b/src/pkg/encoding/json/encode.go index 7d6c71d7a..741ddd89c 100644 --- a/src/pkg/encoding/json/encode.go +++ b/src/pkg/encoding/json/encode.go @@ -44,6 +44,7 @@ import ( // if an invalid UTF-8 sequence is encountered. // The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" // to keep some browsers from misinterpreting JSON output as HTML. +// Ampersand "&" is also escaped to "\u0026" for the same reason. // // Array and slice values encode as JSON arrays, except that // []byte encodes as a base64-encoded string, and a nil slice @@ -241,24 +242,15 @@ type encodeState struct { scratch [64]byte } -// TODO(bradfitz): use a sync.Cache here -var encodeStatePool = make(chan *encodeState, 8) +var encodeStatePool sync.Pool func newEncodeState() *encodeState { - select { - case e := <-encodeStatePool: + if v := encodeStatePool.Get(); v != nil { + e := v.(*encodeState) e.Reset() return e - default: - return new(encodeState) - } -} - -func putEncodeState(e *encodeState) { - select { - case encodeStatePool <- e: - default: } + return new(encodeState) } func (e *encodeState) marshal(v interface{}) (err error) { @@ -813,7 +805,7 @@ func (e *encodeState) string(s string) (int, error) { e.WriteByte('r') default: // This encodes bytes < 0x20 except for \n and \r, - // as well as < and >. The latter are escaped because they + // as well as <, > and &. The latter are escaped because they // can lead to security holes when user-controlled strings // are rendered into JSON and served to some browsers. e.WriteString(`\u00`) @@ -936,6 +928,9 @@ func (e *encodeState) stringBytes(s []byte) (int, error) { // A field represents a single field found in a struct. type field struct { name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + tag bool index []int typ reflect.Type @@ -943,6 +938,12 @@ type field struct { quoted bool } +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + // byName sorts field by name, breaking ties with depth, // then breaking ties with "name came from json tag", then // breaking ties with index sequence. @@ -1042,8 +1043,14 @@ func typeFields(t reflect.Type) []field { if name == "" { name = sf.Name } - fields = append(fields, field{name, tagged, index, ft, - opts.Contains("omitempty"), opts.Contains("string")}) + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. @@ -1057,7 +1064,7 @@ func typeFields(t reflect.Type) []field { // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { - next = append(next, field{name: ft.Name(), index: index, typ: ft}) + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) } } } diff --git a/src/pkg/encoding/json/encode_test.go b/src/pkg/encoding/json/encode_test.go index 9395db7cb..2e89a78eb 100644 --- a/src/pkg/encoding/json/encode_test.go +++ b/src/pkg/encoding/json/encode_test.go @@ -25,13 +25,30 @@ type Optionals struct { Mr map[string]interface{} `json:"mr"` Mo map[string]interface{} `json:",omitempty"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitempty"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitempty"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitempty"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitempty"` } var optionalsExpected = `{ "sr": "", "omitempty": 0, "slr": null, - "mr": {} + "mr": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "sto": {} }` func TestOmitEmpty(t *testing.T) { @@ -76,7 +93,7 @@ func TestStringTag(t *testing.T) { // Verify that it round-trips. var s2 StringTag - err = NewDecoder(bytes.NewBuffer(got)).Decode(&s2) + err = NewDecoder(bytes.NewReader(got)).Decode(&s2) if err != nil { t.Fatalf("Decode: %v", err) } @@ -425,3 +442,13 @@ func TestIssue6458(t *testing.T) { t.Errorf("Marshal(x) = %#q; want %#q", b, want) } } + +func TestHTMLEscape(t *testing.T) { + var b, want bytes.Buffer + m := `{"M":"<html>foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `</html>"}` + want.Write([]byte(`{"M":"\u003chtml\u003efoo \u0026\u2028 \u2029\u003c/html\u003e"}`)) + HTMLEscape(&b, []byte(m)) + if !bytes.Equal(b.Bytes(), want.Bytes()) { + t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) + } +} diff --git a/src/pkg/encoding/json/example_test.go b/src/pkg/encoding/json/example_test.go index ea0bc149c..ca4e5ae68 100644 --- a/src/pkg/encoding/json/example_test.go +++ b/src/pkg/encoding/json/example_test.go @@ -5,6 +5,7 @@ package json_test import ( + "bytes" "encoding/json" "fmt" "io" @@ -127,3 +128,34 @@ func ExampleRawMessage() { // YCbCr &{255 0 -10} // RGB &{98 218 255} } + +func ExampleIndent() { + type Road struct { + Name string + Number int + } + roads := []Road{ + {"Diamond Fork", 29}, + {"Sheep Creek", 51}, + } + + b, err := json.Marshal(roads) + if err != nil { + log.Fatal(err) + } + + var out bytes.Buffer + json.Indent(&out, b, "=", "\t") + out.WriteTo(os.Stdout) + // Output: + // [ + // = { + // = "Name": "Diamond Fork", + // = "Number": 29 + // = }, + // = { + // = "Name": "Sheep Creek", + // = "Number": 51 + // = } + // =] +} diff --git a/src/pkg/encoding/json/fold.go b/src/pkg/encoding/json/fold.go new file mode 100644 index 000000000..d6f77c93e --- /dev/null +++ b/src/pkg/encoding/json/fold.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "unicode/utf8" +) + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} diff --git a/src/pkg/encoding/json/fold_test.go b/src/pkg/encoding/json/fold_test.go new file mode 100644 index 000000000..9fb94646a --- /dev/null +++ b/src/pkg/encoding/json/fold_test.go @@ -0,0 +1,116 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "strings" + "testing" + "unicode/utf8" +) + +var foldTests = []struct { + fn func(s, t []byte) bool + s, t string + want bool +}{ + {equalFoldRight, "", "", true}, + {equalFoldRight, "a", "a", true}, + {equalFoldRight, "", "a", false}, + {equalFoldRight, "a", "", false}, + {equalFoldRight, "a", "A", true}, + {equalFoldRight, "AB", "ab", true}, + {equalFoldRight, "AB", "ac", false}, + {equalFoldRight, "sbkKc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKK", false}, + {equalFoldRight, "e", "é", false}, + {equalFoldRight, "s", "S", true}, + + {simpleLetterEqualFold, "", "", true}, + {simpleLetterEqualFold, "abc", "abc", true}, + {simpleLetterEqualFold, "abc", "ABC", true}, + {simpleLetterEqualFold, "abc", "ABCD", false}, + {simpleLetterEqualFold, "abc", "xxx", false}, + + {asciiEqualFold, "a_B", "A_b", true}, + {asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent +} + +func TestFold(t *testing.T) { + for i, tt := range foldTests { + if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want { + t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want) + } + truth := strings.EqualFold(tt.s, tt.t) + if truth != tt.want { + t.Errorf("strings.EqualFold doesn't agree with case %d", i) + } + } +} + +func TestFoldAgainstUnicode(t *testing.T) { + const bufSize = 5 + buf1 := make([]byte, 0, bufSize) + buf2 := make([]byte, 0, bufSize) + var runes []rune + for i := 0x20; i <= 0x7f; i++ { + runes = append(runes, rune(i)) + } + runes = append(runes, kelvin, smallLongEss) + + funcs := []struct { + name string + fold func(s, t []byte) bool + letter bool // must be ASCII letter + simple bool // must be simple ASCII letter (not 'S' or 'K') + }{ + { + name: "equalFoldRight", + fold: equalFoldRight, + }, + { + name: "asciiEqualFold", + fold: asciiEqualFold, + simple: true, + }, + { + name: "simpleLetterEqualFold", + fold: simpleLetterEqualFold, + simple: true, + letter: true, + }, + } + + for _, ff := range funcs { + for _, r := range runes { + if r >= utf8.RuneSelf { + continue + } + if ff.letter && !isASCIILetter(byte(r)) { + continue + } + if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') { + continue + } + for _, r2 := range runes { + buf1 := append(buf1[:0], 'x') + buf2 := append(buf2[:0], 'x') + buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] + buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] + buf1 = append(buf1, 'x') + buf2 = append(buf2, 'x') + want := bytes.EqualFold(buf1, buf2) + if got := ff.fold(buf1, buf2); got != want { + t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) + } + } + } + } +} + +func isASCIILetter(b byte) bool { + return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') +} diff --git a/src/pkg/encoding/json/indent.go b/src/pkg/encoding/json/indent.go index 11ef709cc..e1bacafd6 100644 --- a/src/pkg/encoding/json/indent.go +++ b/src/pkg/encoding/json/indent.go @@ -69,8 +69,9 @@ func newline(dst *bytes.Buffer, prefix, indent string, depth int) { // Each element in a JSON object or array begins on a new, // indented line beginning with prefix followed by one or more // copies of indent according to the indentation nesting. -// The data appended to dst has no trailing newline, to make it easier -// to embed inside other formatted JSON data. +// The data appended to dst does not begin with the prefix nor +// any indentation, and has no trailing newline, to make it +// easier to embed inside other formatted JSON data. func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { origLen := dst.Len() var scan scanner diff --git a/src/pkg/encoding/json/scanner_test.go b/src/pkg/encoding/json/scanner_test.go index 90e45ff03..788034290 100644 --- a/src/pkg/encoding/json/scanner_test.go +++ b/src/pkg/encoding/json/scanner_test.go @@ -239,23 +239,16 @@ func trim(b []byte) []byte { var jsonBig []byte -const ( - big = 10000 - small = 100 -) - func initBig() { - n := big + n := 10000 if testing.Short() { - n = small + n = 100 } - if len(jsonBig) != n { - b, err := Marshal(genValue(n)) - if err != nil { - panic(err) - } - jsonBig = b + b, err := Marshal(genValue(n)) + if err != nil { + panic(err) } + jsonBig = b } func genValue(n int) interface{} { @@ -296,6 +289,9 @@ func genArray(n int) []interface{} { if f > n { f = n } + if f < 1 { + f = 1 + } x := make([]interface{}, f) for i := range x { x[i] = genValue(((i+1)*n)/f - (i*n)/f) diff --git a/src/pkg/encoding/json/stream.go b/src/pkg/encoding/json/stream.go index 1928abadb..1cb289fd8 100644 --- a/src/pkg/encoding/json/stream.go +++ b/src/pkg/encoding/json/stream.go @@ -148,7 +148,8 @@ func NewEncoder(w io.Writer) *Encoder { return &Encoder{w: w} } -// Encode writes the JSON encoding of v to the stream. +// Encode writes the JSON encoding of v to the stream, +// followed by a newline character. // // See the documentation for Marshal for details about the // conversion of Go values to JSON. @@ -173,7 +174,7 @@ func (enc *Encoder) Encode(v interface{}) error { if _, err = enc.w.Write(e.Bytes()); err != nil { enc.err = err } - putEncodeState(e) + encodeStatePool.Put(e) return err } diff --git a/src/pkg/encoding/xml/marshal.go b/src/pkg/encoding/xml/marshal.go index d9522e0b3..8c6342013 100644 --- a/src/pkg/encoding/xml/marshal.go +++ b/src/pkg/encoding/xml/marshal.go @@ -184,10 +184,12 @@ var ( // EncodeToken does not call Flush, because usually it is part of a larger operation // such as Encode or EncodeElement (or a custom Marshaler's MarshalXML invoked // during those), and those will call Flush when finished. -// // Callers that create an Encoder and then invoke EncodeToken directly, without // using Encode or EncodeElement, need to call Flush when finished to ensure // that the XML is written to the underlying writer. +// +// EncodeToken allows writing a ProcInst with Target set to "xml" only as the first token +// in the stream. func (enc *Encoder) EncodeToken(t Token) error { p := &enc.p switch t := t.(type) { @@ -210,7 +212,12 @@ func (enc *Encoder) EncodeToken(t Token) error { p.WriteString("-->") return p.cachedWriteError() case ProcInst: - if t.Target == "xml" || !isNameString(t.Target) { + // First token to be encoded which is also a ProcInst with target of xml + // is the xml declaration. The only ProcInst where target of xml is allowed. + if t.Target == "xml" && p.Buffered() != 0 { + return fmt.Errorf("xml: EncodeToken of ProcInst xml target only valid for xml declaration, first token encoded") + } + if !isNameString(t.Target) { return fmt.Errorf("xml: EncodeToken of ProcInst with invalid Target") } if bytes.Contains(t.Inst, endProcInst) { diff --git a/src/pkg/encoding/xml/marshal_test.go b/src/pkg/encoding/xml/marshal_test.go index d34118a3d..14f73a75d 100644 --- a/src/pkg/encoding/xml/marshal_test.go +++ b/src/pkg/encoding/xml/marshal_test.go @@ -314,6 +314,31 @@ type MarshalerStruct struct { Foo MyMarshalerAttrTest `xml:",attr"` } +type InnerStruct struct { + XMLName Name `xml:"testns outer"` +} + +type OuterStruct struct { + InnerStruct + IntAttr int `xml:"int,attr"` +} + +type OuterNamedStruct struct { + InnerStruct + XMLName Name `xml:"outerns test"` + IntAttr int `xml:"int,attr"` +} + +type OuterNamedOrderedStruct struct { + XMLName Name `xml:"outerns test"` + InnerStruct + IntAttr int `xml:"int,attr"` +} + +type OuterOuterStruct struct { + OuterStruct +} + func ifaceptr(x interface{}) interface{} { return &x } @@ -883,6 +908,22 @@ var marshalTests = []struct { ExpectXML: `<MarshalerStruct Foo="hello world"></MarshalerStruct>`, Value: &MarshalerStruct{}, }, + { + ExpectXML: `<outer xmlns="testns" int="10"></outer>`, + Value: &OuterStruct{IntAttr: 10}, + }, + { + ExpectXML: `<test xmlns="outerns" int="10"></test>`, + Value: &OuterNamedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, + }, + { + ExpectXML: `<test xmlns="outerns" int="10"></test>`, + Value: &OuterNamedOrderedStruct{XMLName: Name{Space: "outerns", Local: "test"}, IntAttr: 10}, + }, + { + ExpectXML: `<outer xmlns="testns" int="10"></outer>`, + Value: &OuterOuterStruct{OuterStruct{IntAttr: 10}}, + }, } func TestMarshal(t *testing.T) { @@ -1149,3 +1190,77 @@ func TestStructPointerMarshal(t *testing.T) { t.Fatal(err) } } + +var encodeTokenTests = []struct { + tok Token + want string + ok bool +}{ + {StartElement{Name{"space", "local"}, nil}, "<local xmlns=\"space\">", true}, + {StartElement{Name{"space", ""}, nil}, "", false}, + {EndElement{Name{"space", ""}}, "", false}, + {CharData("foo"), "foo", true}, + {Comment("foo"), "<!--foo-->", true}, + {Comment("foo-->"), "", false}, + {ProcInst{"Target", []byte("Instruction")}, "<?Target Instruction?>", true}, + {ProcInst{"", []byte("Instruction")}, "", false}, + {ProcInst{"Target", []byte("Instruction?>")}, "", false}, + {Directive("foo"), "<!foo>", true}, + {Directive("foo>"), "", false}, +} + +func TestEncodeToken(t *testing.T) { + for _, tt := range encodeTokenTests { + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.EncodeToken(tt.tok) + switch { + case !tt.ok && err == nil: + t.Errorf("enc.EncodeToken(%#v): expected error; got none", tt.tok) + case tt.ok && err != nil: + t.Fatalf("enc.EncodeToken: %v", err) + case !tt.ok && err != nil: + // expected error, got one + } + if err := enc.Flush(); err != nil { + t.Fatalf("enc.EncodeToken: %v", err) + } + if got := buf.String(); got != tt.want { + t.Errorf("enc.EncodeToken = %s; want: %s", got, tt.want) + } + } +} + +func TestProcInstEncodeToken(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err != nil { + t.Fatalf("enc.EncodeToken: expected to be able to encode xml target ProcInst as first token, %s", err) + } + + if err := enc.EncodeToken(ProcInst{"Target", []byte("Instruction")}); err != nil { + t.Fatalf("enc.EncodeToken: expected to be able to add non-xml target ProcInst") + } + + if err := enc.EncodeToken(ProcInst{"xml", []byte("Instruction")}); err == nil { + t.Fatalf("enc.EncodeToken: expected to not be allowed to encode xml target ProcInst when not first token") + } +} + +func TestDecodeEncode(t *testing.T) { + var in, out bytes.Buffer + in.WriteString(`<?xml version="1.0" encoding="UTF-8"?> +<?Target Instruction?> +<root> +</root> +`) + dec := NewDecoder(&in) + enc := NewEncoder(&out) + for tok, err := dec.Token(); err == nil; tok, err = dec.Token() { + err = enc.EncodeToken(tok) + if err != nil { + t.Fatalf("enc.EncodeToken: Unable to encode token (%#v), %v", tok, err) + } + } +} diff --git a/src/pkg/encoding/xml/read.go b/src/pkg/encoding/xml/read.go index 8890508f8..75b9f2ba1 100644 --- a/src/pkg/encoding/xml/read.go +++ b/src/pkg/encoding/xml/read.go @@ -112,7 +112,7 @@ import ( // to a freshly allocated value and then mapping the element to that value. // func Unmarshal(data []byte, v interface{}) error { - return NewDecoder(bytes.NewBuffer(data)).Decode(v) + return NewDecoder(bytes.NewReader(data)).Decode(v) } // Decode works like xml.Unmarshal, except it reads the decoder @@ -284,6 +284,15 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } } + // Load value from interface, but only if the result will be + // usefully addressable. + if val.Kind() == reflect.Interface && !val.IsNil() { + e := val.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() { + val = e + } + } + if val.Kind() == reflect.Ptr { if val.IsNil() { val.Set(reflect.New(val.Type().Elem())) diff --git a/src/pkg/encoding/xml/read_test.go b/src/pkg/encoding/xml/read_test.go index 1404c900f..01f55d0dd 100644 --- a/src/pkg/encoding/xml/read_test.go +++ b/src/pkg/encoding/xml/read_test.go @@ -685,3 +685,30 @@ func TestUnmarshaler(t *testing.T) { t.Errorf("m=%#+v\n", m) } } + +type Pea struct { + Cotelydon string +} + +type Pod struct { + Pea interface{} `xml:"Pea"` +} + +// https://code.google.com/p/go/issues/detail?id=6836 +func TestUnmarshalIntoInterface(t *testing.T) { + pod := new(Pod) + pod.Pea = new(Pea) + xml := `<Pod><Pea><Cotelydon>Green stuff</Cotelydon></Pea></Pod>` + err := Unmarshal([]byte(xml), pod) + if err != nil { + t.Fatalf("failed to unmarshal %q: %v", xml, err) + } + pea, ok := pod.Pea.(*Pea) + if !ok { + t.Fatalf("unmarshalled into wrong type: have %T want *Pea", pod.Pea) + } + have, want := pea.Cotelydon, "Green stuff" + if have != want { + t.Errorf("failed to unmarshal into interface, have %q want %q", have, want) + } +} diff --git a/src/pkg/encoding/xml/typeinfo.go b/src/pkg/encoding/xml/typeinfo.go index 83e65402c..22248d20a 100644 --- a/src/pkg/encoding/xml/typeinfo.go +++ b/src/pkg/encoding/xml/typeinfo.go @@ -75,6 +75,9 @@ func getTypeInfo(typ reflect.Type) (*typeInfo, error) { if err != nil { return nil, err } + if tinfo.xmlname == nil { + tinfo.xmlname = inner.xmlname + } for _, finfo := range inner.fields { finfo.idx = append([]int{i}, finfo.idx...) if err := addFieldInfo(typ, tinfo, &finfo); err != nil { diff --git a/src/pkg/encoding/xml/xml.go b/src/pkg/encoding/xml/xml.go index 5b9d67002..b473cb845 100644 --- a/src/pkg/encoding/xml/xml.go +++ b/src/pkg/encoding/xml/xml.go @@ -200,6 +200,8 @@ type Decoder struct { } // NewDecoder creates a new XML parser reading from r. +// If r does not implement io.ByteReader, NewDecoder will +// do its own buffering. func NewDecoder(r io.Reader) *Decoder { d := &Decoder{ ns: make(map[string]string), |