diff options
author | Tianon Gravi <admwiggin@gmail.com> | 2015-01-15 11:54:00 -0700 |
---|---|---|
committer | Tianon Gravi <admwiggin@gmail.com> | 2015-01-15 11:54:00 -0700 |
commit | f154da9e12608589e8d5f0508f908a0c3e88a1bb (patch) | |
tree | f8255d51e10c6f1e0ed69702200b966c9556a431 /src/bytes | |
parent | 8d8329ed5dfb9622c82a9fbec6fd99a580f9c9f6 (diff) | |
download | golang-upstream/1.4.tar.gz |
Imported Upstream version 1.4upstream/1.4
Diffstat (limited to 'src/bytes')
-rw-r--r-- | src/bytes/buffer.go | 412 | ||||
-rw-r--r-- | src/bytes/buffer_test.go | 527 | ||||
-rw-r--r-- | src/bytes/bytes.go | 703 | ||||
-rw-r--r-- | src/bytes/bytes_decl.go | 24 | ||||
-rw-r--r-- | src/bytes/bytes_test.go | 1240 | ||||
-rw-r--r-- | src/bytes/compare_test.go | 208 | ||||
-rw-r--r-- | src/bytes/equal_test.go | 47 | ||||
-rw-r--r-- | src/bytes/example_test.go | 85 | ||||
-rw-r--r-- | src/bytes/export_test.go | 13 | ||||
-rw-r--r-- | src/bytes/reader.go | 144 | ||||
-rw-r--r-- | src/bytes/reader_test.go | 246 |
11 files changed, 3649 insertions, 0 deletions
diff --git a/src/bytes/buffer.go b/src/bytes/buffer.go new file mode 100644 index 000000000..46ca1d5ad --- /dev/null +++ b/src/bytes/buffer.go @@ -0,0 +1,412 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Simple byte buffer for marshaling data. + +import ( + "errors" + "io" + "unicode/utf8" +) + +// A Buffer is a variable-sized buffer of bytes with Read and Write methods. +// The zero value for Buffer is an empty buffer ready to use. +type Buffer struct { + buf []byte // contents are the bytes buf[off : len(buf)] + off int // read at &buf[off], write at &buf[len(buf)] + runeBytes [utf8.UTFMax]byte // avoid allocation of slice on each WriteByte or Rune + bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation. + lastRead readOp // last read operation, so that Unread* can work correctly. +} + +// The readOp constants describe the last action performed on +// the buffer, so that UnreadRune and UnreadByte can +// check for invalid usage. +type readOp int + +const ( + opInvalid readOp = iota // Non-read operation. + opReadRune // Read rune. + opRead // Any other read operation. +) + +// ErrTooLarge is passed to panic if memory cannot be allocated to store data in a buffer. +var ErrTooLarge = errors.New("bytes.Buffer: too large") + +// Bytes returns a slice of the contents of the unread portion of the buffer; +// len(b.Bytes()) == b.Len(). If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } + +// String returns the contents of the unread portion of the buffer +// as a string. If the Buffer is a nil pointer, it returns "<nil>". +func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "<nil>" + } + return string(b.buf[b.off:]) +} + +// Len returns the number of bytes of the unread portion of the buffer; +// b.Len() == len(b.Bytes()). +func (b *Buffer) Len() int { return len(b.buf) - b.off } + +// Truncate discards all but the first n unread bytes from the buffer. +// It panics if n is negative or greater than the length of the buffer. +func (b *Buffer) Truncate(n int) { + b.lastRead = opInvalid + switch { + case n < 0 || n > b.Len(): + panic("bytes.Buffer: truncation out of range") + case n == 0: + // Reuse buffer space. + b.off = 0 + } + b.buf = b.buf[0 : b.off+n] +} + +// Reset resets the buffer so it has no content. +// b.Reset() is the same as b.Truncate(0). +func (b *Buffer) Reset() { b.Truncate(0) } + +// grow grows the buffer to guarantee space for n more bytes. +// It returns the index where bytes should be written. +// If the buffer can't grow it will panic with ErrTooLarge. +func (b *Buffer) grow(n int) int { + m := b.Len() + // If buffer is empty, reset to recover space. + if m == 0 && b.off != 0 { + b.Truncate(0) + } + if len(b.buf)+n > cap(b.buf) { + var buf []byte + if b.buf == nil && n <= len(b.bootstrap) { + buf = b.bootstrap[0:] + } else if m+n <= cap(b.buf)/2 { + // We can slide things down instead of allocating a new + // slice. We only need m+n <= cap(b.buf) to slide, but + // we instead let capacity get twice as large so we + // don't spend all our time copying. + copy(b.buf[:], b.buf[b.off:]) + buf = b.buf[:m] + } else { + // not enough space anywhere + buf = makeSlice(2*cap(b.buf) + n) + copy(buf, b.buf[b.off:]) + } + b.buf = buf + b.off = 0 + } + b.buf = b.buf[0 : b.off+m+n] + return b.off + m +} + +// Grow grows the buffer's capacity, if necessary, to guarantee space for +// another n bytes. After Grow(n), at least n bytes can be written to the +// buffer without another allocation. +// If n is negative, Grow will panic. +// If the buffer can't grow it will panic with ErrTooLarge. +func (b *Buffer) Grow(n int) { + if n < 0 { + panic("bytes.Buffer.Grow: negative count") + } + m := b.grow(n) + b.buf = b.buf[0:m] +} + +// Write appends the contents of p to the buffer, growing the buffer as +// needed. The return value n is the length of p; err is always nil. If the +// buffer becomes too large, Write will panic with ErrTooLarge. +func (b *Buffer) Write(p []byte) (n int, err error) { + b.lastRead = opInvalid + m := b.grow(len(p)) + return copy(b.buf[m:], p), nil +} + +// WriteString appends the contents of s to the buffer, growing the buffer as +// needed. The return value n is the length of s; err is always nil. If the +// buffer becomes too large, WriteString will panic with ErrTooLarge. +func (b *Buffer) WriteString(s string) (n int, err error) { + b.lastRead = opInvalid + m := b.grow(len(s)) + return copy(b.buf[m:], s), nil +} + +// MinRead is the minimum slice size passed to a Read call by +// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond +// what is required to hold the contents of r, ReadFrom will not grow the +// underlying buffer. +const MinRead = 512 + +// ReadFrom reads data from r until EOF and appends it to the buffer, growing +// the buffer as needed. The return value n is the number of bytes read. Any +// error except io.EOF encountered during the read is also returned. If the +// buffer becomes too large, ReadFrom will panic with ErrTooLarge. +func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) { + b.lastRead = opInvalid + // If buffer is empty, reset to recover space. + if b.off >= len(b.buf) { + b.Truncate(0) + } + for { + if free := cap(b.buf) - len(b.buf); free < MinRead { + // not enough space at end + newBuf := b.buf + if b.off+free < MinRead { + // not enough space using beginning of buffer; + // double buffer capacity + newBuf = makeSlice(2*cap(b.buf) + MinRead) + } + copy(newBuf, b.buf[b.off:]) + b.buf = newBuf[:len(b.buf)-b.off] + b.off = 0 + } + m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) + b.buf = b.buf[0 : len(b.buf)+m] + n += int64(m) + if e == io.EOF { + break + } + if e != nil { + return n, e + } + } + return n, nil // err is EOF, so return nil explicitly +} + +// makeSlice allocates a slice of size n. If the allocation fails, it panics +// with ErrTooLarge. +func makeSlice(n int) []byte { + // If the make fails, give a known error. + defer func() { + if recover() != nil { + panic(ErrTooLarge) + } + }() + return make([]byte, n) +} + +// WriteTo writes data to w until the buffer is drained or an error occurs. +// The return value n is the number of bytes written; it always fits into an +// int, but it is int64 to match the io.WriterTo interface. Any error +// encountered during the write is also returned. +func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) { + b.lastRead = opInvalid + if b.off < len(b.buf) { + nBytes := b.Len() + m, e := w.Write(b.buf[b.off:]) + if m > nBytes { + panic("bytes.Buffer.WriteTo: invalid Write count") + } + b.off += m + n = int64(m) + if e != nil { + return n, e + } + // all bytes should have been written, by definition of + // Write method in io.Writer + if m != nBytes { + return n, io.ErrShortWrite + } + } + // Buffer is now empty; reset. + b.Truncate(0) + return +} + +// WriteByte appends the byte c to the buffer, growing the buffer as needed. +// The returned error is always nil, but is included to match bufio.Writer's +// WriteByte. If the buffer becomes too large, WriteByte will panic with +// ErrTooLarge. +func (b *Buffer) WriteByte(c byte) error { + b.lastRead = opInvalid + m := b.grow(1) + b.buf[m] = c + return nil +} + +// WriteRune appends the UTF-8 encoding of Unicode code point r to the +// buffer, returning its length and an error, which is always nil but is +// included to match bufio.Writer's WriteRune. The buffer is grown as needed; +// if it becomes too large, WriteRune will panic with ErrTooLarge. +func (b *Buffer) WriteRune(r rune) (n int, err error) { + if r < utf8.RuneSelf { + b.WriteByte(byte(r)) + return 1, nil + } + n = utf8.EncodeRune(b.runeBytes[0:], r) + b.Write(b.runeBytes[0:n]) + return n, nil +} + +// Read reads the next len(p) bytes from the buffer or until the buffer +// is drained. The return value n is the number of bytes read. If the +// buffer has no data to return, err is io.EOF (unless len(p) is zero); +// otherwise it is nil. +func (b *Buffer) Read(p []byte) (n int, err error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + if len(p) == 0 { + return + } + return 0, io.EOF + } + n = copy(p, b.buf[b.off:]) + b.off += n + if n > 0 { + b.lastRead = opRead + } + return +} + +// Next returns a slice containing the next n bytes from the buffer, +// advancing the buffer as if the bytes had been returned by Read. +// If there are fewer than n bytes in the buffer, Next returns the entire buffer. +// The slice is only valid until the next call to a read or write method. +func (b *Buffer) Next(n int) []byte { + b.lastRead = opInvalid + m := b.Len() + if n > m { + n = m + } + data := b.buf[b.off : b.off+n] + b.off += n + if n > 0 { + b.lastRead = opRead + } + return data +} + +// ReadByte reads and returns the next byte from the buffer. +// If no byte is available, it returns error io.EOF. +func (b *Buffer) ReadByte() (c byte, err error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, io.EOF + } + c = b.buf[b.off] + b.off++ + b.lastRead = opRead + return c, nil +} + +// ReadRune reads and returns the next UTF-8-encoded +// Unicode code point from the buffer. +// If no bytes are available, the error returned is io.EOF. +// If the bytes are an erroneous UTF-8 encoding, it +// consumes one byte and returns U+FFFD, 1. +func (b *Buffer) ReadRune() (r rune, size int, err error) { + b.lastRead = opInvalid + if b.off >= len(b.buf) { + // Buffer is empty, reset to recover space. + b.Truncate(0) + return 0, 0, io.EOF + } + b.lastRead = opReadRune + c := b.buf[b.off] + if c < utf8.RuneSelf { + b.off++ + return rune(c), 1, nil + } + r, n := utf8.DecodeRune(b.buf[b.off:]) + b.off += n + return r, n, nil +} + +// UnreadRune unreads the last rune returned by ReadRune. +// If the most recent read or write operation on the buffer was +// not a ReadRune, UnreadRune returns an error. (In this regard +// it is stricter than UnreadByte, which will unread the last byte +// from any read operation.) +func (b *Buffer) UnreadRune() error { + if b.lastRead != opReadRune { + return errors.New("bytes.Buffer: UnreadRune: previous operation was not ReadRune") + } + b.lastRead = opInvalid + if b.off > 0 { + _, n := utf8.DecodeLastRune(b.buf[0:b.off]) + b.off -= n + } + return nil +} + +// UnreadByte unreads the last byte returned by the most recent +// read operation. If write has happened since the last read, UnreadByte +// returns an error. +func (b *Buffer) UnreadByte() error { + if b.lastRead != opReadRune && b.lastRead != opRead { + return errors.New("bytes.Buffer: UnreadByte: previous operation was not a read") + } + b.lastRead = opInvalid + if b.off > 0 { + b.off-- + } + return nil +} + +// ReadBytes reads until the first occurrence of delim in the input, +// returning a slice containing the data up to and including the delimiter. +// If ReadBytes encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often io.EOF). +// ReadBytes returns err != nil if and only if the returned data does not end in +// delim. +func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) { + slice, err := b.readSlice(delim) + // return a copy of slice. The buffer's backing array may + // be overwritten by later calls. + line = append(line, slice...) + return +} + +// readSlice is like ReadBytes but returns a reference to internal buffer data. +func (b *Buffer) readSlice(delim byte) (line []byte, err error) { + i := IndexByte(b.buf[b.off:], delim) + end := b.off + i + 1 + if i < 0 { + end = len(b.buf) + err = io.EOF + } + line = b.buf[b.off:end] + b.off = end + b.lastRead = opRead + return line, err +} + +// ReadString reads until the first occurrence of delim in the input, +// returning a string containing the data up to and including the delimiter. +// If ReadString encounters an error before finding a delimiter, +// it returns the data read before the error and the error itself (often io.EOF). +// ReadString returns err != nil if and only if the returned data does not end +// in delim. +func (b *Buffer) ReadString(delim byte) (line string, err error) { + slice, err := b.readSlice(delim) + return string(slice), err +} + +// NewBuffer creates and initializes a new Buffer using buf as its initial +// contents. It is intended to prepare a Buffer to read existing data. It +// can also be used to size the internal buffer for writing. To do that, +// buf should have the desired capacity but a length of zero. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// sufficient to initialize a Buffer. +func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } + +// NewBufferString creates and initializes a new Buffer using string s as its +// initial contents. It is intended to prepare a buffer to read an existing +// string. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// sufficient to initialize a Buffer. +func NewBufferString(s string) *Buffer { + return &Buffer{buf: []byte(s)} +} diff --git a/src/bytes/buffer_test.go b/src/bytes/buffer_test.go new file mode 100644 index 000000000..75145b05e --- /dev/null +++ b/src/bytes/buffer_test.go @@ -0,0 +1,527 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "io" + "math/rand" + "runtime" + "testing" + "unicode/utf8" +) + +const N = 10000 // make this bigger for a larger (and slower) test +var data string // test data for write tests +var testBytes []byte // test data; same as data but as a slice. + +func init() { + testBytes = make([]byte, N) + for i := 0; i < N; i++ { + testBytes[i] = 'a' + byte(i%26) + } + data = string(testBytes) +} + +// Verify that contents of buf match the string s. +func check(t *testing.T, testname string, buf *Buffer, s string) { + bytes := buf.Bytes() + str := buf.String() + if buf.Len() != len(bytes) { + t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes)) + } + + if buf.Len() != len(str) { + t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str)) + } + + if buf.Len() != len(s) { + t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s)) + } + + if string(bytes) != s { + t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s) + } +} + +// Fill buf through n writes of string fus. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillString(t *testing.T, testname string, buf *Buffer, s string, n int, fus string) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.WriteString(fus) + if m != len(fus) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fus)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += fus + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +// Fill buf through n writes of byte slice fub. +// The initial contents of buf corresponds to the string s; +// the result is the final contents of buf returned as a string. +func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string { + check(t, testname+" (fill 1)", buf, s) + for ; n > 0; n-- { + m, err := buf.Write(fub) + if m != len(fub) { + t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub)) + } + if err != nil { + t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) + } + s += string(fub) + check(t, testname+" (fill 4)", buf, s) + } + return s +} + +func TestNewBuffer(t *testing.T) { + buf := NewBuffer(testBytes) + check(t, "NewBuffer", buf, data) +} + +func TestNewBufferString(t *testing.T) { + buf := NewBufferString(data) + check(t, "NewBufferString", buf, data) +} + +// Empty buf through repeated reads into fub. +// The initial contents of buf corresponds to the string s. +func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) { + check(t, testname+" (empty 1)", buf, s) + + for { + n, err := buf.Read(fub) + if n == 0 { + break + } + if err != nil { + t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err) + } + s = s[n:] + check(t, testname+" (empty 3)", buf, s) + } + + check(t, testname+" (empty 4)", buf, "") +} + +func TestBasicOperations(t *testing.T) { + var buf Buffer + + for i := 0; i < 5; i++ { + check(t, "TestBasicOperations (1)", &buf, "") + + buf.Reset() + check(t, "TestBasicOperations (2)", &buf, "") + + buf.Truncate(0) + check(t, "TestBasicOperations (3)", &buf, "") + + n, err := buf.Write([]byte(data[0:1])) + if n != 1 { + t.Errorf("wrote 1 byte, but n == %d", n) + } + if err != nil { + t.Errorf("err should always be nil, but err == %s", err) + } + check(t, "TestBasicOperations (4)", &buf, "a") + + buf.WriteByte(data[1]) + check(t, "TestBasicOperations (5)", &buf, "ab") + + n, err = buf.Write([]byte(data[2:26])) + if n != 24 { + t.Errorf("wrote 25 bytes, but n == %d", n) + } + check(t, "TestBasicOperations (6)", &buf, string(data[0:26])) + + buf.Truncate(26) + check(t, "TestBasicOperations (7)", &buf, string(data[0:26])) + + buf.Truncate(20) + check(t, "TestBasicOperations (8)", &buf, string(data[0:20])) + + empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5)) + empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100)) + + buf.WriteByte(data[1]) + c, err := buf.ReadByte() + if err != nil { + t.Error("ReadByte unexpected eof") + } + if c != data[1] { + t.Errorf("ReadByte wrong value c=%v", c) + } + c, err = buf.ReadByte() + if err == nil { + t.Error("ReadByte unexpected not eof") + } + } +} + +func TestLargeStringWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillString(t, "TestLargeWrites (1)", &buf, "", 5, data) + empty(t, "TestLargeStringWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeStringWrites (3)", &buf, "") +} + +func TestLargeByteWrites(t *testing.T) { + var buf Buffer + limit := 30 + if testing.Short() { + limit = 9 + } + for i := 3; i < limit; i += 3 { + s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes) + empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i)) + } + check(t, "TestLargeByteWrites (3)", &buf, "") +} + +func TestLargeStringReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillString(t, "TestLargeReads (1)", &buf, "", 5, data[0:len(data)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeStringReads (3)", &buf, "") +} + +func TestLargeByteReads(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) + empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) + } + check(t, "TestLargeByteReads (3)", &buf, "") +} + +func TestMixedReadsAndWrites(t *testing.T) { + var buf Buffer + s := "" + for i := 0; i < 50; i++ { + wlen := rand.Intn(len(data)) + if i%2 == 0 { + s = fillString(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, data[0:wlen]) + } else { + s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen]) + } + + rlen := rand.Intn(len(data)) + fub := make([]byte, rlen) + n, _ := buf.Read(fub) + s = s[n:] + } + empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len())) +} + +func TestNil(t *testing.T) { + var b *Buffer + if b.String() != "<nil>" { + t.Errorf("expected <nil>; got %q", b.String()) + } +} + +func TestReadFrom(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) + var b Buffer + b.ReadFrom(&buf) + empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) + } +} + +func TestWriteTo(t *testing.T) { + var buf Buffer + for i := 3; i < 30; i += 3 { + s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) + var b Buffer + buf.WriteTo(&b) + empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data))) + } +} + +func TestRuneIO(t *testing.T) { + const NRune = 1000 + // Built a test slice while we write the data + b := make([]byte, utf8.UTFMax*NRune) + var buf Buffer + n := 0 + for r := rune(0); r < NRune; r++ { + size := utf8.EncodeRune(b[n:], r) + nbytes, err := buf.WriteRune(r) + if err != nil { + t.Fatalf("WriteRune(%U) error: %s", r, err) + } + if nbytes != size { + t.Fatalf("WriteRune(%U) expected %d, got %d", r, size, nbytes) + } + n += size + } + b = b[0:n] + + // Check the resulting bytes + if !Equal(buf.Bytes(), b) { + t.Fatalf("incorrect result from WriteRune: %q not %q", buf.Bytes(), b) + } + + p := make([]byte, utf8.UTFMax) + // Read it back with ReadRune + for r := rune(0); r < NRune; r++ { + size := utf8.EncodeRune(p, r) + nr, nbytes, err := buf.ReadRune() + if nr != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) got %U,%d not %U,%d (err=%s)", r, nr, nbytes, r, size, err) + } + } + + // Check that UnreadRune works + buf.Reset() + buf.Write(b) + for r := rune(0); r < NRune; r++ { + r1, size, _ := buf.ReadRune() + if err := buf.UnreadRune(); err != nil { + t.Fatalf("UnreadRune(%U) got error %q", r, err) + } + r2, nbytes, err := buf.ReadRune() + if r1 != r2 || r1 != r || nbytes != size || err != nil { + t.Fatalf("ReadRune(%U) after UnreadRune got %U,%d not %U,%d (err=%s)", r, r2, nbytes, r, size, err) + } + } +} + +func TestNext(t *testing.T) { + b := []byte{0, 1, 2, 3, 4} + tmp := make([]byte, 5) + for i := 0; i <= 5; i++ { + for j := i; j <= 5; j++ { + for k := 0; k <= 6; k++ { + // 0 <= i <= j <= 5; 0 <= k <= 6 + // Check that if we start with a buffer + // of length j at offset i and ask for + // Next(k), we get the right bytes. + buf := NewBuffer(b[0:j]) + n, _ := buf.Read(tmp[0:i]) + if n != i { + t.Fatalf("Read %d returned %d", i, n) + } + bb := buf.Next(k) + want := k + if want > j-i { + want = j - i + } + if len(bb) != want { + t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb)) + } + for l, v := range bb { + if v != byte(l+i) { + t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i) + } + } + } + } + } +} + +var readBytesTests = []struct { + buffer string + delim byte + expected []string + err error +}{ + {"", 0, []string{""}, io.EOF}, + {"a\x00", 0, []string{"a\x00"}, nil}, + {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil}, + {"hello\x01world", 1, []string{"hello\x01"}, nil}, + {"foo\nbar", 0, []string{"foo\nbar"}, io.EOF}, + {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil}, + {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, io.EOF}, +} + +func TestReadBytes(t *testing.T) { + for _, test := range readBytesTests { + buf := NewBufferString(test.buffer) + var err error + for _, expected := range test.expected { + var bytes []byte + bytes, err = buf.ReadBytes(test.delim) + if string(bytes) != expected { + t.Errorf("expected %q, got %q", expected, bytes) + } + if err != nil { + break + } + } + if err != test.err { + t.Errorf("expected error %v, got %v", test.err, err) + } + } +} + +func TestReadString(t *testing.T) { + for _, test := range readBytesTests { + buf := NewBufferString(test.buffer) + var err error + for _, expected := range test.expected { + var s string + s, err = buf.ReadString(test.delim) + if s != expected { + t.Errorf("expected %q, got %q", expected, s) + } + if err != nil { + break + } + } + if err != test.err { + t.Errorf("expected error %v, got %v", test.err, err) + } + } +} + +func BenchmarkReadString(b *testing.B) { + const n = 32 << 10 + + data := make([]byte, n) + data[n-1] = 'x' + b.SetBytes(int64(n)) + for i := 0; i < b.N; i++ { + buf := NewBuffer(data) + _, err := buf.ReadString('x') + if err != nil { + b.Fatal(err) + } + } +} + +func TestGrow(t *testing.T) { + x := []byte{'x'} + y := []byte{'y'} + tmp := make([]byte, 72) + for _, startLen := range []int{0, 100, 1000, 10000, 100000} { + xBytes := Repeat(x, startLen) + for _, growLen := range []int{0, 100, 1000, 10000, 100000} { + buf := NewBuffer(xBytes) + // If we read, this affects buf.off, which is good to test. + readBytes, _ := buf.Read(tmp) + buf.Grow(growLen) + yBytes := Repeat(y, growLen) + // Check no allocation occurs in write, as long as we're single-threaded. + var m1, m2 runtime.MemStats + runtime.ReadMemStats(&m1) + buf.Write(yBytes) + runtime.ReadMemStats(&m2) + if runtime.GOMAXPROCS(-1) == 1 && m1.Mallocs != m2.Mallocs { + t.Errorf("allocation occurred during write") + } + // Check that buffer has correct data. + if !Equal(buf.Bytes()[0:startLen-readBytes], xBytes[readBytes:]) { + t.Errorf("bad initial data at %d %d", startLen, growLen) + } + if !Equal(buf.Bytes()[startLen-readBytes:startLen-readBytes+growLen], yBytes) { + t.Errorf("bad written data at %d %d", startLen, growLen) + } + } + } +} + +// Was a bug: used to give EOF reading empty slice at EOF. +func TestReadEmptyAtEOF(t *testing.T) { + b := new(Buffer) + slice := make([]byte, 0) + n, err := b.Read(slice) + if err != nil { + t.Errorf("read error: %v", err) + } + if n != 0 { + t.Errorf("wrong count; got %d want 0", n) + } +} + +func TestUnreadByte(t *testing.T) { + b := new(Buffer) + b.WriteString("abcdefghijklmnopqrstuvwxyz") + + _, err := b.ReadBytes('m') + if err != nil { + t.Fatalf("ReadBytes: %v", err) + } + + err = b.UnreadByte() + if err != nil { + t.Fatalf("UnreadByte: %v", err) + } + c, err := b.ReadByte() + if err != nil { + t.Fatalf("ReadByte: %v", err) + } + if c != 'm' { + t.Errorf("ReadByte = %q; want %q", c, 'm') + } +} + +// Tests that we occasionally compact. Issue 5154. +func TestBufferGrowth(t *testing.T) { + var b Buffer + buf := make([]byte, 1024) + b.Write(buf[0:1]) + var cap0 int + for i := 0; i < 5<<10; i++ { + b.Write(buf) + b.Read(buf) + if i == 0 { + cap0 = b.Cap() + } + } + cap1 := b.Cap() + // (*Buffer).grow allows for 2x capacity slop before sliding, + // so set our error threshold at 3x. + if cap1 > cap0*3 { + t.Errorf("buffer cap = %d; too big (grew from %d)", cap1, cap0) + } +} + +// From Issue 5154. +func BenchmarkBufferNotEmptyWriteRead(b *testing.B) { + buf := make([]byte, 1024) + for i := 0; i < b.N; i++ { + var b Buffer + b.Write(buf[0:1]) + for i := 0; i < 5<<10; i++ { + b.Write(buf) + b.Read(buf) + } + } +} + +// Check that we don't compact too often. From Issue 5154. +func BenchmarkBufferFullSmallReads(b *testing.B) { + buf := make([]byte, 1024) + for i := 0; i < b.N; i++ { + var b Buffer + b.Write(buf) + for b.Len()+20 < b.Cap() { + b.Write(buf[:10]) + } + for i := 0; i < 5<<10; i++ { + b.Read(buf[:1]) + b.Write(buf[:1]) + } + } +} diff --git a/src/bytes/bytes.go b/src/bytes/bytes.go new file mode 100644 index 000000000..7634707b3 --- /dev/null +++ b/src/bytes/bytes.go @@ -0,0 +1,703 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bytes implements functions for the manipulation of byte slices. +// It is analogous to the facilities of the strings package. +package bytes + +import ( + "unicode" + "unicode/utf8" +) + +func equalPortable(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i, c := range a { + if c != b[i] { + return false + } + } + return true +} + +// explode splits s into a slice of UTF-8 sequences, one per Unicode character (still slices of bytes), +// up to a maximum of n byte slices. Invalid UTF-8 sequences are chopped into individual bytes. +func explode(s []byte, n int) [][]byte { + if n <= 0 { + n = len(s) + } + a := make([][]byte, n) + var size int + na := 0 + for len(s) > 0 { + if na+1 >= n { + a[na] = s + na++ + break + } + _, size = utf8.DecodeRune(s) + a[na] = s[0:size] + s = s[size:] + na++ + } + return a[0:na] +} + +// Count counts the number of non-overlapping instances of sep in s. +func Count(s, sep []byte) int { + n := len(sep) + if n == 0 { + return utf8.RuneCount(s) + 1 + } + if n > len(s) { + return 0 + } + count := 0 + c := sep[0] + i := 0 + t := s[:len(s)-n+1] + for i < len(t) { + if t[i] != c { + o := IndexByte(t[i:], c) + if o < 0 { + break + } + i += o + } + if n == 1 || Equal(s[i:i+n], sep) { + count++ + i += n + continue + } + i++ + } + return count +} + +// Contains reports whether subslice is within b. +func Contains(b, subslice []byte) bool { + return Index(b, subslice) != -1 +} + +// Index returns the index of the first instance of sep in s, or -1 if sep is not present in s. +func Index(s, sep []byte) int { + n := len(sep) + if n == 0 { + return 0 + } + if n > len(s) { + return -1 + } + c := sep[0] + if n == 1 { + return IndexByte(s, c) + } + i := 0 + t := s[:len(s)-n+1] + for i < len(t) { + if t[i] != c { + o := IndexByte(t[i:], c) + if o < 0 { + break + } + i += o + } + if Equal(s[i:i+n], sep) { + return i + } + i++ + } + return -1 +} + +func indexBytePortable(s []byte, c byte) int { + for i, b := range s { + if b == c { + return i + } + } + return -1 +} + +// LastIndex returns the index of the last instance of sep in s, or -1 if sep is not present in s. +func LastIndex(s, sep []byte) int { + n := len(sep) + if n == 0 { + return len(s) + } + c := sep[0] + for i := len(s) - n; i >= 0; i-- { + if s[i] == c && (n == 1 || Equal(s[i:i+n], sep)) { + return i + } + } + return -1 +} + +// IndexRune interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of the given rune. +// It returns -1 if rune is not present in s. +func IndexRune(s []byte, r rune) int { + for i := 0; i < len(s); { + r1, size := utf8.DecodeRune(s[i:]) + if r == r1 { + return i + } + i += size + } + return -1 +} + +// IndexAny interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index of the first occurrence in s of any of the Unicode +// code points in chars. It returns -1 if chars is empty or if there is no code +// point in common. +func IndexAny(s []byte, chars string) int { + if len(chars) > 0 { + var r rune + var width int + for i := 0; i < len(s); i += width { + r = rune(s[i]) + if r < utf8.RuneSelf { + width = 1 + } else { + r, width = utf8.DecodeRune(s[i:]) + } + for _, ch := range chars { + if r == ch { + return i + } + } + } + } + return -1 +} + +// LastIndexAny interprets s as a sequence of UTF-8-encoded Unicode code +// points. It returns the byte index of the last occurrence in s of any of +// the Unicode code points in chars. It returns -1 if chars is empty or if +// there is no code point in common. +func LastIndexAny(s []byte, chars string) int { + if len(chars) > 0 { + for i := len(s); i > 0; { + r, size := utf8.DecodeLastRune(s[0:i]) + i -= size + for _, ch := range chars { + if r == ch { + return i + } + } + } + } + return -1 +} + +// Generic split: splits after each instance of sep, +// including sepSave bytes of sep in the subslices. +func genSplit(s, sep []byte, sepSave, n int) [][]byte { + if n == 0 { + return nil + } + if len(sep) == 0 { + return explode(s, n) + } + if n < 0 { + n = Count(s, sep) + 1 + } + c := sep[0] + start := 0 + a := make([][]byte, n) + na := 0 + for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { + if s[i] == c && (len(sep) == 1 || Equal(s[i:i+len(sep)], sep)) { + a[na] = s[start : i+sepSave] + na++ + start = i + len(sep) + i += len(sep) - 1 + } + } + a[na] = s[start:] + return a[0 : na+1] +} + +// SplitN slices s into subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, SplitN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitN(s, sep []byte, n int) [][]byte { return genSplit(s, sep, 0, n) } + +// SplitAfterN slices s into subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfterN splits after each UTF-8 sequence. +// The count determines the number of subslices to return: +// n > 0: at most n subslices; the last subslice will be the unsplit remainder. +// n == 0: the result is nil (zero subslices) +// n < 0: all subslices +func SplitAfterN(s, sep []byte, n int) [][]byte { + return genSplit(s, sep, len(sep), n) +} + +// Split slices s into all subslices separated by sep and returns a slice of +// the subslices between those separators. +// If sep is empty, Split splits after each UTF-8 sequence. +// It is equivalent to SplitN with a count of -1. +func Split(s, sep []byte) [][]byte { return genSplit(s, sep, 0, -1) } + +// SplitAfter slices s into all subslices after each instance of sep and +// returns a slice of those subslices. +// If sep is empty, SplitAfter splits after each UTF-8 sequence. +// It is equivalent to SplitAfterN with a count of -1. +func SplitAfter(s, sep []byte) [][]byte { + return genSplit(s, sep, len(sep), -1) +} + +// Fields splits the slice s around each instance of one or more consecutive white space +// characters, returning a slice of subslices of s or an empty list if s contains only white space. +func Fields(s []byte) [][]byte { + return FieldsFunc(s, unicode.IsSpace) +} + +// FieldsFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It splits the slice s at each run of code points c satisfying f(c) and +// returns a slice of subslices of s. If all code points in s satisfy f(c), or +// len(s) == 0, an empty slice is returned. +// FieldsFunc makes no guarantees about the order in which it calls f(c). +// If f does not return consistent results for a given c, FieldsFunc may crash. +func FieldsFunc(s []byte, f func(rune) bool) [][]byte { + n := 0 + inField := false + for i := 0; i < len(s); { + r, size := utf8.DecodeRune(s[i:]) + wasInField := inField + inField = !f(r) + if inField && !wasInField { + n++ + } + i += size + } + + a := make([][]byte, n) + na := 0 + fieldStart := -1 + for i := 0; i <= len(s) && na < n; { + r, size := utf8.DecodeRune(s[i:]) + if fieldStart < 0 && size > 0 && !f(r) { + fieldStart = i + i += size + continue + } + if fieldStart >= 0 && (size == 0 || f(r)) { + a[na] = s[fieldStart:i] + na++ + fieldStart = -1 + } + if size == 0 { + break + } + i += size + } + return a[0:na] +} + +// Join concatenates the elements of s to create a new byte slice. The separator +// sep is placed between elements in the resulting slice. +func Join(s [][]byte, sep []byte) []byte { + if len(s) == 0 { + return []byte{} + } + if len(s) == 1 { + // Just return a copy. + return append([]byte(nil), s[0]...) + } + n := len(sep) * (len(s) - 1) + for _, v := range s { + n += len(v) + } + + b := make([]byte, n) + bp := copy(b, s[0]) + for _, v := range s[1:] { + bp += copy(b[bp:], sep) + bp += copy(b[bp:], v) + } + return b +} + +// HasPrefix tests whether the byte slice s begins with prefix. +func HasPrefix(s, prefix []byte) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the byte slice s ends with suffix. +func HasSuffix(s, suffix []byte) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} + +// Map returns a copy of the byte slice s with all its characters modified +// according to the mapping function. If mapping returns a negative value, the character is +// dropped from the string with no replacement. The characters in s and the +// output are interpreted as UTF-8-encoded Unicode code points. +func Map(mapping func(r rune) rune, s []byte) []byte { + // In the worst case, the slice can grow when mapped, making + // things unpleasant. But it's so rare we barge in assuming it's + // fine. It could also shrink but that falls out naturally. + maxbytes := len(s) // length of b + nbytes := 0 // number of bytes encoded in b + b := make([]byte, maxbytes) + for i := 0; i < len(s); { + wid := 1 + r := rune(s[i]) + if r >= utf8.RuneSelf { + r, wid = utf8.DecodeRune(s[i:]) + } + r = mapping(r) + if r >= 0 { + rl := utf8.RuneLen(r) + if rl < 0 { + rl = len(string(utf8.RuneError)) + } + if nbytes+rl > maxbytes { + // Grow the buffer. + maxbytes = maxbytes*2 + utf8.UTFMax + nb := make([]byte, maxbytes) + copy(nb, b[0:nbytes]) + b = nb + } + nbytes += utf8.EncodeRune(b[nbytes:maxbytes], r) + } + i += wid + } + return b[0:nbytes] +} + +// Repeat returns a new byte slice consisting of count copies of b. +func Repeat(b []byte, count int) []byte { + nb := make([]byte, len(b)*count) + bp := copy(nb, b) + for bp < len(nb) { + copy(nb[bp:], nb[:bp]) + bp *= 2 + } + return nb +} + +// ToUpper returns a copy of the byte slice s with all Unicode letters mapped to their upper case. +func ToUpper(s []byte) []byte { return Map(unicode.ToUpper, s) } + +// ToLower returns a copy of the byte slice s with all Unicode letters mapped to their lower case. +func ToLower(s []byte) []byte { return Map(unicode.ToLower, s) } + +// ToTitle returns a copy of the byte slice s with all Unicode letters mapped to their title case. +func ToTitle(s []byte) []byte { return Map(unicode.ToTitle, s) } + +// ToUpperSpecial returns a copy of the byte slice s with all Unicode letters mapped to their +// upper case, giving priority to the special casing rules. +func ToUpperSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r rune) rune { return _case.ToUpper(r) }, s) +} + +// ToLowerSpecial returns a copy of the byte slice s with all Unicode letters mapped to their +// lower case, giving priority to the special casing rules. +func ToLowerSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r rune) rune { return _case.ToLower(r) }, s) +} + +// ToTitleSpecial returns a copy of the byte slice s with all Unicode letters mapped to their +// title case, giving priority to the special casing rules. +func ToTitleSpecial(_case unicode.SpecialCase, s []byte) []byte { + return Map(func(r rune) rune { return _case.ToTitle(r) }, s) +} + +// isSeparator reports whether the rune could mark a word boundary. +// TODO: update when package unicode captures more of the properties. +func isSeparator(r rune) bool { + // ASCII alphanumerics and underscore are not separators + if r <= 0x7F { + switch { + case '0' <= r && r <= '9': + return false + case 'a' <= r && r <= 'z': + return false + case 'A' <= r && r <= 'Z': + return false + case r == '_': + return false + } + return true + } + // Letters and digits are not separators + if unicode.IsLetter(r) || unicode.IsDigit(r) { + return false + } + // Otherwise, all we can do for now is treat spaces as separators. + return unicode.IsSpace(r) +} + +// Title returns a copy of s with all Unicode letters that begin words +// mapped to their title case. +// +// BUG: The rule Title uses for word boundaries does not handle Unicode punctuation properly. +func Title(s []byte) []byte { + // Use a closure here to remember state. + // Hackish but effective. Depends on Map scanning in order and calling + // the closure once per rune. + prev := ' ' + return Map( + func(r rune) rune { + if isSeparator(prev) { + prev = r + return unicode.ToTitle(r) + } + prev = r + return r + }, + s) +} + +// TrimLeftFunc returns a subslice of s by slicing off all leading UTF-8-encoded +// Unicode code points c that satisfy f(c). +func TrimLeftFunc(s []byte, f func(r rune) bool) []byte { + i := indexFunc(s, f, false) + if i == -1 { + return nil + } + return s[i:] +} + +// TrimRightFunc returns a subslice of s by slicing off all trailing UTF-8 +// encoded Unicode code points c that satisfy f(c). +func TrimRightFunc(s []byte, f func(r rune) bool) []byte { + i := lastIndexFunc(s, f, false) + if i >= 0 && s[i] >= utf8.RuneSelf { + _, wid := utf8.DecodeRune(s[i:]) + i += wid + } else { + i++ + } + return s[0:i] +} + +// TrimFunc returns a subslice of s by slicing off all leading and trailing +// UTF-8-encoded Unicode code points c that satisfy f(c). +func TrimFunc(s []byte, f func(r rune) bool) []byte { + return TrimRightFunc(TrimLeftFunc(s, f), f) +} + +// TrimPrefix returns s without the provided leading prefix string. +// If s doesn't start with prefix, s is returned unchanged. +func TrimPrefix(s, prefix []byte) []byte { + if HasPrefix(s, prefix) { + return s[len(prefix):] + } + return s +} + +// TrimSuffix returns s without the provided trailing suffix string. +// If s doesn't end with suffix, s is returned unchanged. +func TrimSuffix(s, suffix []byte) []byte { + if HasSuffix(s, suffix) { + return s[:len(s)-len(suffix)] + } + return s +} + +// IndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the first Unicode +// code point satisfying f(c), or -1 if none do. +func IndexFunc(s []byte, f func(r rune) bool) int { + return indexFunc(s, f, true) +} + +// LastIndexFunc interprets s as a sequence of UTF-8-encoded Unicode code points. +// It returns the byte index in s of the last Unicode +// code point satisfying f(c), or -1 if none do. +func LastIndexFunc(s []byte, f func(r rune) bool) int { + return lastIndexFunc(s, f, true) +} + +// indexFunc is the same as IndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func indexFunc(s []byte, f func(r rune) bool, truth bool) int { + start := 0 + for start < len(s) { + wid := 1 + r := rune(s[start]) + if r >= utf8.RuneSelf { + r, wid = utf8.DecodeRune(s[start:]) + } + if f(r) == truth { + return start + } + start += wid + } + return -1 +} + +// lastIndexFunc is the same as LastIndexFunc except that if +// truth==false, the sense of the predicate function is +// inverted. +func lastIndexFunc(s []byte, f func(r rune) bool, truth bool) int { + for i := len(s); i > 0; { + r, size := rune(s[i-1]), 1 + if r >= utf8.RuneSelf { + r, size = utf8.DecodeLastRune(s[0:i]) + } + i -= size + if f(r) == truth { + return i + } + } + return -1 +} + +func makeCutsetFunc(cutset string) func(r rune) bool { + return func(r rune) bool { + for _, c := range cutset { + if c == r { + return true + } + } + return false + } +} + +// Trim returns a subslice of s by slicing off all leading and +// trailing UTF-8-encoded Unicode code points contained in cutset. +func Trim(s []byte, cutset string) []byte { + return TrimFunc(s, makeCutsetFunc(cutset)) +} + +// TrimLeft returns a subslice of s by slicing off all leading +// UTF-8-encoded Unicode code points contained in cutset. +func TrimLeft(s []byte, cutset string) []byte { + return TrimLeftFunc(s, makeCutsetFunc(cutset)) +} + +// TrimRight returns a subslice of s by slicing off all trailing +// UTF-8-encoded Unicode code points that are contained in cutset. +func TrimRight(s []byte, cutset string) []byte { + return TrimRightFunc(s, makeCutsetFunc(cutset)) +} + +// TrimSpace returns a subslice of s by slicing off all leading and +// trailing white space, as defined by Unicode. +func TrimSpace(s []byte) []byte { + return TrimFunc(s, unicode.IsSpace) +} + +// Runes returns a slice of runes (Unicode code points) equivalent to s. +func Runes(s []byte) []rune { + t := make([]rune, utf8.RuneCount(s)) + i := 0 + for len(s) > 0 { + r, l := utf8.DecodeRune(s) + t[i] = r + i++ + s = s[l:] + } + return t +} + +// Replace returns a copy of the slice s with the first n +// non-overlapping instances of old replaced by new. +// If old is empty, it matches at the beginning of the slice +// and after each UTF-8 sequence, yielding up to k+1 replacements +// for a k-rune slice. +// If n < 0, there is no limit on the number of replacements. +func Replace(s, old, new []byte, n int) []byte { + m := 0 + if n != 0 { + // Compute number of replacements. + m = Count(s, old) + } + if m == 0 { + // Just return a copy. + return append([]byte(nil), s...) + } + if n < 0 || m < n { + n = m + } + + // Apply replacements to buffer. + t := make([]byte, len(s)+n*(len(new)-len(old))) + w := 0 + start := 0 + for i := 0; i < n; i++ { + j := start + if len(old) == 0 { + if i > 0 { + _, wid := utf8.DecodeRune(s[start:]) + j += wid + } + } else { + j += Index(s[start:], old) + } + w += copy(t[w:], s[start:j]) + w += copy(t[w:], new) + start = j + len(old) + } + w += copy(t[w:], s[start:]) + return t[0:w] +} + +// EqualFold reports whether s and t, interpreted as UTF-8 strings, +// are equal under Unicode case-folding. +func EqualFold(s, t []byte) bool { + for len(s) != 0 && len(t) != 0 { + // Extract first rune from each. + var sr, tr rune + if s[0] < utf8.RuneSelf { + sr, s = rune(s[0]), s[1:] + } else { + r, size := utf8.DecodeRune(s) + sr, s = r, s[size:] + } + if t[0] < utf8.RuneSelf { + tr, t = rune(t[0]), t[1:] + } else { + r, size := utf8.DecodeRune(t) + tr, t = r, t[size:] + } + + // If they match, keep going; if not, return false. + + // Easy case. + if tr == sr { + continue + } + + // Make sr < tr to simplify what follows. + if tr < sr { + tr, sr = sr, tr + } + // Fast check for ASCII. + if tr < utf8.RuneSelf && 'A' <= sr && sr <= 'Z' { + // ASCII, and sr is upper case. tr must be lower case. + if tr == sr+'a'-'A' { + continue + } + return false + } + + // General case. SimpleFold(x) returns the next equivalent rune > x + // or wraps around to smaller values. + r := unicode.SimpleFold(sr) + for r != sr && r < tr { + r = unicode.SimpleFold(r) + } + if r == tr { + continue + } + return false + } + + // One string is empty. Are both? + return len(s) == len(t) +} diff --git a/src/bytes/bytes_decl.go b/src/bytes/bytes_decl.go new file mode 100644 index 000000000..617d7489a --- /dev/null +++ b/src/bytes/bytes_decl.go @@ -0,0 +1,24 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +//go:noescape + +// IndexByte returns the index of the first instance of c in s, or -1 if c is not present in s. +func IndexByte(s []byte, c byte) int // ../runtime/asm_$GOARCH.s + +//go:noescape + +// Equal returns a boolean reporting whether a and b +// are the same length and contain the same bytes. +// A nil argument is equivalent to an empty slice. +func Equal(a, b []byte) bool // ../runtime/asm_$GOARCH.s + +//go:noescape + +// Compare returns an integer comparing two byte slices lexicographically. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. +// A nil argument is equivalent to an empty slice. +func Compare(a, b []byte) int // ../runtime/noasm_arm.goc or ../runtime/asm_{386,amd64}.s diff --git a/src/bytes/bytes_test.go b/src/bytes/bytes_test.go new file mode 100644 index 000000000..980c41d75 --- /dev/null +++ b/src/bytes/bytes_test.go @@ -0,0 +1,1240 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "math/rand" + "reflect" + "testing" + "unicode" + "unicode/utf8" +) + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} + +func sliceOfString(s [][]byte) []string { + result := make([]string, len(s)) + for i, v := range s { + result[i] = string(v) + } + return result +} + +// For ease of reading, the test cases use strings that are converted to byte +// slices before invoking the functions. + +var abcd = "abcd" +var faces = "☺☻☹" +var commas = "1,2,3,4" +var dots = "1....2....3....4" + +type BinOpTest struct { + a string + b string + i int +} + +var equalTests = []struct { + a, b []byte + i int +}{ + {[]byte(""), []byte(""), 0}, + {[]byte("a"), []byte(""), 1}, + {[]byte(""), []byte("a"), -1}, + {[]byte("abc"), []byte("abc"), 0}, + {[]byte("ab"), []byte("abc"), -1}, + {[]byte("abc"), []byte("ab"), 1}, + {[]byte("x"), []byte("ab"), 1}, + {[]byte("ab"), []byte("x"), -1}, + {[]byte("x"), []byte("a"), 1}, + {[]byte("b"), []byte("x"), -1}, + // test runtime·memeq's chunked implementation + {[]byte("abcdefgh"), []byte("abcdefgh"), 0}, + {[]byte("abcdefghi"), []byte("abcdefghi"), 0}, + {[]byte("abcdefghi"), []byte("abcdefghj"), -1}, + // nil tests + {nil, nil, 0}, + {[]byte(""), nil, 0}, + {nil, []byte(""), 0}, + {[]byte("a"), nil, 1}, + {nil, []byte("a"), -1}, +} + +func TestEqual(t *testing.T) { + for _, tt := range compareTests { + eql := Equal(tt.a, tt.b) + if eql != (tt.i == 0) { + t.Errorf(`Equal(%q, %q) = %v`, tt.a, tt.b, eql) + } + eql = EqualPortable(tt.a, tt.b) + if eql != (tt.i == 0) { + t.Errorf(`EqualPortable(%q, %q) = %v`, tt.a, tt.b, eql) + } + } +} + +func TestEqualExhaustive(t *testing.T) { + var size = 128 + if testing.Short() { + size = 32 + } + a := make([]byte, size) + b := make([]byte, size) + b_init := make([]byte, size) + // randomish but deterministic data + for i := 0; i < size; i++ { + a[i] = byte(17 * i) + b_init[i] = byte(23*i + 100) + } + + for len := 0; len <= size; len++ { + for x := 0; x <= size-len; x++ { + for y := 0; y <= size-len; y++ { + copy(b, b_init) + copy(b[y:y+len], a[x:x+len]) + if !Equal(a[x:x+len], b[y:y+len]) || !Equal(b[y:y+len], a[x:x+len]) { + t.Errorf("Equal(%d, %d, %d) = false", len, x, y) + } + } + } + } +} + +// make sure Equal returns false for minimally different strings. The data +// is all zeros except for a single one in one location. +func TestNotEqual(t *testing.T) { + var size = 128 + if testing.Short() { + size = 32 + } + a := make([]byte, size) + b := make([]byte, size) + + for len := 0; len <= size; len++ { + for x := 0; x <= size-len; x++ { + for y := 0; y <= size-len; y++ { + for diffpos := x; diffpos < x+len; diffpos++ { + a[diffpos] = 1 + if Equal(a[x:x+len], b[y:y+len]) || Equal(b[y:y+len], a[x:x+len]) { + t.Errorf("NotEqual(%d, %d, %d, %d) = true", len, x, y, diffpos) + } + a[diffpos] = 0 + } + } + } + } +} + +var indexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "baz", -1}, + {"foo", "foo", 0}, + {"oofofoofooo", "f", 2}, + {"oofofoofooo", "foo", 4}, + {"barfoobarfoo", "foo", 3}, + {"foo", "", 0}, + {"foo", "o", 1}, + {"abcABCabc", "A", 3}, + // cases with one byte strings - test IndexByte and special case in Index() + {"", "a", -1}, + {"x", "a", -1}, + {"x", "x", 0}, + {"abc", "a", 0}, + {"abc", "b", 1}, + {"abc", "c", 2}, + {"abc", "x", -1}, + {"barfoobarfooyyyzzzyyyzzzyyyzzzyyyxxxzzzyyy", "x", 33}, + {"foofyfoobarfoobar", "y", 4}, + {"oooooooooooooooooooooo", "r", -1}, +} + +var lastIndexTests = []BinOpTest{ + {"", "", 0}, + {"", "a", -1}, + {"", "foo", -1}, + {"fo", "foo", -1}, + {"foo", "foo", 0}, + {"foo", "f", 0}, + {"oofofoofooo", "f", 7}, + {"oofofoofooo", "foo", 7}, + {"barfoobarfoo", "foo", 9}, + {"foo", "", 3}, + {"foo", "o", 2}, + {"abcABCabc", "A", 3}, + {"abcABCabc", "a", 6}, +} + +var indexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 0}, + {"abc", "xyz", -1}, + {"abc", "xcz", 2}, + {"ab☺c", "x☺yz", 2}, + {"aRegExp*", ".(|)*+?^$[]", 7}, + {dots + dots + dots, " ", -1}, +} + +var lastIndexAnyTests = []BinOpTest{ + {"", "", -1}, + {"", "a", -1}, + {"", "abc", -1}, + {"a", "", -1}, + {"a", "a", 0}, + {"aaa", "a", 2}, + {"abc", "xyz", -1}, + {"abc", "ab", 1}, + {"a☺b☻c☹d", "uvw☻xyz", 2 + len("☺")}, + {"a.RegExp*", ".(|)*+?^$[]", 8}, + {dots + dots + dots, " ", -1}, +} + +var indexRuneTests = []BinOpTest{ + {"", "a", -1}, + {"", "☺", -1}, + {"foo", "☹", -1}, + {"foo", "o", 1}, + {"foo☺bar", "☺", 3}, + {"foo☺☻☹bar", "☹", 9}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runIndexTests(t *testing.T, f func(s, sep []byte) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + b := []byte(test.b) + actual := f(a, b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, b, actual, test.i) + } + } +} + +func runIndexAnyTests(t *testing.T, f func(s []byte, chars string) int, funcName string, testCases []BinOpTest) { + for _, test := range testCases { + a := []byte(test.a) + actual := f(a, test.b) + if actual != test.i { + t.Errorf("%s(%q,%q) = %v; want %v", funcName, a, test.b, actual, test.i) + } + } +} + +func TestIndex(t *testing.T) { runIndexTests(t, Index, "Index", indexTests) } +func TestLastIndex(t *testing.T) { runIndexTests(t, LastIndex, "LastIndex", lastIndexTests) } +func TestIndexAny(t *testing.T) { runIndexAnyTests(t, IndexAny, "IndexAny", indexAnyTests) } +func TestLastIndexAny(t *testing.T) { + runIndexAnyTests(t, LastIndexAny, "LastIndexAny", lastIndexAnyTests) +} + +func TestIndexByte(t *testing.T) { + for _, tt := range indexTests { + if len(tt.b) != 1 { + continue + } + a := []byte(tt.a) + b := tt.b[0] + pos := IndexByte(a, b) + if pos != tt.i { + t.Errorf(`IndexByte(%q, '%c') = %v`, tt.a, b, pos) + } + posp := IndexBytePortable(a, b) + if posp != tt.i { + t.Errorf(`indexBytePortable(%q, '%c') = %v`, tt.a, b, posp) + } + } +} + +// test a larger buffer with different sizes and alignments +func TestIndexByteBig(t *testing.T) { + var n = 1024 + if testing.Short() { + n = 128 + } + b := make([]byte, n) + for i := 0; i < n; i++ { + // different start alignments + b1 := b[i:] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different end alignments + b1 = b[:i] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + // different start and end alignments + b1 = b[i/2 : n-(i+1)/2] + for j := 0; j < len(b1); j++ { + b1[j] = 'x' + pos := IndexByte(b1, 'x') + if pos != j { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + b1[j] = 0 + pos = IndexByte(b1, 'x') + if pos != -1 { + t.Errorf("IndexByte(%q, 'x') = %v", b1, pos) + } + } + } +} + +func TestIndexRune(t *testing.T) { + for _, tt := range indexRuneTests { + a := []byte(tt.a) + r, _ := utf8.DecodeRuneInString(tt.b) + pos := IndexRune(a, r) + if pos != tt.i { + t.Errorf(`IndexRune(%q, '%c') = %v`, tt.a, r, pos) + } + } +} + +var bmbuf []byte + +func BenchmarkIndexByte32(b *testing.B) { bmIndexByte(b, IndexByte, 32) } +func BenchmarkIndexByte4K(b *testing.B) { bmIndexByte(b, IndexByte, 4<<10) } +func BenchmarkIndexByte4M(b *testing.B) { bmIndexByte(b, IndexByte, 4<<20) } +func BenchmarkIndexByte64M(b *testing.B) { bmIndexByte(b, IndexByte, 64<<20) } +func BenchmarkIndexBytePortable32(b *testing.B) { bmIndexByte(b, IndexBytePortable, 32) } +func BenchmarkIndexBytePortable4K(b *testing.B) { bmIndexByte(b, IndexBytePortable, 4<<10) } +func BenchmarkIndexBytePortable4M(b *testing.B) { bmIndexByte(b, IndexBytePortable, 4<<20) } +func BenchmarkIndexBytePortable64M(b *testing.B) { bmIndexByte(b, IndexBytePortable, 64<<20) } + +func bmIndexByte(b *testing.B, index func([]byte, byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, 'x') + if j != n-1 { + b.Fatal("bad index", j) + } + } + buf[n-1] = '\x00' +} + +func BenchmarkEqual0(b *testing.B) { + var buf [4]byte + buf1 := buf[0:0] + buf2 := buf[1:1] + for i := 0; i < b.N; i++ { + eq := Equal(buf1, buf2) + if !eq { + b.Fatal("bad equal") + } + } +} + +func BenchmarkEqual1(b *testing.B) { bmEqual(b, Equal, 1) } +func BenchmarkEqual6(b *testing.B) { bmEqual(b, Equal, 6) } +func BenchmarkEqual9(b *testing.B) { bmEqual(b, Equal, 9) } +func BenchmarkEqual15(b *testing.B) { bmEqual(b, Equal, 15) } +func BenchmarkEqual16(b *testing.B) { bmEqual(b, Equal, 16) } +func BenchmarkEqual20(b *testing.B) { bmEqual(b, Equal, 20) } +func BenchmarkEqual32(b *testing.B) { bmEqual(b, Equal, 32) } +func BenchmarkEqual4K(b *testing.B) { bmEqual(b, Equal, 4<<10) } +func BenchmarkEqual4M(b *testing.B) { bmEqual(b, Equal, 4<<20) } +func BenchmarkEqual64M(b *testing.B) { bmEqual(b, Equal, 64<<20) } +func BenchmarkEqualPort1(b *testing.B) { bmEqual(b, EqualPortable, 1) } +func BenchmarkEqualPort6(b *testing.B) { bmEqual(b, EqualPortable, 6) } +func BenchmarkEqualPort32(b *testing.B) { bmEqual(b, EqualPortable, 32) } +func BenchmarkEqualPort4K(b *testing.B) { bmEqual(b, EqualPortable, 4<<10) } +func BenchmarkEqualPortable4M(b *testing.B) { bmEqual(b, EqualPortable, 4<<20) } +func BenchmarkEqualPortable64M(b *testing.B) { bmEqual(b, EqualPortable, 64<<20) } + +func bmEqual(b *testing.B, equal func([]byte, []byte) bool, n int) { + if len(bmbuf) < 2*n { + bmbuf = make([]byte, 2*n) + } + b.SetBytes(int64(n)) + buf1 := bmbuf[0:n] + buf2 := bmbuf[n : 2*n] + buf1[n-1] = 'x' + buf2[n-1] = 'x' + for i := 0; i < b.N; i++ { + eq := equal(buf1, buf2) + if !eq { + b.Fatal("bad equal") + } + } + buf1[n-1] = '\x00' + buf2[n-1] = '\x00' +} + +func BenchmarkIndex32(b *testing.B) { bmIndex(b, Index, 32) } +func BenchmarkIndex4K(b *testing.B) { bmIndex(b, Index, 4<<10) } +func BenchmarkIndex4M(b *testing.B) { bmIndex(b, Index, 4<<20) } +func BenchmarkIndex64M(b *testing.B) { bmIndex(b, Index, 64<<20) } + +func bmIndex(b *testing.B, index func([]byte, []byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, buf[n-7:]) + if j != n-7 { + b.Fatal("bad index", j) + } + } + buf[n-1] = '\x00' +} + +func BenchmarkIndexEasy32(b *testing.B) { bmIndexEasy(b, Index, 32) } +func BenchmarkIndexEasy4K(b *testing.B) { bmIndexEasy(b, Index, 4<<10) } +func BenchmarkIndexEasy4M(b *testing.B) { bmIndexEasy(b, Index, 4<<20) } +func BenchmarkIndexEasy64M(b *testing.B) { bmIndexEasy(b, Index, 64<<20) } + +func bmIndexEasy(b *testing.B, index func([]byte, []byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + buf[n-7] = 'x' + for i := 0; i < b.N; i++ { + j := index(buf, buf[n-7:]) + if j != n-7 { + b.Fatal("bad index", j) + } + } + buf[n-1] = '\x00' + buf[n-7] = '\x00' +} + +func BenchmarkCount32(b *testing.B) { bmCount(b, Count, 32) } +func BenchmarkCount4K(b *testing.B) { bmCount(b, Count, 4<<10) } +func BenchmarkCount4M(b *testing.B) { bmCount(b, Count, 4<<20) } +func BenchmarkCount64M(b *testing.B) { bmCount(b, Count, 64<<20) } + +func bmCount(b *testing.B, count func([]byte, []byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + for i := 0; i < b.N; i++ { + j := count(buf, buf[n-7:]) + if j != 1 { + b.Fatal("bad count", j) + } + } + buf[n-1] = '\x00' +} + +func BenchmarkCountEasy32(b *testing.B) { bmCountEasy(b, Count, 32) } +func BenchmarkCountEasy4K(b *testing.B) { bmCountEasy(b, Count, 4<<10) } +func BenchmarkCountEasy4M(b *testing.B) { bmCountEasy(b, Count, 4<<20) } +func BenchmarkCountEasy64M(b *testing.B) { bmCountEasy(b, Count, 64<<20) } + +func bmCountEasy(b *testing.B, count func([]byte, []byte) int, n int) { + if len(bmbuf) < n { + bmbuf = make([]byte, n) + } + b.SetBytes(int64(n)) + buf := bmbuf[0:n] + buf[n-1] = 'x' + buf[n-7] = 'x' + for i := 0; i < b.N; i++ { + j := count(buf, buf[n-7:]) + if j != 1 { + b.Fatal("bad count", j) + } + } + buf[n-1] = '\x00' + buf[n-7] = '\x00' +} + +type ExplodeTest struct { + s string + n int + a []string +} + +var explodetests = []ExplodeTest{ + {"", -1, []string{}}, + {abcd, -1, []string{"a", "b", "c", "d"}}, + {faces, -1, []string{"☺", "☻", "☹"}}, + {abcd, 2, []string{"a", "bcd"}}, +} + +func TestExplode(t *testing.T) { + for _, tt := range explodetests { + a := SplitN([]byte(tt.s), nil, tt.n) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Explode("%s", %d) = %v; want %v`, tt.s, tt.n, result, tt.a) + continue + } + s := Join(a, []byte{}) + if string(s) != tt.s { + t.Errorf(`Join(Explode("%s", %d), "") = "%s"`, tt.s, tt.n, s) + } + } +} + +type SplitTest struct { + s string + sep string + n int + a []string +} + +var splittests = []SplitTest{ + {abcd, "a", 0, nil}, + {abcd, "a", -1, []string{"", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1", "2", "3", "4"}}, + {dots, "...", -1, []string{"1", ".2", ".3", ".4"}}, + {faces, "☹", -1, []string{"☺☻", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1", "2", "3 4"}}, + {"1 2", " ", 3, []string{"1", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplit(t *testing.T) { + for _, tt := range splittests { + a := SplitN([]byte(tt.s), []byte(tt.sep), tt.n) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + if tt.n == 0 { + continue + } + s := Join(a, []byte(tt.sep)) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := Split([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("Split disagrees withSplitN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + if len(a) > 0 { + in, out := a[0], s + if cap(in) == cap(out) && &in[:1][0] == &out[:1][0] { + t.Errorf("Join(%#v, %q) didn't copy", a, tt.sep) + } + } + } +} + +var splitaftertests = []SplitTest{ + {abcd, "a", -1, []string{"a", "bcd"}}, + {abcd, "z", -1, []string{"abcd"}}, + {abcd, "", -1, []string{"a", "b", "c", "d"}}, + {commas, ",", -1, []string{"1,", "2,", "3,", "4"}}, + {dots, "...", -1, []string{"1...", ".2...", ".3...", ".4"}}, + {faces, "☹", -1, []string{"☺☻☹", ""}}, + {faces, "~", -1, []string{faces}}, + {faces, "", -1, []string{"☺", "☻", "☹"}}, + {"1 2 3 4", " ", 3, []string{"1 ", "2 ", "3 4"}}, + {"1 2 3", " ", 3, []string{"1 ", "2 ", "3"}}, + {"1 2", " ", 3, []string{"1 ", "2"}}, + {"123", "", 2, []string{"1", "23"}}, + {"123", "", 17, []string{"1", "2", "3"}}, +} + +func TestSplitAfter(t *testing.T) { + for _, tt := range splitaftertests { + a := SplitAfterN([]byte(tt.s), []byte(tt.sep), tt.n) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf(`Split(%q, %q, %d) = %v; want %v`, tt.s, tt.sep, tt.n, result, tt.a) + continue + } + s := Join(a, nil) + if string(s) != tt.s { + t.Errorf(`Join(Split(%q, %q, %d), %q) = %q`, tt.s, tt.sep, tt.n, tt.sep, s) + } + if tt.n < 0 { + b := SplitAfter([]byte(tt.s), []byte(tt.sep)) + if !reflect.DeepEqual(a, b) { + t.Errorf("SplitAfter disagrees withSplitAfterN(%q, %q, %d) = %v; want %v", tt.s, tt.sep, tt.n, b, a) + } + } + } +} + +type FieldsTest struct { + s string + a []string +} + +var fieldstests = []FieldsTest{ + {"", []string{}}, + {" ", []string{}}, + {" \t ", []string{}}, + {" abc ", []string{"abc"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1 2 3 4", []string{"1", "2", "3", "4"}}, + {"1\t\t2\t\t3\t4", []string{"1", "2", "3", "4"}}, + {"1\u20002\u20013\u20024", []string{"1", "2", "3", "4"}}, + {"\u2000\u2001\u2002", []string{}}, + {"\n™\t™\n", []string{"™", "™"}}, + {faces, []string{faces}}, +} + +func TestFields(t *testing.T) { + for _, tt := range fieldstests { + a := Fields([]byte(tt.s)) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf("Fields(%q) = %v; want %v", tt.s, a, tt.a) + continue + } + } +} + +func TestFieldsFunc(t *testing.T) { + for _, tt := range fieldstests { + a := FieldsFunc([]byte(tt.s), unicode.IsSpace) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf("FieldsFunc(%q, unicode.IsSpace) = %v; want %v", tt.s, a, tt.a) + continue + } + } + pred := func(c rune) bool { return c == 'X' } + var fieldsFuncTests = []FieldsTest{ + {"", []string{}}, + {"XX", []string{}}, + {"XXhiXXX", []string{"hi"}}, + {"aXXbXXXcX", []string{"a", "b", "c"}}, + } + for _, tt := range fieldsFuncTests { + a := FieldsFunc([]byte(tt.s), pred) + result := sliceOfString(a) + if !eq(result, tt.a) { + t.Errorf("FieldsFunc(%q) = %v, want %v", tt.s, a, tt.a) + } + } +} + +// Test case for any function which accepts and returns a byte slice. +// For ease of creation, we write the byte slices as strings. +type StringTest struct { + in, out string +} + +var upperTests = []StringTest{ + {"", ""}, + {"abc", "ABC"}, + {"AbC123", "ABC123"}, + {"azAZ09_", "AZAZ09_"}, + {"\u0250\u0250\u0250\u0250\u0250", "\u2C6F\u2C6F\u2C6F\u2C6F\u2C6F"}, // grows one byte per char +} + +var lowerTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {"AbC123", "abc123"}, + {"azAZ09_", "azaz09_"}, + {"\u2C6D\u2C6D\u2C6D\u2C6D\u2C6D", "\u0251\u0251\u0251\u0251\u0251"}, // shrinks one byte per char +} + +const space = "\t\v\r\f\n\u0085\u00a0\u2000\u3000" + +var trimSpaceTests = []StringTest{ + {"", ""}, + {"abc", "abc"}, + {space + "abc" + space, "abc"}, + {" ", ""}, + {" \t\r\n \t\t\r\r\n\n ", ""}, + {" \t\r\n x\t\t\r\r\n\n ", "x"}, + {" \u2000\t\r\n x\t\t\r\r\ny\n \u3000", "x\t\t\r\r\ny"}, + {"1 \t\r\n2", "1 \t\r\n2"}, + {" x\x80", "x\x80"}, + {" x\xc0", "x\xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x \xc0", "x \xc0"}, + {"x \xc0 ", "x \xc0"}, + {"x \xc0\xc0 ", "x \xc0\xc0"}, + {"x ☺\xc0\xc0 ", "x ☺\xc0\xc0"}, + {"x ☺ ", "x ☺"}, +} + +// Execute f on each test case. funcName should be the name of f; it's used +// in failure reports. +func runStringTests(t *testing.T, f func([]byte) []byte, funcName string, testCases []StringTest) { + for _, tc := range testCases { + actual := string(f([]byte(tc.in))) + if actual != tc.out { + t.Errorf("%s(%q) = %q; want %q", funcName, tc.in, actual, tc.out) + } + } +} + +func tenRunes(r rune) string { + runes := make([]rune, 10) + for i := range runes { + runes[i] = r + } + return string(runes) +} + +// User-defined self-inverse mapping function +func rot13(r rune) rune { + const step = 13 + if r >= 'a' && r <= 'z' { + return ((r - 'a' + step) % 26) + 'a' + } + if r >= 'A' && r <= 'Z' { + return ((r - 'A' + step) % 26) + 'A' + } + return r +} + +func TestMap(t *testing.T) { + // Run a couple of awful growth/shrinkage tests + a := tenRunes('a') + + // 1. Grow. This triggers two reallocations in Map. + maxRune := func(r rune) rune { return unicode.MaxRune } + m := Map(maxRune, []byte(a)) + expect := tenRunes(unicode.MaxRune) + if string(m) != expect { + t.Errorf("growing: expected %q got %q", expect, m) + } + + // 2. Shrink + minRune := func(r rune) rune { return 'a' } + m = Map(minRune, []byte(tenRunes(unicode.MaxRune))) + expect = a + if string(m) != expect { + t.Errorf("shrinking: expected %q got %q", expect, m) + } + + // 3. Rot13 + m = Map(rot13, []byte("a to zed")) + expect = "n gb mrq" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 4. Rot13^2 + m = Map(rot13, Map(rot13, []byte("a to zed"))) + expect = "a to zed" + if string(m) != expect { + t.Errorf("rot13: expected %q got %q", expect, m) + } + + // 5. Drop + dropNotLatin := func(r rune) rune { + if unicode.Is(unicode.Latin, r) { + return r + } + return -1 + } + m = Map(dropNotLatin, []byte("Hello, 세계")) + expect = "Hello" + if string(m) != expect { + t.Errorf("drop: expected %q got %q", expect, m) + } + + // 6. Invalid rune + invalidRune := func(r rune) rune { + return utf8.MaxRune + 1 + } + m = Map(invalidRune, []byte("x")) + expect = "\uFFFD" + if string(m) != expect { + t.Errorf("invalidRune: expected %q got %q", expect, m) + } +} + +func TestToUpper(t *testing.T) { runStringTests(t, ToUpper, "ToUpper", upperTests) } + +func TestToLower(t *testing.T) { runStringTests(t, ToLower, "ToLower", lowerTests) } + +func TestTrimSpace(t *testing.T) { runStringTests(t, TrimSpace, "TrimSpace", trimSpaceTests) } + +type RepeatTest struct { + in, out string + count int +} + +var RepeatTests = []RepeatTest{ + {"", "", 0}, + {"", "", 1}, + {"", "", 2}, + {"-", "", 0}, + {"-", "-", 1}, + {"-", "----------", 10}, + {"abc ", "abc abc abc ", 3}, +} + +func TestRepeat(t *testing.T) { + for _, tt := range RepeatTests { + tin := []byte(tt.in) + tout := []byte(tt.out) + a := Repeat(tin, tt.count) + if !Equal(a, tout) { + t.Errorf("Repeat(%q, %d) = %q; want %q", tin, tt.count, a, tout) + continue + } + } +} + +func runesEqual(a, b []rune) bool { + if len(a) != len(b) { + return false + } + for i, r := range a { + if r != b[i] { + return false + } + } + return true +} + +type RunesTest struct { + in string + out []rune + lossy bool +} + +var RunesTests = []RunesTest{ + {"", []rune{}, false}, + {" ", []rune{32}, false}, + {"ABC", []rune{65, 66, 67}, false}, + {"abc", []rune{97, 98, 99}, false}, + {"\u65e5\u672c\u8a9e", []rune{26085, 26412, 35486}, false}, + {"ab\x80c", []rune{97, 98, 0xFFFD, 99}, true}, + {"ab\xc0c", []rune{97, 98, 0xFFFD, 99}, true}, +} + +func TestRunes(t *testing.T) { + for _, tt := range RunesTests { + tin := []byte(tt.in) + a := Runes(tin) + if !runesEqual(a, tt.out) { + t.Errorf("Runes(%q) = %v; want %v", tin, a, tt.out) + continue + } + if !tt.lossy { + // can only test reassembly if we didn't lose information + s := string(a) + if s != tt.in { + t.Errorf("string(Runes(%q)) = %x; want %x", tin, s, tin) + } + } + } +} + +type TrimTest struct { + f string + in, arg, out string +} + +var trimTests = []TrimTest{ + {"Trim", "abba", "a", "bb"}, + {"Trim", "abba", "ab", ""}, + {"TrimLeft", "abba", "ab", ""}, + {"TrimRight", "abba", "ab", ""}, + {"TrimLeft", "abba", "a", "bba"}, + {"TrimRight", "abba", "a", "abb"}, + {"Trim", "<tag>", "<>", "tag"}, + {"Trim", "* listitem", " *", "listitem"}, + {"Trim", `"quote"`, `"`, "quote"}, + {"Trim", "\u2C6F\u2C6F\u0250\u0250\u2C6F\u2C6F", "\u2C6F", "\u0250\u0250"}, + //empty string tests + {"Trim", "abba", "", "abba"}, + {"Trim", "", "123", ""}, + {"Trim", "", "", ""}, + {"TrimLeft", "abba", "", "abba"}, + {"TrimLeft", "", "123", ""}, + {"TrimLeft", "", "", ""}, + {"TrimRight", "abba", "", "abba"}, + {"TrimRight", "", "123", ""}, + {"TrimRight", "", "", ""}, + {"TrimRight", "☺\xc0", "☺", "☺\xc0"}, + {"TrimPrefix", "aabb", "a", "abb"}, + {"TrimPrefix", "aabb", "b", "aabb"}, + {"TrimSuffix", "aabb", "a", "aabb"}, + {"TrimSuffix", "aabb", "b", "aab"}, +} + +func TestTrim(t *testing.T) { + for _, tc := range trimTests { + name := tc.f + var f func([]byte, string) []byte + var fb func([]byte, []byte) []byte + switch name { + case "Trim": + f = Trim + case "TrimLeft": + f = TrimLeft + case "TrimRight": + f = TrimRight + case "TrimPrefix": + fb = TrimPrefix + case "TrimSuffix": + fb = TrimSuffix + default: + t.Errorf("Undefined trim function %s", name) + } + var actual string + if f != nil { + actual = string(f([]byte(tc.in), tc.arg)) + } else { + actual = string(fb([]byte(tc.in), []byte(tc.arg))) + } + if actual != tc.out { + t.Errorf("%s(%q, %q) = %q; want %q", name, tc.in, tc.arg, actual, tc.out) + } + } +} + +type predicate struct { + f func(r rune) bool + name string +} + +var isSpace = predicate{unicode.IsSpace, "IsSpace"} +var isDigit = predicate{unicode.IsDigit, "IsDigit"} +var isUpper = predicate{unicode.IsUpper, "IsUpper"} +var isValidRune = predicate{ + func(r rune) bool { + return r != utf8.RuneError + }, + "IsValidRune", +} + +type TrimFuncTest struct { + f predicate + in, out string +} + +func not(p predicate) predicate { + return predicate{ + func(r rune) bool { + return !p.f(r) + }, + "not " + p.name, + } +} + +var trimFuncTests = []TrimFuncTest{ + {isSpace, space + " hello " + space, "hello"}, + {isDigit, "\u0e50\u0e5212hello34\u0e50\u0e51", "hello"}, + {isUpper, "\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", "hello"}, + {not(isSpace), "hello" + space + "hello", space}, + {not(isDigit), "hello\u0e50\u0e521234\u0e50\u0e51helo", "\u0e50\u0e521234\u0e50\u0e51"}, + {isValidRune, "ab\xc0a\xc0cd", "\xc0a\xc0"}, + {not(isValidRune), "\xc0a\xc0", "a"}, +} + +func TestTrimFunc(t *testing.T) { + for _, tc := range trimFuncTests { + actual := string(TrimFunc([]byte(tc.in), tc.f.f)) + if actual != tc.out { + t.Errorf("TrimFunc(%q, %q) = %q; want %q", tc.in, tc.f.name, actual, tc.out) + } + } +} + +type IndexFuncTest struct { + in string + f predicate + first, last int +} + +var indexFuncTests = []IndexFuncTest{ + {"", isValidRune, -1, -1}, + {"abc", isDigit, -1, -1}, + {"0123", isDigit, 0, 3}, + {"a1b", isDigit, 1, 1}, + {space, isSpace, 0, len(space) - 3}, // last rune in space is 3 bytes + {"\u0e50\u0e5212hello34\u0e50\u0e51", isDigit, 0, 18}, + {"\u2C6F\u2C6F\u2C6F\u2C6FABCDhelloEF\u2C6F\u2C6FGH\u2C6F\u2C6F", isUpper, 0, 34}, + {"12\u0e50\u0e52hello34\u0e50\u0e51", not(isDigit), 8, 12}, + + // tests of invalid UTF-8 + {"\x801", isDigit, 1, 1}, + {"\x80abc", isDigit, -1, -1}, + {"\xc0a\xc0", isValidRune, 1, 1}, + {"\xc0a\xc0", not(isValidRune), 0, 2}, + {"\xc0☺\xc0", not(isValidRune), 0, 4}, + {"\xc0☺\xc0\xc0", not(isValidRune), 0, 5}, + {"ab\xc0a\xc0cd", not(isValidRune), 2, 4}, + {"a\xe0\x80cd", not(isValidRune), 1, 2}, +} + +func TestIndexFunc(t *testing.T) { + for _, tc := range indexFuncTests { + first := IndexFunc([]byte(tc.in), tc.f.f) + if first != tc.first { + t.Errorf("IndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, first, tc.first) + } + last := LastIndexFunc([]byte(tc.in), tc.f.f) + if last != tc.last { + t.Errorf("LastIndexFunc(%q, %s) = %d; want %d", tc.in, tc.f.name, last, tc.last) + } + } +} + +type ReplaceTest struct { + in string + old, new string + n int + out string +} + +var ReplaceTests = []ReplaceTest{ + {"hello", "l", "L", 0, "hello"}, + {"hello", "l", "L", -1, "heLLo"}, + {"hello", "x", "X", -1, "hello"}, + {"", "x", "X", -1, ""}, + {"radar", "r", "<r>", -1, "<r>ada<r>"}, + {"", "", "<>", -1, "<>"}, + {"banana", "a", "<>", -1, "b<>n<>n<>"}, + {"banana", "a", "<>", 1, "b<>nana"}, + {"banana", "a", "<>", 1000, "b<>n<>n<>"}, + {"banana", "an", "<>", -1, "b<><>a"}, + {"banana", "ana", "<>", -1, "b<>na"}, + {"banana", "", "<>", -1, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 10, "<>b<>a<>n<>a<>n<>a<>"}, + {"banana", "", "<>", 6, "<>b<>a<>n<>a<>n<>a"}, + {"banana", "", "<>", 5, "<>b<>a<>n<>a<>na"}, + {"banana", "", "<>", 1, "<>banana"}, + {"banana", "a", "a", -1, "banana"}, + {"banana", "a", "a", 1, "banana"}, + {"☺☻☹", "", "<>", -1, "<>☺<>☻<>☹<>"}, +} + +func TestReplace(t *testing.T) { + for _, tt := range ReplaceTests { + in := append([]byte(tt.in), "<spare>"...) + in = in[:len(tt.in)] + out := Replace(in, []byte(tt.old), []byte(tt.new), tt.n) + if s := string(out); s != tt.out { + t.Errorf("Replace(%q, %q, %q, %d) = %q, want %q", tt.in, tt.old, tt.new, tt.n, s, tt.out) + } + if cap(in) == cap(out) && &in[:1][0] == &out[:1][0] { + t.Errorf("Replace(%q, %q, %q, %d) didn't copy", tt.in, tt.old, tt.new, tt.n) + } + } +} + +type TitleTest struct { + in, out string +} + +var TitleTests = []TitleTest{ + {"", ""}, + {"a", "A"}, + {" aaa aaa aaa ", " Aaa Aaa Aaa "}, + {" Aaa Aaa Aaa ", " Aaa Aaa Aaa "}, + {"123a456", "123a456"}, + {"double-blind", "Double-Blind"}, + {"ÿøû", "Ÿøû"}, + {"with_underscore", "With_underscore"}, + {"unicode \xe2\x80\xa8 line separator", "Unicode \xe2\x80\xa8 Line Separator"}, +} + +func TestTitle(t *testing.T) { + for _, tt := range TitleTests { + if s := string(Title([]byte(tt.in))); s != tt.out { + t.Errorf("Title(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} + +var ToTitleTests = []TitleTest{ + {"", ""}, + {"a", "A"}, + {" aaa aaa aaa ", " AAA AAA AAA "}, + {" Aaa Aaa Aaa ", " AAA AAA AAA "}, + {"123a456", "123A456"}, + {"double-blind", "DOUBLE-BLIND"}, + {"ÿøû", "ŸØÛ"}, +} + +func TestToTitle(t *testing.T) { + for _, tt := range ToTitleTests { + if s := string(ToTitle([]byte(tt.in))); s != tt.out { + t.Errorf("ToTitle(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} + +var EqualFoldTests = []struct { + s, t string + out bool +}{ + {"abc", "abc", true}, + {"ABcd", "ABcd", true}, + {"123abc", "123ABC", true}, + {"αβδ", "ΑΒΔ", true}, + {"abc", "xyz", false}, + {"abc", "XYZ", false}, + {"abcdefghijk", "abcdefghijX", false}, + {"abcdefghijk", "abcdefghij\u212A", true}, + {"abcdefghijK", "abcdefghij\u212A", true}, + {"abcdefghijkz", "abcdefghij\u212Ay", false}, + {"abcdefghijKz", "abcdefghij\u212Ay", false}, +} + +func TestEqualFold(t *testing.T) { + for _, tt := range EqualFoldTests { + if out := EqualFold([]byte(tt.s), []byte(tt.t)); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.s, tt.t, out, tt.out) + } + if out := EqualFold([]byte(tt.t), []byte(tt.s)); out != tt.out { + t.Errorf("EqualFold(%#q, %#q) = %v, want %v", tt.t, tt.s, out, tt.out) + } + } +} + +func TestBufferGrowNegative(t *testing.T) { + defer func() { + if err := recover(); err == nil { + t.Fatal("Grow(-1) should have panicked") + } + }() + var b Buffer + b.Grow(-1) +} + +func TestBufferTruncateNegative(t *testing.T) { + defer func() { + if err := recover(); err == nil { + t.Fatal("Truncate(-1) should have panicked") + } + }() + var b Buffer + b.Truncate(-1) +} + +func TestBufferTruncateOutOfRange(t *testing.T) { + defer func() { + if err := recover(); err == nil { + t.Fatal("Truncate(20) should have panicked") + } + }() + var b Buffer + b.Write(make([]byte, 10)) + b.Truncate(20) +} + +var containsTests = []struct { + b, subslice []byte + want bool +}{ + {[]byte("hello"), []byte("hel"), true}, + {[]byte("日本語"), []byte("日本"), true}, + {[]byte("hello"), []byte("Hello, world"), false}, + {[]byte("東京"), []byte("京東"), false}, +} + +func TestContains(t *testing.T) { + for _, tt := range containsTests { + if got := Contains(tt.b, tt.subslice); got != tt.want { + t.Errorf("Contains(%q, %q) = %v, want %v", tt.b, tt.subslice, got, tt.want) + } + } +} + +var makeFieldsInput = func() []byte { + x := make([]byte, 1<<20) + // Input is ~10% space, ~10% 2-byte UTF-8, rest ASCII non-space. + for i := range x { + switch rand.Intn(10) { + case 0: + x[i] = ' ' + case 1: + if i > 0 && x[i-1] == 'x' { + copy(x[i-1:], "χ") + break + } + fallthrough + default: + x[i] = 'x' + } + } + return x +} + +var fieldsInput = makeFieldsInput() + +func BenchmarkFields(b *testing.B) { + b.SetBytes(int64(len(fieldsInput))) + for i := 0; i < b.N; i++ { + Fields(fieldsInput) + } +} + +func BenchmarkFieldsFunc(b *testing.B) { + b.SetBytes(int64(len(fieldsInput))) + for i := 0; i < b.N; i++ { + FieldsFunc(fieldsInput, unicode.IsSpace) + } +} + +func BenchmarkTrimSpace(b *testing.B) { + s := []byte(" Some text. \n") + for i := 0; i < b.N; i++ { + TrimSpace(s) + } +} + +func BenchmarkRepeat(b *testing.B) { + for i := 0; i < b.N; i++ { + Repeat([]byte("-"), 80) + } +} diff --git a/src/bytes/compare_test.go b/src/bytes/compare_test.go new file mode 100644 index 000000000..63522374e --- /dev/null +++ b/src/bytes/compare_test.go @@ -0,0 +1,208 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "testing" +) + +var compareTests = []struct { + a, b []byte + i int +}{ + {[]byte(""), []byte(""), 0}, + {[]byte("a"), []byte(""), 1}, + {[]byte(""), []byte("a"), -1}, + {[]byte("abc"), []byte("abc"), 0}, + {[]byte("ab"), []byte("abc"), -1}, + {[]byte("abc"), []byte("ab"), 1}, + {[]byte("x"), []byte("ab"), 1}, + {[]byte("ab"), []byte("x"), -1}, + {[]byte("x"), []byte("a"), 1}, + {[]byte("b"), []byte("x"), -1}, + // test runtime·memeq's chunked implementation + {[]byte("abcdefgh"), []byte("abcdefgh"), 0}, + {[]byte("abcdefghi"), []byte("abcdefghi"), 0}, + {[]byte("abcdefghi"), []byte("abcdefghj"), -1}, + // nil tests + {nil, nil, 0}, + {[]byte(""), nil, 0}, + {nil, []byte(""), 0}, + {[]byte("a"), nil, 1}, + {nil, []byte("a"), -1}, +} + +func TestCompare(t *testing.T) { + for _, tt := range compareTests { + cmp := Compare(tt.a, tt.b) + if cmp != tt.i { + t.Errorf(`Compare(%q, %q) = %v`, tt.a, tt.b, cmp) + } + } +} + +func TestCompareIdenticalSlice(t *testing.T) { + var b = []byte("Hello Gophers!") + if Compare(b, b) != 0 { + t.Error("b != b") + } + if Compare(b, b[:1]) != 1 { + t.Error("b > b[:1] failed") + } +} + +func TestCompareBytes(t *testing.T) { + n := 128 + a := make([]byte, n+1) + b := make([]byte, n+1) + for len := 0; len < 128; len++ { + // randomish but deterministic data. No 0 or 255. + for i := 0; i < len; i++ { + a[i] = byte(1 + 31*i%254) + b[i] = byte(1 + 31*i%254) + } + // data past the end is different + for i := len; i <= n; i++ { + a[i] = 8 + b[i] = 9 + } + cmp := Compare(a[:len], b[:len]) + if cmp != 0 { + t.Errorf(`CompareIdentical(%d) = %d`, len, cmp) + } + if len > 0 { + cmp = Compare(a[:len-1], b[:len]) + if cmp != -1 { + t.Errorf(`CompareAshorter(%d) = %d`, len, cmp) + } + cmp = Compare(a[:len], b[:len-1]) + if cmp != 1 { + t.Errorf(`CompareBshorter(%d) = %d`, len, cmp) + } + } + for k := 0; k < len; k++ { + b[k] = a[k] - 1 + cmp = Compare(a[:len], b[:len]) + if cmp != 1 { + t.Errorf(`CompareAbigger(%d,%d) = %d`, len, k, cmp) + } + b[k] = a[k] + 1 + cmp = Compare(a[:len], b[:len]) + if cmp != -1 { + t.Errorf(`CompareBbigger(%d,%d) = %d`, len, k, cmp) + } + b[k] = a[k] + } + } +} + +func BenchmarkCompareBytesEqual(b *testing.B) { + b1 := []byte("Hello Gophers!") + b2 := []byte("Hello Gophers!") + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 0 { + b.Fatal("b1 != b2") + } + } +} + +func BenchmarkCompareBytesToNil(b *testing.B) { + b1 := []byte("Hello Gophers!") + var b2 []byte + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 1 { + b.Fatal("b1 > b2 failed") + } + } +} + +func BenchmarkCompareBytesEmpty(b *testing.B) { + b1 := []byte("") + b2 := b1 + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 0 { + b.Fatal("b1 != b2") + } + } +} + +func BenchmarkCompareBytesIdentical(b *testing.B) { + b1 := []byte("Hello Gophers!") + b2 := b1 + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 0 { + b.Fatal("b1 != b2") + } + } +} + +func BenchmarkCompareBytesSameLength(b *testing.B) { + b1 := []byte("Hello Gophers!") + b2 := []byte("Hello, Gophers") + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != -1 { + b.Fatal("b1 < b2 failed") + } + } +} + +func BenchmarkCompareBytesDifferentLength(b *testing.B) { + b1 := []byte("Hello Gophers!") + b2 := []byte("Hello, Gophers!") + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != -1 { + b.Fatal("b1 < b2 failed") + } + } +} + +func BenchmarkCompareBytesBigUnaligned(b *testing.B) { + b.StopTimer() + b1 := make([]byte, 0, 1<<20) + for len(b1) < 1<<20 { + b1 = append(b1, "Hello Gophers!"...) + } + b2 := append([]byte("hello"), b1...) + b.StartTimer() + for i := 0; i < b.N; i++ { + if Compare(b1, b2[len("hello"):]) != 0 { + b.Fatal("b1 != b2") + } + } + b.SetBytes(int64(len(b1))) +} + +func BenchmarkCompareBytesBig(b *testing.B) { + b.StopTimer() + b1 := make([]byte, 0, 1<<20) + for len(b1) < 1<<20 { + b1 = append(b1, "Hello Gophers!"...) + } + b2 := append([]byte{}, b1...) + b.StartTimer() + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 0 { + b.Fatal("b1 != b2") + } + } + b.SetBytes(int64(len(b1))) +} + +func BenchmarkCompareBytesBigIdentical(b *testing.B) { + b.StopTimer() + b1 := make([]byte, 0, 1<<20) + for len(b1) < 1<<20 { + b1 = append(b1, "Hello Gophers!"...) + } + b2 := b1 + b.StartTimer() + for i := 0; i < b.N; i++ { + if Compare(b1, b2) != 0 { + b.Fatal("b1 != b2") + } + } + b.SetBytes(int64(len(b1))) +} diff --git a/src/bytes/equal_test.go b/src/bytes/equal_test.go new file mode 100644 index 000000000..1bf19a74b --- /dev/null +++ b/src/bytes/equal_test.go @@ -0,0 +1,47 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// +build linux + +package bytes_test + +import ( + . "bytes" + "syscall" + "testing" + "unsafe" +) + +// This file tests the situation where memeq is checking +// data very near to a page boundary. We want to make sure +// equal does not read across the boundary and cause a page +// fault where it shouldn't. + +// This test runs only on linux. The code being tested is +// not OS-specific, so it does not need to be tested on all +// operating systems. + +func TestEqualNearPageBoundary(t *testing.T) { + pagesize := syscall.Getpagesize() + b := make([]byte, 4*pagesize) + i := pagesize + for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ { + } + syscall.Mprotect(b[i-pagesize:i], 0) + syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0) + defer syscall.Mprotect(b[i-pagesize:i], syscall.PROT_READ|syscall.PROT_WRITE) + defer syscall.Mprotect(b[i+pagesize:i+2*pagesize], syscall.PROT_READ|syscall.PROT_WRITE) + + // both of these should fault + //pagesize += int(b[i-1]) + //pagesize += int(b[i+pagesize]) + + for j := 0; j < pagesize; j++ { + b[i+j] = 'A' + } + for j := 0; j <= pagesize; j++ { + Equal(b[i:i+j], b[i+pagesize-j:i+pagesize]) + Equal(b[i+pagesize-j:i+pagesize], b[i:i+j]) + } +} diff --git a/src/bytes/example_test.go b/src/bytes/example_test.go new file mode 100644 index 000000000..ad2dbc69b --- /dev/null +++ b/src/bytes/example_test.go @@ -0,0 +1,85 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "os" + "sort" +) + +func ExampleBuffer() { + var b bytes.Buffer // A Buffer needs no initialization. + b.Write([]byte("Hello ")) + fmt.Fprintf(&b, "world!") + b.WriteTo(os.Stdout) + // Output: Hello world! +} + +func ExampleBuffer_reader() { + // A Buffer can turn a string or a []byte into an io.Reader. + buf := bytes.NewBufferString("R29waGVycyBydWxlIQ==") + dec := base64.NewDecoder(base64.StdEncoding, buf) + io.Copy(os.Stdout, dec) + // Output: Gophers rule! +} + +func ExampleCompare() { + // Interpret Compare's result by comparing it to zero. + var a, b []byte + if bytes.Compare(a, b) < 0 { + // a less b + } + if bytes.Compare(a, b) <= 0 { + // a less or equal b + } + if bytes.Compare(a, b) > 0 { + // a greater b + } + if bytes.Compare(a, b) >= 0 { + // a greater or equal b + } + + // Prefer Equal to Compare for equality comparisons. + if bytes.Equal(a, b) { + // a equal b + } + if !bytes.Equal(a, b) { + // a not equal b + } +} + +func ExampleCompare_search() { + // Binary search to find a matching byte slice. + var needle []byte + var haystack [][]byte // Assume sorted + i := sort.Search(len(haystack), func(i int) bool { + // Return haystack[i] >= needle. + return bytes.Compare(haystack[i], needle) >= 0 + }) + if i < len(haystack) && bytes.Equal(haystack[i], needle) { + // Found it! + } +} + +func ExampleTrimSuffix() { + var b = []byte("Hello, goodbye, etc!") + b = bytes.TrimSuffix(b, []byte("goodbye, etc!")) + b = bytes.TrimSuffix(b, []byte("gopher")) + b = append(b, bytes.TrimSuffix([]byte("world!"), []byte("x!"))...) + os.Stdout.Write(b) + // Output: Hello, world! +} + +func ExampleTrimPrefix() { + var b = []byte("Goodbye,, world!") + b = bytes.TrimPrefix(b, []byte("Goodbye,")) + b = bytes.TrimPrefix(b, []byte("See ya,")) + fmt.Printf("Hello%s", b) + // Output: Hello, world! +} diff --git a/src/bytes/export_test.go b/src/bytes/export_test.go new file mode 100644 index 000000000..3b915d5ea --- /dev/null +++ b/src/bytes/export_test.go @@ -0,0 +1,13 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +// Export func for testing +var IndexBytePortable = indexBytePortable +var EqualPortable = equalPortable + +func (b *Buffer) Cap() int { + return cap(b.buf) +} diff --git a/src/bytes/reader.go b/src/bytes/reader.go new file mode 100644 index 000000000..d2d40fa7c --- /dev/null +++ b/src/bytes/reader.go @@ -0,0 +1,144 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes + +import ( + "errors" + "io" + "unicode/utf8" +) + +// A Reader implements the io.Reader, io.ReaderAt, io.WriterTo, io.Seeker, +// io.ByteScanner, and io.RuneScanner interfaces by reading from +// a byte slice. +// Unlike a Buffer, a Reader is read-only and supports seeking. +type Reader struct { + s []byte + i int64 // current reading index + prevRune int // index of previous rune; or < 0 +} + +// Len returns the number of bytes of the unread portion of the +// slice. +func (r *Reader) Len() int { + if r.i >= int64(len(r.s)) { + return 0 + } + return int(int64(len(r.s)) - r.i) +} + +func (r *Reader) Read(b []byte) (n int, err error) { + if len(b) == 0 { + return 0, nil + } + if r.i >= int64(len(r.s)) { + return 0, io.EOF + } + r.prevRune = -1 + n = copy(b, r.s[r.i:]) + r.i += int64(n) + return +} + +func (r *Reader) ReadAt(b []byte, off int64) (n int, err error) { + // cannot modify state - see io.ReaderAt + if off < 0 { + return 0, errors.New("bytes.Reader.ReadAt: negative offset") + } + if off >= int64(len(r.s)) { + return 0, io.EOF + } + n = copy(b, r.s[off:]) + if n < len(b) { + err = io.EOF + } + return +} + +func (r *Reader) ReadByte() (b byte, err error) { + r.prevRune = -1 + if r.i >= int64(len(r.s)) { + return 0, io.EOF + } + b = r.s[r.i] + r.i++ + return +} + +func (r *Reader) UnreadByte() error { + r.prevRune = -1 + if r.i <= 0 { + return errors.New("bytes.Reader.UnreadByte: at beginning of slice") + } + r.i-- + return nil +} + +func (r *Reader) ReadRune() (ch rune, size int, err error) { + if r.i >= int64(len(r.s)) { + r.prevRune = -1 + return 0, 0, io.EOF + } + r.prevRune = int(r.i) + if c := r.s[r.i]; c < utf8.RuneSelf { + r.i++ + return rune(c), 1, nil + } + ch, size = utf8.DecodeRune(r.s[r.i:]) + r.i += int64(size) + return +} + +func (r *Reader) UnreadRune() error { + if r.prevRune < 0 { + return errors.New("bytes.Reader.UnreadRune: previous operation was not ReadRune") + } + r.i = int64(r.prevRune) + r.prevRune = -1 + return nil +} + +// Seek implements the io.Seeker interface. +func (r *Reader) Seek(offset int64, whence int) (int64, error) { + r.prevRune = -1 + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(r.i) + offset + case 2: + abs = int64(len(r.s)) + offset + default: + return 0, errors.New("bytes.Reader.Seek: invalid whence") + } + if abs < 0 { + return 0, errors.New("bytes.Reader.Seek: negative position") + } + r.i = abs + return abs, nil +} + +// WriteTo implements the io.WriterTo interface. +func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { + r.prevRune = -1 + if r.i >= int64(len(r.s)) { + return 0, nil + } + b := r.s[r.i:] + m, err := w.Write(b) + if m > len(b) { + panic("bytes.Reader.WriteTo: invalid Write count") + } + r.i += int64(m) + n = int64(m) + if m != len(b) && err == nil { + err = io.ErrShortWrite + } + return +} + +// NewReader returns a new Reader reading from b. +func NewReader(b []byte) *Reader { return &Reader{b, 0, -1} } diff --git a/src/bytes/reader_test.go b/src/bytes/reader_test.go new file mode 100644 index 000000000..d3dce5349 --- /dev/null +++ b/src/bytes/reader_test.go @@ -0,0 +1,246 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bytes_test + +import ( + . "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + "testing" +) + +func TestReader(t *testing.T) { + r := NewReader([]byte("0123456789")) + tests := []struct { + off int64 + seek int + n int + want string + wantpos int64 + seekerr string + }{ + {seek: os.SEEK_SET, off: 0, n: 20, want: "0123456789"}, + {seek: os.SEEK_SET, off: 1, n: 1, want: "1"}, + {seek: os.SEEK_CUR, off: 1, wantpos: 3, n: 2, want: "34"}, + {seek: os.SEEK_SET, off: -1, seekerr: "bytes.Reader.Seek: negative position"}, + {seek: os.SEEK_SET, off: 1 << 33, wantpos: 1 << 33}, + {seek: os.SEEK_CUR, off: 1, wantpos: 1<<33 + 1}, + {seek: os.SEEK_SET, n: 5, want: "01234"}, + {seek: os.SEEK_CUR, n: 5, want: "56789"}, + {seek: os.SEEK_END, off: -1, n: 1, wantpos: 9, want: "9"}, + } + + for i, tt := range tests { + pos, err := r.Seek(tt.off, tt.seek) + if err == nil && tt.seekerr != "" { + t.Errorf("%d. want seek error %q", i, tt.seekerr) + continue + } + if err != nil && err.Error() != tt.seekerr { + t.Errorf("%d. seek error = %q; want %q", i, err.Error(), tt.seekerr) + continue + } + if tt.wantpos != 0 && tt.wantpos != pos { + t.Errorf("%d. pos = %d, want %d", i, pos, tt.wantpos) + } + buf := make([]byte, tt.n) + n, err := r.Read(buf) + if err != nil { + t.Errorf("%d. read = %v", i, err) + continue + } + got := string(buf[:n]) + if got != tt.want { + t.Errorf("%d. got %q; want %q", i, got, tt.want) + } + } +} + +func TestReadAfterBigSeek(t *testing.T) { + r := NewReader([]byte("0123456789")) + if _, err := r.Seek(1<<31+5, os.SEEK_SET); err != nil { + t.Fatal(err) + } + if n, err := r.Read(make([]byte, 10)); n != 0 || err != io.EOF { + t.Errorf("Read = %d, %v; want 0, EOF", n, err) + } +} + +func TestReaderAt(t *testing.T) { + r := NewReader([]byte("0123456789")) + tests := []struct { + off int64 + n int + want string + wanterr interface{} + }{ + {0, 10, "0123456789", nil}, + {1, 10, "123456789", io.EOF}, + {1, 9, "123456789", nil}, + {11, 10, "", io.EOF}, + {0, 0, "", nil}, + {-1, 0, "", "bytes.Reader.ReadAt: negative offset"}, + } + for i, tt := range tests { + b := make([]byte, tt.n) + rn, err := r.ReadAt(b, tt.off) + got := string(b[:rn]) + if got != tt.want { + t.Errorf("%d. got %q; want %q", i, got, tt.want) + } + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.wanterr) { + t.Errorf("%d. got error = %v; want %v", i, err, tt.wanterr) + } + } +} + +func TestReaderAtConcurrent(t *testing.T) { + // Test for the race detector, to verify ReadAt doesn't mutate + // any state. + r := NewReader([]byte("0123456789")) + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + var buf [1]byte + r.ReadAt(buf[:], int64(i)) + }(i) + } + wg.Wait() +} + +func TestEmptyReaderConcurrent(t *testing.T) { + // Test for the race detector, to verify a Read that doesn't yield any bytes + // is okay to use from multiple goroutines. This was our historic behavior. + // See golang.org/issue/7856 + r := NewReader([]byte{}) + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(2) + go func() { + defer wg.Done() + var buf [1]byte + r.Read(buf[:]) + }() + go func() { + defer wg.Done() + r.Read(nil) + }() + } + wg.Wait() +} + +func TestReaderWriteTo(t *testing.T) { + for i := 0; i < 30; i += 3 { + var l int + if i > 0 { + l = len(data) / i + } + s := data[:l] + r := NewReader(testBytes[:l]) + var b Buffer + n, err := r.WriteTo(&b) + if expect := int64(len(s)); n != expect { + t.Errorf("got %v; want %v", n, expect) + } + if err != nil { + t.Errorf("for length %d: got error = %v; want nil", l, err) + } + if b.String() != s { + t.Errorf("got string %q; want %q", b.String(), s) + } + if r.Len() != 0 { + t.Errorf("reader contains %v bytes; want 0", r.Len()) + } + } +} + +func TestReaderLen(t *testing.T) { + const data = "hello world" + r := NewReader([]byte(data)) + if got, want := r.Len(), 11; got != want { + t.Errorf("r.Len(): got %d, want %d", got, want) + } + if n, err := r.Read(make([]byte, 10)); err != nil || n != 10 { + t.Errorf("Read failed: read %d %v", n, err) + } + if got, want := r.Len(), 1; got != want { + t.Errorf("r.Len(): got %d, want %d", got, want) + } + if n, err := r.Read(make([]byte, 1)); err != nil || n != 1 { + t.Errorf("Read failed: read %d %v", n, err) + } + if got, want := r.Len(), 0; got != want { + t.Errorf("r.Len(): got %d, want %d", got, want) + } +} + +var UnreadRuneErrorTests = []struct { + name string + f func(*Reader) +}{ + {"Read", func(r *Reader) { r.Read([]byte{0}) }}, + {"ReadByte", func(r *Reader) { r.ReadByte() }}, + {"UnreadRune", func(r *Reader) { r.UnreadRune() }}, + {"Seek", func(r *Reader) { r.Seek(0, 1) }}, + {"WriteTo", func(r *Reader) { r.WriteTo(&Buffer{}) }}, +} + +func TestUnreadRuneError(t *testing.T) { + for _, tt := range UnreadRuneErrorTests { + reader := NewReader([]byte("0123456789")) + if _, _, err := reader.ReadRune(); err != nil { + // should not happen + t.Fatal(err) + } + tt.f(reader) + err := reader.UnreadRune() + if err == nil { + t.Errorf("Unreading after %s: expected error", tt.name) + } + } +} + +func TestReaderDoubleUnreadRune(t *testing.T) { + buf := NewBuffer([]byte("groucho")) + if _, _, err := buf.ReadRune(); err != nil { + // should not happen + t.Fatal(err) + } + if err := buf.UnreadByte(); err != nil { + // should not happen + t.Fatal(err) + } + if err := buf.UnreadByte(); err == nil { + t.Fatal("UnreadByte: expected error, got nil") + } +} + +// verify that copying from an empty reader always has the same results, +// regardless of the presence of a WriteTo method. +func TestReaderCopyNothing(t *testing.T) { + type nErr struct { + n int64 + err error + } + type justReader struct { + io.Reader + } + type justWriter struct { + io.Writer + } + discard := justWriter{ioutil.Discard} // hide ReadFrom + + var with, withOut nErr + with.n, with.err = io.Copy(discard, NewReader(nil)) + withOut.n, withOut.err = io.Copy(discard, justReader{NewReader(nil)}) + if with != withOut { + t.Errorf("behavior differs: with = %#v; without: %#v", with, withOut) + } +} |