3 files changed, 290 insertions, 109 deletions

diff --git a/src/pkg/bufio/Makefile b/src/pkg/bufio/Makefile
index 1f5fc349b..85430e8e8 100644
--- a/src/pkg/bufio/Makefile
+++ b/src/pkg/bufio/Makefile
@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-include ../../Make.$(GOARCH)
+include ../../Make.inc
 
 TARG=bufio
 GOFILES=\
diff --git a/src/pkg/bufio/bufio.go b/src/pkg/bufio/bufio.go
index 6a73c41ef..c13456a63 100644
--- a/src/pkg/bufio/bufio.go
+++ b/src/pkg/bufio/bufio.go
@@ -27,7 +27,9 @@ type Error struct {
 
 var (
 	ErrInvalidUnreadByte os.Error = &Error{"bufio: invalid use of UnreadByte"}
+	ErrInvalidUnreadRune os.Error = &Error{"bufio: invalid use of UnreadRune"}
 	ErrBufferFull        os.Error = &Error{"bufio: buffer full"}
+	ErrNegativeCount     os.Error = &Error{"bufio: negative count"}
 	errInternal          os.Error = &Error{"bufio: internal error"}
 )
 
@@ -43,11 +45,12 @@ func (b BufSizeError) String() string {
 
 // Reader implements buffering for an io.Reader object.
 type Reader struct {
-	buf      []byte
-	rd       io.Reader
-	r, w     int
-	err      os.Error
-	lastbyte int
+	buf          []byte
+	rd           io.Reader
+	r, w         int
+	err          os.Error
+	lastByte     int
+	lastRuneSize int
 }
 
 // NewReaderSize creates a new Reader whose buffer has the specified size,
@@ -66,7 +69,8 @@ func NewReaderSize(rd io.Reader, size int) (*Reader, os.Error) {
 	b = new(Reader)
 	b.buf = make([]byte, size)
 	b.rd = rd
-	b.lastbyte = -1
+	b.lastByte = -1
+	b.lastRuneSize = -1
 	return b, nil
 }
 
@@ -83,13 +87,11 @@ func NewReader(rd io.Reader) *Reader {
 // fill reads a new chunk into the buffer.
 func (b *Reader) fill() {
 	// Slide existing data to beginning.
-	if b.w > b.r {
-		copy(b.buf[0:b.w-b.r], b.buf[b.r:b.w])
+	if b.r > 0 {
+		copy(b.buf, b.buf[b.r:b.w])
 		b.w -= b.r
-	} else {
-		b.w = 0
+		b.r = 0
 	}
-	b.r = 0
 
 	// Read new data.
 	n, e := b.rd.Read(b.buf[b.w:])
@@ -99,48 +101,75 @@ func (b *Reader) fill() {
 	}
 }
 
+// Peek returns the next n bytes without advancing the reader. The bytes stop
+// being valid at the next read call. If Peek returns fewer than n bytes, it
+// also returns an error explaining why the read is short. The error is
+// ErrBufferFull if n is larger than b's buffer size.
+func (b *Reader) Peek(n int) ([]byte, os.Error) {
+	if n < 0 {
+		return nil, ErrNegativeCount
+	}
+	if n > len(b.buf) {
+		return nil, ErrBufferFull
+	}
+	for b.w-b.r < n && b.err == nil {
+		b.fill()
+	}
+	m := b.w - b.r
+	if m > n {
+		m = n
+	}
+	err := b.err
+	if m < n && err == nil {
+		err = ErrBufferFull
+	}
+	return b.buf[b.r : b.r+m], err
+}
+
 // Read reads data into p.
 // It returns the number of bytes read into p.
-// If nn < len(p), also returns an error explaining
-// why the read is short.  At EOF, the count will be
-// zero and err will be os.EOF.
-func (b *Reader) Read(p []byte) (nn int, err os.Error) {
-	nn = 0
-	for len(p) > 0 {
-		n := len(p)
-		if b.w == b.r {
-			if b.err != nil {
-				return nn, b.err
-			}
-			if len(p) >= len(b.buf) {
-				// Large read, empty buffer.
-				// Read directly into p to avoid copy.
-				n, b.err = b.rd.Read(p)
-				if n > 0 {
-					b.lastbyte = int(p[n-1])
-				}
-				p = p[n:]
-				nn += n
-				continue
+// It calls Read at most once on the underlying Reader,
+// hence n may be less than len(p).
+// At EOF, the count will be zero and err will be os.EOF.
+func (b *Reader) Read(p []byte) (n int, err os.Error) {
+	n = len(p)
+	if n == 0 {
+		return 0, b.err
+	}
+	if b.w == b.r {
+		if b.err != nil {
+			return 0, b.err
+		}
+		if len(p) >= len(b.buf) {
+			// Large read, empty buffer.
+			// Read directly into p to avoid copy.
+			n, b.err = b.rd.Read(p)
+			if n > 0 {
+				b.lastByte = int(p[n-1])
+				b.lastRuneSize = -1
 			}
-			b.fill()
-			continue
+			return n, b.err
 		}
-		if n > b.w-b.r {
-			n = b.w - b.r
+		b.fill()
+		if b.w == b.r {
+			return 0, b.err
 		}
-		copy(p[0:n], b.buf[b.r:b.r+n])
-		p = p[n:]
-		b.r += n
-		b.lastbyte = int(b.buf[b.r-1])
-		nn += n
 	}
-	return nn, nil
+
+	if n > b.w-b.r {
+		n = b.w - b.r
+	}
+	copy(p[0:n], b.buf[b.r:])
+	b.r += n
+	b.lastByte = int(b.buf[b.r-1])
+	b.lastRuneSize = -1
+	return n, nil
 }
 
 // ReadByte reads and returns a single byte.
 // If no byte is available, returns an error.
 func (b *Reader) ReadByte() (c byte, err os.Error) {
+	b.lastRuneSize = -1
 	for b.w == b.r {
 		if b.err != nil {
 			return 0, b.err
@@ -149,24 +178,25 @@ func (b *Reader) ReadByte() (c byte, err os.Error) {
 	}
 	c = b.buf[b.r]
 	b.r++
-	b.lastbyte = int(c)
+	b.lastByte = int(c)
 	return c, nil
 }
 
 // UnreadByte unreads the last byte.  Only the most recently read byte can be unread.
 func (b *Reader) UnreadByte() os.Error {
-	if b.r == b.w && b.lastbyte >= 0 {
+	b.lastRuneSize = -1
+	if b.r == b.w && b.lastByte >= 0 {
 		b.w = 1
 		b.r = 0
-		b.buf[0] = byte(b.lastbyte)
-		b.lastbyte = -1
+		b.buf[0] = byte(b.lastByte)
+		b.lastByte = -1
 		return nil
 	}
 	if b.r <= 0 {
 		return ErrInvalidUnreadByte
 	}
 	b.r--
-	b.lastbyte = -1
+	b.lastByte = -1
 	return nil
 }
 
@@ -176,6 +206,7 @@ func (b *Reader) ReadRune() (rune int, size int, err os.Error) {
 	for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil {
 		b.fill()
 	}
+	b.lastRuneSize = -1
 	if b.r == b.w {
 		return 0, 0, b.err
 	}
@@ -184,10 +215,25 @@ func (b *Reader) ReadRune() (rune int, size int, err os.Error) {
 		rune, size = utf8.DecodeRune(b.buf[b.r:b.w])
 	}
 	b.r += size
-	b.lastbyte = int(b.buf[b.r-1])
+	b.lastByte = int(b.buf[b.r-1])
+	b.lastRuneSize = size
 	return rune, size, nil
 }
 
+// UnreadRune unreads the last rune.  If the most recent read operation on
+// the buffer was not a ReadRune, UnreadRune returns an error.  (In this
+// regard it is stricter than UnreadByte, which will unread the last byte
+// from any read operation.)
+func (b *Reader) UnreadRune() os.Error {
+	if b.lastRuneSize < 0 || b.r == 0 {
+		return ErrInvalidUnreadRune
+	}
+	b.r -= b.lastRuneSize
+	b.lastByte = -1
+	b.lastRuneSize = -1
+	return nil
+}
+
 // Buffered returns the number of bytes that can be read from the current buffer.
 func (b *Reader) Buffered() int { return b.w - b.r }
 
@@ -237,7 +283,7 @@ func (b *Reader) ReadSlice(delim byte) (line []byte, err os.Error) {
 }
 
 // ReadBytes reads until the first occurrence of delim in the input,
-// returning a string containing the data up to and including the delimiter.
+// returning a slice containing the data up to and including the delimiter.
 // If ReadBytes encounters an error before finding a delimiter,
 // it returns the data read before the error and the error itself (often os.EOF).
 // ReadBytes returns err != nil if and only if line does not end in delim.
@@ -246,7 +292,6 @@ func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) {
 	// accumulating full buffers.
 	var frag []byte
 	var full [][]byte
-	nfull := 0
 	err = nil
 
 	for {
@@ -263,26 +308,12 @@ func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) {
 		// Make a copy of the buffer.
 		buf := make([]byte, len(frag))
 		copy(buf, frag)
-
-		// Grow list if needed.
-		if full == nil {
-			full = make([][]byte, 16)
-		} else if nfull >= len(full) {
-			newfull := make([][]byte, len(full)*2)
-			for i := 0; i < len(full); i++ {
-				newfull[i] = full[i]
-			}
-			full = newfull
-		}
-
-		// Save buffer
-		full[nfull] = buf
-		nfull++
+		full = append(full, buf)
 	}
 
 	// Allocate new buffer to hold the full pieces and the fragment.
 	n := 0
-	for i := 0; i < nfull; i++ {
+	for i := range full {
 		n += len(full[i])
 	}
 	n += len(frag)
@@ -290,11 +321,10 @@ func (b *Reader) ReadBytes(delim byte) (line []byte, err os.Error) {
 	// Copy full pieces and fragment in.
 	buf := make([]byte, n)
 	n = 0
-	for i := 0; i < nfull; i++ {
-		copy(buf[n:n+len(full[i])], full[i])
-		n += len(full[i])
+	for i := range full {
+		n += copy(buf[n:], full[i])
 	}
-	copy(buf[n:n+len(frag)], frag)
+	copy(buf[n:], frag)
 	return buf, err
 }
 
@@ -392,7 +422,7 @@ func (b *Writer) Write(p []byte) (nn int, err os.Error) {
 			}
 			n = b.Available()
 		}
-		if b.Available() == 0 && len(p) >= len(b.buf) {
+		if b.Buffered() == 0 && len(p) >= len(b.buf) {
 			// Large write, empty buffer.
 			// Write directly from p to avoid copy.
 			n, b.err = b.wr.Write(p)
@@ -451,7 +481,7 @@ func (b *Writer) WriteRune(rune int) (size int, err os.Error) {
 			return b.WriteString(string(rune))
 		}
 	}
-	size = utf8.EncodeRune(rune, b.buf[b.n:])
+	size = utf8.EncodeRune(b.buf[b.n:], rune)
 	b.n += size
 	return size, nil
 }
diff --git a/src/pkg/bufio/bufio_test.go b/src/pkg/bufio/bufio_test.go
index 2279fe3b1..059ca6dd2 100644
--- a/src/pkg/bufio/bufio_test.go
+++ b/src/pkg/bufio/bufio_test.go
@@ -69,7 +69,7 @@ func TestReaderSimple(t *testing.T) {
 
 	b = NewReader(newRot13Reader(bytes.NewBufferString(data)))
 	if s := readBytes(b); s != "uryyb jbeyq" {
-		t.Error("rot13 hello world test failed: got %q", s)
+		t.Errorf("rot13 hello world test failed: got %q", s)
 	}
 }
 
@@ -80,10 +80,10 @@ type readMaker struct {
 }
 
 var readMakers = []readMaker{
-	readMaker{"full", func(r io.Reader) io.Reader { return r }},
-	readMaker{"byte", iotest.OneByteReader},
-	readMaker{"half", iotest.HalfReader},
-	readMaker{"data+err", iotest.DataErrReader},
+	{"full", func(r io.Reader) io.Reader { return r }},
+	{"byte", iotest.OneByteReader},
+	{"half", iotest.HalfReader},
+	{"data+err", iotest.DataErrReader},
 }
 
 // Call ReadString (which ends up calling everything else)
@@ -123,14 +123,14 @@ type bufReader struct {
 }
 
 var bufreaders = []bufReader{
-	bufReader{"1", func(b *Reader) string { return reads(b, 1) }},
-	bufReader{"2", func(b *Reader) string { return reads(b, 2) }},
-	bufReader{"3", func(b *Reader) string { return reads(b, 3) }},
-	bufReader{"4", func(b *Reader) string { return reads(b, 4) }},
-	bufReader{"5", func(b *Reader) string { return reads(b, 5) }},
-	bufReader{"7", func(b *Reader) string { return reads(b, 7) }},
-	bufReader{"bytes", readBytes},
-	bufReader{"lines", readLines},
+	{"1", func(b *Reader) string { return reads(b, 1) }},
+	{"2", func(b *Reader) string { return reads(b, 2) }},
+	{"3", func(b *Reader) string { return reads(b, 3) }},
+	{"4", func(b *Reader) string { return reads(b, 4) }},
+	{"5", func(b *Reader) string { return reads(b, 5) }},
+	{"7", func(b *Reader) string { return reads(b, 7) }},
+	{"bytes", readBytes},
+	{"lines", readLines},
 }
 
 var bufsizes = []int{
@@ -179,10 +179,7 @@ type StringReader struct {
 func (r *StringReader) Read(p []byte) (n int, err os.Error) {
 	if r.step < len(r.data) {
 		s := r.data[r.step]
-		for i := 0; i < len(s); i++ {
-			p[i] = s[i]
-		}
-		n = len(s)
+		n = copy(p, s)
 		r.step++
 	} else {
 		err = os.EOF
@@ -210,14 +207,14 @@ func readRuneSegments(t *testing.T, segments []string) {
 }
 
 var segmentList = [][]string{
-	[]string{},
-	[]string{""},
-	[]string{"日", "本語"},
-	[]string{"\u65e5", "\u672c", "\u8a9e"},
-	[]string{"\U000065e5", "\U0000672c", "\U00008a9e"},
-	[]string{"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"},
-	[]string{"Hello", ", ", "World", "!"},
-	[]string{"Hello", ", ", "", "World", "!"},
+	{},
+	{""},
+	{"日", "本語"},
+	{"\u65e5", "\u672c", "\u8a9e"},
+	{"\U000065e5", "\U0000672c", "\U00008a9e"},
+	{"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"},
+	{"Hello", ", ", "World", "!"},
+	{"Hello", ", ", "", "World", "!"},
 }
 
 func TestReadRune(t *testing.T) {
@@ -226,6 +223,113 @@ func TestReadRune(t *testing.T) {
 	}
 }
 
+func TestUnreadRune(t *testing.T) {
+	got := ""
+	segments := []string{"Hello, world:", "日本語"}
+	data := strings.Join(segments, "")
+	r := NewReader(&StringReader{data: segments})
+	// Normal execution.
+	for {
+		rune, _, err := r.ReadRune()
+		if err != nil {
+			if err != os.EOF {
+				t.Error("unexpected EOF")
+			}
+			break
+		}
+		got += string(rune)
+		// Put it back and read it again
+		if err = r.UnreadRune(); err != nil {
+			t.Error("unexpected error on UnreadRune:", err)
+		}
+		rune1, _, err := r.ReadRune()
+		if err != nil {
+			t.Error("unexpected error reading after unreading:", err)
+		}
+		if rune != rune1 {
+			t.Errorf("incorrect rune after unread: got %c wanted %c", rune1, rune)
+		}
+	}
+	if got != data {
+		t.Errorf("want=%q got=%q", data, got)
+	}
+}
+
+// Test that UnreadRune fails if the preceding operation was not a ReadRune.
+func TestUnreadRuneError(t *testing.T) {
+	buf := make([]byte, 3) // All runes in this test are 3 bytes long
+	r := NewReader(&StringReader{data: []string{"日本語日本語日本語"}})
+	if r.UnreadRune() == nil {
+		t.Error("expected error on UnreadRune from fresh buffer")
+	}
+	_, _, err := r.ReadRune()
+	if err != nil {
+		t.Error("unexpected error on ReadRune (1):", err)
+	}
+	if err = r.UnreadRune(); err != nil {
+		t.Error("unexpected error on UnreadRune (1):", err)
+	}
+	if r.UnreadRune() == nil {
+		t.Error("expected error after UnreadRune (1)")
+	}
+	// Test error after Read.
+	_, _, err = r.ReadRune() // reset state
+	if err != nil {
+		t.Error("unexpected error on ReadRune (2):", err)
+	}
+	_, err = r.Read(buf)
+	if err != nil {
+		t.Error("unexpected error on Read (2):", err)
+	}
+	if r.UnreadRune() == nil {
+		t.Error("expected error after Read (2)")
+	}
+	// Test error after ReadByte.
+	_, _, err = r.ReadRune() // reset state
+	if err != nil {
+		t.Error("unexpected error on ReadRune (2):", err)
+	}
+	for _ = range buf {
+		_, err = r.ReadByte()
+		if err != nil {
+			t.Error("unexpected error on ReadByte (2):", err)
+		}
+	}
+	if r.UnreadRune() == nil {
+		t.Error("expected error after ReadByte")
+	}
+	// Test error after UnreadByte.
+	_, _, err = r.ReadRune() // reset state
+	if err != nil {
+		t.Error("unexpected error on ReadRune (3):", err)
+	}
+	_, err = r.ReadByte()
+	if err != nil {
+		t.Error("unexpected error on ReadByte (3):", err)
+	}
+	err = r.UnreadByte()
+	if err != nil {
+		t.Error("unexpected error on UnreadByte (3):", err)
+	}
+	if r.UnreadRune() == nil {
+		t.Error("expected error after UnreadByte (3)")
+	}
+}
+
+func TestUnreadRuneAtEOF(t *testing.T) {
+	// UnreadRune/ReadRune should error at EOF (was a bug; used to panic)
+	r := NewReader(strings.NewReader("x"))
+	r.ReadRune()
+	r.ReadRune()
+	r.UnreadRune()
+	_, _, err := r.ReadRune()
+	if err == nil {
+		t.Error("expected error at EOF")
+	} else if err != os.EOF {
+		t.Error("expected EOF; got", err)
+	}
+}
+
 func TestReadWriteRune(t *testing.T) {
 	const NRune = 1000
 	byteBuf := new(bytes.Buffer)
@@ -233,7 +337,7 @@ func TestReadWriteRune(t *testing.T) {
 	// Write the runes out using WriteRune
 	buf := make([]byte, utf8.UTFMax)
 	for rune := 0; rune < NRune; rune++ {
-		size := utf8.EncodeRune(rune, buf)
+		size := utf8.EncodeRune(buf, rune)
 		nbytes, err := w.WriteRune(rune)
 		if err != nil {
 			t.Fatalf("WriteRune(0x%x) error: %s", rune, err)
@@ -247,7 +351,7 @@ func TestReadWriteRune(t *testing.T) {
 	r := NewReader(byteBuf)
 	// Read them back with ReadRune
 	for rune := 0; rune < NRune; rune++ {
-		size := utf8.EncodeRune(rune, buf)
+		size := utf8.EncodeRune(buf, rune)
 		nr, nbytes, err := r.ReadRune()
 		if nr != rune || nbytes != size || err != nil {
 			t.Fatalf("ReadRune(0x%x) got 0x%x,%d not 0x%x,%d (err=%s)", r, nr, nbytes, r, size, err)
@@ -293,9 +397,9 @@ func TestWriter(t *testing.T) {
 			}
 			for l := 0; l < len(written); l++ {
 				if written[i] != data[i] {
-					t.Errorf("%s: wrong bytes written")
-					t.Errorf("want=%s", data[0:len(written)])
-					t.Errorf("have=%s", written)
+					t.Errorf("wrong bytes written")
+					t.Errorf("want=%q", data[0:len(written)])
+					t.Errorf("have=%q", written)
 				}
 			}
 		}
@@ -315,12 +419,12 @@ func (w errorWriterTest) Write(p []byte) (int, os.Error) {
 }
 
 var errorWriterTests = []errorWriterTest{
-	errorWriterTest{0, 1, nil, io.ErrShortWrite},
-	errorWriterTest{1, 2, nil, io.ErrShortWrite},
-	errorWriterTest{1, 1, nil, nil},
-	errorWriterTest{0, 1, os.EPIPE, os.EPIPE},
-	errorWriterTest{1, 2, os.EPIPE, os.EPIPE},
-	errorWriterTest{1, 1, os.EPIPE, os.EPIPE},
+	{0, 1, nil, io.ErrShortWrite},
+	{1, 2, nil, io.ErrShortWrite},
+	{1, 1, nil, nil},
+	{0, 1, os.EPIPE, os.EPIPE},
+	{1, 2, os.EPIPE, os.EPIPE},
+	{1, 1, os.EPIPE, os.EPIPE},
 }
 
 func TestWriteErrors(t *testing.T) {
@@ -419,3 +523,50 @@ func TestBufferFull(t *testing.T) {
 		t.Errorf("second ReadSlice(,) = %q, %v", line, err)
 	}
 }
+
+func TestPeek(t *testing.T) {
+	p := make([]byte, 10)
+	buf, _ := NewReaderSize(strings.NewReader("abcdefghij"), 4)
+	if s, err := buf.Peek(1); string(s) != "a" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "a", string(s), err)
+	}
+	if s, err := buf.Peek(4); string(s) != "abcd" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "abcd", string(s), err)
+	}
+	if _, err := buf.Peek(5); err != ErrBufferFull {
+		t.Fatalf("want ErrBufFull got %v", err)
+	}
+	if _, err := buf.Read(p[0:3]); string(p[0:3]) != "abc" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "abc", string(p[0:3]), err)
+	}
+	if s, err := buf.Peek(1); string(s) != "d" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "d", string(s), err)
+	}
+	if s, err := buf.Peek(2); string(s) != "de" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "de", string(s), err)
+	}
+	if _, err := buf.Read(p[0:3]); string(p[0:3]) != "def" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "def", string(p[0:3]), err)
+	}
+	if s, err := buf.Peek(4); string(s) != "ghij" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "ghij", string(s), err)
+	}
+	if _, err := buf.Read(p[0:4]); string(p[0:4]) != "ghij" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "ghij", string(p[0:3]), err)
+	}
+	if s, err := buf.Peek(0); string(s) != "" || err != nil {
+		t.Fatalf("want %q got %q, err=%v", "", string(s), err)
+	}
+	if _, err := buf.Peek(1); err != os.EOF {
+		t.Fatalf("want EOF got %v", err)
+	}
+}
+
+func TestPeekThenUnreadRune(t *testing.T) {
+	// This sequence used to cause a crash.
+	r := NewReader(strings.NewReader("x"))
+	r.ReadRune()
+	r.Peek(1)
+	r.UnreadRune()
+	r.ReadRune() // Used to panic here
+}