diff options
Diffstat (limited to 'src/pkg/io')
-rw-r--r-- | src/pkg/io/Makefile | 13 | ||||
-rw-r--r-- | src/pkg/io/io.go | 439 | ||||
-rw-r--r-- | src/pkg/io/io_test.go | 179 | ||||
-rw-r--r-- | src/pkg/io/ioutil/Makefile | 12 | ||||
-rw-r--r-- | src/pkg/io/ioutil/ioutil.go | 130 | ||||
-rw-r--r-- | src/pkg/io/ioutil/ioutil_test.go | 91 | ||||
-rw-r--r-- | src/pkg/io/ioutil/tempfile.go | 91 | ||||
-rw-r--r-- | src/pkg/io/ioutil/tempfile_test.go | 54 | ||||
-rw-r--r-- | src/pkg/io/multi.go | 58 | ||||
-rw-r--r-- | src/pkg/io/multi_test.go | 89 | ||||
-rw-r--r-- | src/pkg/io/pipe.go | 182 | ||||
-rw-r--r-- | src/pkg/io/pipe_test.go | 271 |
12 files changed, 1609 insertions, 0 deletions
diff --git a/src/pkg/io/Makefile b/src/pkg/io/Makefile new file mode 100644 index 000000000..9786002f0 --- /dev/null +++ b/src/pkg/io/Makefile @@ -0,0 +1,13 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../Make.inc + +TARG=io +GOFILES=\ + io.go\ + multi.go\ + pipe.go\ + +include ../../Make.pkg diff --git a/src/pkg/io/io.go b/src/pkg/io/io.go new file mode 100644 index 000000000..b879fe5b7 --- /dev/null +++ b/src/pkg/io/io.go @@ -0,0 +1,439 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package io provides basic interfaces to I/O primitives. +// Its primary job is to wrap existing implementations of such primitives, +// such as those in package os, into shared public interfaces that +// abstract the functionality, plus some other related primitives. +package io + +import "os" + +// Error represents an unexpected I/O behavior. +type Error struct { + ErrorString string +} + +func (err *Error) String() string { return err.ErrorString } + +// ErrShortWrite means that a write accepted fewer bytes than requested +// but failed to return an explicit error. +var ErrShortWrite os.Error = &Error{"short write"} + +// ErrShortBuffer means that a read required a longer buffer than was provided. +var ErrShortBuffer os.Error = &Error{"short buffer"} + +// ErrUnexpectedEOF means that os.EOF was encountered in the +// middle of reading a fixed-size block or data structure. +var ErrUnexpectedEOF os.Error = &Error{"unexpected EOF"} + +// Reader is the interface that wraps the basic Read method. +// +// Read reads up to len(p) bytes into p. It returns the number of bytes +// read (0 <= n <= len(p)) and any error encountered. Even if Read +// returns n < len(p), it may use all of p as scratch space during the call. +// If some data is available but not len(p) bytes, Read conventionally +// returns what is available instead of waiting for more. +// +// When Read encounters an error or end-of-file condition after +// successfully reading n > 0 bytes, it returns the number of +// bytes read. It may return the (non-nil) error from the same call +// or return the error (and n == 0) from a subsequent call. +// An instance of this general case is that a Reader returning +// a non-zero number of bytes at the end of the input stream may +// return either err == os.EOF or err == nil. The next Read should +// return 0, os.EOF regardless. +// +// Callers should always process the n > 0 bytes returned before +// considering the error err. Doing so correctly handles I/O errors +// that happen after reading some bytes and also both of the +// allowed EOF behaviors. +type Reader interface { + Read(p []byte) (n int, err os.Error) +} + +// Writer is the interface that wraps the basic Write method. +// +// Write writes len(p) bytes from p to the underlying data stream. +// It returns the number of bytes written from p (0 <= n <= len(p)) +// and any error encountered that caused the write to stop early. +// Write must return a non-nil error if it returns n < len(p). +type Writer interface { + Write(p []byte) (n int, err os.Error) +} + +// Closer is the interface that wraps the basic Close method. +type Closer interface { + Close() os.Error +} + +// Seeker is the interface that wraps the basic Seek method. +// +// Seek sets the offset for the next Read or Write to offset, +// interpreted according to whence: 0 means relative to the origin of +// the file, 1 means relative to the current offset, and 2 means +// relative to the end. Seek returns the new offset and an Error, if +// any. +type Seeker interface { + Seek(offset int64, whence int) (ret int64, err os.Error) +} + +// ReadWriter is the interface that groups the basic Read and Write methods. +type ReadWriter interface { + Reader + Writer +} + +// ReadCloser is the interface that groups the basic Read and Close methods. +type ReadCloser interface { + Reader + Closer +} + +// WriteCloser is the interface that groups the basic Write and Close methods. +type WriteCloser interface { + Writer + Closer +} + +// ReadWriteCloser is the interface that groups the basic Read, Write and Close methods. +type ReadWriteCloser interface { + Reader + Writer + Closer +} + +// ReadSeeker is the interface that groups the basic Read and Seek methods. +type ReadSeeker interface { + Reader + Seeker +} + +// WriteSeeker is the interface that groups the basic Write and Seek methods. +type WriteSeeker interface { + Writer + Seeker +} + +// ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods. +type ReadWriteSeeker interface { + Reader + Writer + Seeker +} + +// ReaderFrom is the interface that wraps the ReadFrom method. +type ReaderFrom interface { + ReadFrom(r Reader) (n int64, err os.Error) +} + +// WriterTo is the interface that wraps the WriteTo method. +type WriterTo interface { + WriteTo(w Writer) (n int64, err os.Error) +} + +// ReaderAt is the interface that wraps the basic ReadAt method. +// +// ReadAt reads len(p) bytes into p starting at offset off in the +// underlying input source. It returns the number of bytes +// read (0 <= n <= len(p)) and any error encountered. +// +// When ReadAt returns n < len(p), it returns a non-nil error +// explaining why more bytes were not returned. In this respect, +// ReadAt is stricter than Read. +// +// Even if ReadAt returns n < len(p), it may use all of p as scratch +// space during the call. If some data is available but not len(p) bytes, +// ReadAt blocks until either all the data is available or an error occurs. +// In this respect ReadAt is different from Read. +// +// If the n = len(p) bytes returned by ReadAt are at the end of the +// input source, ReadAt may return either err == os.EOF or err == nil. +// +// If ReadAt is reading from an input source with a seek offset, +// ReadAt should not affect nor be affected by the underlying +// seek offset. +type ReaderAt interface { + ReadAt(p []byte, off int64) (n int, err os.Error) +} + +// WriterAt is the interface that wraps the basic WriteAt method. +// +// WriteAt writes len(p) bytes from p to the underlying data stream +// at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) +// and any error encountered that caused the write to stop early. +// WriteAt must return a non-nil error if it returns n < len(p). +type WriterAt interface { + WriteAt(p []byte, off int64) (n int, err os.Error) +} + +// ByteReader is the interface that wraps the ReadByte method. +// +// ReadByte reads and returns the next byte from the input. +// If no byte is available, err will be set. +type ByteReader interface { + ReadByte() (c byte, err os.Error) +} + +// ByteScanner is the interface that adds the UnreadByte method to the +// basic ReadByte method. +// +// UnreadByte causes the next call to ReadByte to return the same byte +// as the previous call to ReadByte. +// It may be an error to call UnreadByte twice without an intervening +// call to ReadByte. +type ByteScanner interface { + ByteReader + UnreadByte() os.Error +} + +// RuneReader is the interface that wraps the ReadRune method. +// +// ReadRune reads a single UTF-8 encoded Unicode character +// and returns the rune and its size in bytes. If no character is +// available, err will be set. +type RuneReader interface { + ReadRune() (rune int, size int, err os.Error) +} + +// RuneScanner is the interface that adds the UnreadRune method to the +// basic ReadRune method. +// +// UnreadRune causes the next call to ReadRune to return the same rune +// as the previous call to ReadRune. +// It may be an error to call UnreadRune twice without an intervening +// call to ReadRune. +type RuneScanner interface { + RuneReader + UnreadRune() os.Error +} + +// stringWriter is the interface that wraps the WriteString method. +type stringWriter interface { + WriteString(s string) (n int, err os.Error) +} + +// WriteString writes the contents of the string s to w, which accepts an array of bytes. +func WriteString(w Writer, s string) (n int, err os.Error) { + if sw, ok := w.(stringWriter); ok { + return sw.WriteString(s) + } + return w.Write([]byte(s)) +} + +// ReadAtLeast reads from r into buf until it has read at least min bytes. +// It returns the number of bytes copied and an error if fewer bytes were read. +// The error is os.EOF only if no bytes were read. +// If an EOF happens after reading fewer than min bytes, +// ReadAtLeast returns ErrUnexpectedEOF. +// If min is greater than the length of buf, ReadAtLeast returns ErrShortBuffer. +func ReadAtLeast(r Reader, buf []byte, min int) (n int, err os.Error) { + if len(buf) < min { + return 0, ErrShortBuffer + } + for n < min && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + n += nn + } + if err == os.EOF { + if n >= min { + err = nil + } else if n > 0 { + err = ErrUnexpectedEOF + } + } + return +} + +// ReadFull reads exactly len(buf) bytes from r into buf. +// It returns the number of bytes copied and an error if fewer bytes were read. +// The error is os.EOF only if no bytes were read. +// If an EOF happens after reading some but not all the bytes, +// ReadFull returns ErrUnexpectedEOF. +func ReadFull(r Reader, buf []byte) (n int, err os.Error) { + return ReadAtLeast(r, buf, len(buf)) +} + +// Copyn copies n bytes (or until an error) from src to dst. +// It returns the number of bytes copied and the earliest +// error encountered while copying. Because Read can +// return the full amount requested as well as an error +// (including os.EOF), so can Copyn. +// +// If dst implements the ReaderFrom interface, +// the copy is implemented by calling dst.ReadFrom(src). +func Copyn(dst Writer, src Reader, n int64) (written int64, err os.Error) { + // If the writer has a ReadFrom method, use it to do the copy. + // Avoids a buffer allocation and a copy. + if rt, ok := dst.(ReaderFrom); ok { + written, err = rt.ReadFrom(LimitReader(src, n)) + if written < n && err == nil { + // rt stopped early; must have been EOF. + err = os.EOF + } + return + } + buf := make([]byte, 32*1024) + for written < n { + l := len(buf) + if d := n - written; d < int64(l) { + l = int(d) + } + nr, er := src.Read(buf[0:l]) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = ErrShortWrite + break + } + } + if er != nil { + err = er + break + } + } + return written, err +} + +// Copy copies from src to dst until either EOF is reached +// on src or an error occurs. It returns the number of bytes +// copied and the first error encountered while copying, if any. +// +// A successful Copy returns err == nil, not err == os.EOF. +// Because Copy is defined to read from src until EOF, it does +// not treat an EOF from Read as an error to be reported. +// +// If dst implements the ReaderFrom interface, +// the copy is implemented by calling dst.ReadFrom(src). +// Otherwise, if src implements the WriterTo interface, +// the copy is implemented by calling src.WriteTo(dst). +func Copy(dst Writer, src Reader) (written int64, err os.Error) { + // If the writer has a ReadFrom method, use it to do the copy. + // Avoids an allocation and a copy. + if rt, ok := dst.(ReaderFrom); ok { + return rt.ReadFrom(src) + } + // Similarly, if the reader has a WriteTo method, use it to do the copy. + if wt, ok := src.(WriterTo); ok { + return wt.WriteTo(dst) + } + buf := make([]byte, 32*1024) + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = ErrShortWrite + break + } + } + if er == os.EOF { + break + } + if er != nil { + err = er + break + } + } + return written, err +} + +// LimitReader returns a Reader that reads from r +// but stops with os.EOF after n bytes. +// The underlying implementation is a *LimitedReader. +func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } + +// A LimitedReader reads from R but limits the amount of +// data returned to just N bytes. Each call to Read +// updates N to reflect the new amount remaining. +type LimitedReader struct { + R Reader // underlying reader + N int64 // max bytes remaining +} + +func (l *LimitedReader) Read(p []byte) (n int, err os.Error) { + if l.N <= 0 { + return 0, os.EOF + } + if int64(len(p)) > l.N { + p = p[0:l.N] + } + n, err = l.R.Read(p) + l.N -= int64(n) + return +} + +// NewSectionReader returns a SectionReader that reads from r +// starting at offset off and stops with os.EOF after n bytes. +func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader { + return &SectionReader{r, off, off, off + n} +} + +// SectionReader implements Read, Seek, and ReadAt on a section +// of an underlying ReaderAt. +type SectionReader struct { + r ReaderAt + base int64 + off int64 + limit int64 +} + +func (s *SectionReader) Read(p []byte) (n int, err os.Error) { + if s.off >= s.limit { + return 0, os.EOF + } + if max := s.limit - s.off; int64(len(p)) > max { + p = p[0:max] + } + n, err = s.r.ReadAt(p, s.off) + s.off += int64(n) + return +} + +func (s *SectionReader) Seek(offset int64, whence int) (ret int64, err os.Error) { + switch whence { + default: + return 0, os.EINVAL + case 0: + offset += s.base + case 1: + offset += s.off + case 2: + offset += s.limit + } + if offset < s.base || offset > s.limit { + return 0, os.EINVAL + } + s.off = offset + return offset - s.base, nil +} + +func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err os.Error) { + if off < 0 || off >= s.limit-s.base { + return 0, os.EOF + } + off += s.base + if max := s.limit - off; int64(len(p)) > max { + p = p[0:max] + } + return s.r.ReadAt(p, off) +} + +// Size returns the size of the section in bytes. +func (s *SectionReader) Size() int64 { return s.limit - s.base } diff --git a/src/pkg/io/io_test.go b/src/pkg/io/io_test.go new file mode 100644 index 000000000..bc4f354af --- /dev/null +++ b/src/pkg/io/io_test.go @@ -0,0 +1,179 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package io_test + +import ( + "bytes" + . "io" + "os" + "strings" + "testing" +) + +// An version of bytes.Buffer without ReadFrom and WriteTo +type Buffer struct { + bytes.Buffer + ReaderFrom // conflicts with and hides bytes.Buffer's ReaderFrom. + WriterTo // conflicts with and hides bytes.Buffer's WriterTo. +} + +// Simple tests, primarily to verify the ReadFrom and WriteTo callouts inside Copy and Copyn. + +func TestCopy(t *testing.T) { + rb := new(Buffer) + wb := new(Buffer) + rb.WriteString("hello, world.") + Copy(wb, rb) + if wb.String() != "hello, world." { + t.Errorf("Copy did not work properly") + } +} + +func TestCopyReadFrom(t *testing.T) { + rb := new(Buffer) + wb := new(bytes.Buffer) // implements ReadFrom. + rb.WriteString("hello, world.") + Copy(wb, rb) + if wb.String() != "hello, world." { + t.Errorf("Copy did not work properly") + } +} + +func TestCopyWriteTo(t *testing.T) { + rb := new(bytes.Buffer) // implements WriteTo. + wb := new(Buffer) + rb.WriteString("hello, world.") + Copy(wb, rb) + if wb.String() != "hello, world." { + t.Errorf("Copy did not work properly") + } +} + +func TestCopyn(t *testing.T) { + rb := new(Buffer) + wb := new(Buffer) + rb.WriteString("hello, world.") + Copyn(wb, rb, 5) + if wb.String() != "hello" { + t.Errorf("Copyn did not work properly") + } +} + +func TestCopynReadFrom(t *testing.T) { + rb := new(Buffer) + wb := new(bytes.Buffer) // implements ReadFrom. + rb.WriteString("hello") + Copyn(wb, rb, 5) + if wb.String() != "hello" { + t.Errorf("Copyn did not work properly") + } +} + +func TestCopynWriteTo(t *testing.T) { + rb := new(bytes.Buffer) // implements WriteTo. + wb := new(Buffer) + rb.WriteString("hello, world.") + Copyn(wb, rb, 5) + if wb.String() != "hello" { + t.Errorf("Copyn did not work properly") + } +} + +type noReadFrom struct { + w Writer +} + +func (w *noReadFrom) Write(p []byte) (n int, err os.Error) { + return w.w.Write(p) +} + +func TestCopynEOF(t *testing.T) { + // Test that EOF behavior is the same regardless of whether + // argument to Copyn has ReadFrom. + + b := new(bytes.Buffer) + + n, err := Copyn(&noReadFrom{b}, strings.NewReader("foo"), 3) + if n != 3 || err != nil { + t.Errorf("Copyn(noReadFrom, foo, 3) = %d, %v; want 3, nil", n, err) + } + + n, err = Copyn(&noReadFrom{b}, strings.NewReader("foo"), 4) + if n != 3 || err != os.EOF { + t.Errorf("Copyn(noReadFrom, foo, 4) = %d, %v; want 3, EOF", n, err) + } + + n, err = Copyn(b, strings.NewReader("foo"), 3) // b has read from + if n != 3 || err != nil { + t.Errorf("Copyn(bytes.Buffer, foo, 3) = %d, %v; want 3, nil", n, err) + } + + n, err = Copyn(b, strings.NewReader("foo"), 4) // b has read from + if n != 3 || err != os.EOF { + t.Errorf("Copyn(bytes.Buffer, foo, 4) = %d, %v; want 3, EOF", n, err) + } +} + +func TestReadAtLeast(t *testing.T) { + var rb bytes.Buffer + testReadAtLeast(t, &rb) +} + +// A version of bytes.Buffer that returns n > 0, os.EOF on Read +// when the input is exhausted. +type dataAndEOFBuffer struct { + bytes.Buffer +} + +func (r *dataAndEOFBuffer) Read(p []byte) (n int, err os.Error) { + n, err = r.Buffer.Read(p) + if n > 0 && r.Buffer.Len() == 0 && err == nil { + err = os.EOF + } + return +} + +func TestReadAtLeastWithDataAndEOF(t *testing.T) { + var rb dataAndEOFBuffer + testReadAtLeast(t, &rb) +} + +func testReadAtLeast(t *testing.T, rb ReadWriter) { + rb.Write([]byte("0123")) + buf := make([]byte, 2) + n, err := ReadAtLeast(rb, buf, 2) + if err != nil { + t.Error(err) + } + n, err = ReadAtLeast(rb, buf, 4) + if err != ErrShortBuffer { + t.Errorf("expected ErrShortBuffer got %v", err) + } + if n != 0 { + t.Errorf("expected to have read 0 bytes, got %v", n) + } + n, err = ReadAtLeast(rb, buf, 1) + if err != nil { + t.Error(err) + } + if n != 2 { + t.Errorf("expected to have read 2 bytes, got %v", n) + } + n, err = ReadAtLeast(rb, buf, 2) + if err != os.EOF { + t.Errorf("expected EOF, got %v", err) + } + if n != 0 { + t.Errorf("expected to have read 0 bytes, got %v", n) + } + rb.Write([]byte("4")) + n, err = ReadAtLeast(rb, buf, 2) + if err != ErrUnexpectedEOF { + t.Errorf("expected ErrUnexpectedEOF, got %v", err) + } + if n != 1 { + t.Errorf("expected to have read 1 bytes, got %v", n) + } +} diff --git a/src/pkg/io/ioutil/Makefile b/src/pkg/io/ioutil/Makefile new file mode 100644 index 000000000..d406d4b7d --- /dev/null +++ b/src/pkg/io/ioutil/Makefile @@ -0,0 +1,12 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=io/ioutil +GOFILES=\ + ioutil.go\ + tempfile.go\ + +include ../../../Make.pkg diff --git a/src/pkg/io/ioutil/ioutil.go b/src/pkg/io/ioutil/ioutil.go new file mode 100644 index 000000000..fffa1320f --- /dev/null +++ b/src/pkg/io/ioutil/ioutil.go @@ -0,0 +1,130 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ioutil implements some I/O utility functions. +package ioutil + +import ( + "bytes" + "io" + "os" + "sort" +) + +// readAll reads from r until an error or EOF and returns the data it read +// from the internal buffer allocated with a specified capacity. +func readAll(r io.Reader, capacity int64) ([]byte, os.Error) { + buf := bytes.NewBuffer(make([]byte, 0, capacity)) + _, err := buf.ReadFrom(r) + return buf.Bytes(), err +} + +// ReadAll reads from r until an error or EOF and returns the data it read. +func ReadAll(r io.Reader) ([]byte, os.Error) { + return readAll(r, bytes.MinRead) +} + +// ReadFile reads the file named by filename and returns the contents. +func ReadFile(filename string) ([]byte, os.Error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + fi, err := f.Stat() + var n int64 + if err == nil && fi.Size < 2e9 { // Don't preallocate a huge buffer, just in case. + n = fi.Size + } + // As initial capacity for readAll, use n + a little extra in case Size is zero, + // and to avoid another allocation after Read has filled the buffer. The readAll + // call will read into its allocated internal buffer cheaply. If the size was + // wrong, we'll either waste some space off the end or reallocate as needed, but + // in the overwhelmingly common case we'll get it just right. + return readAll(f, n+bytes.MinRead) +} + +// WriteFile writes data to a file named by filename. +// If the file does not exist, WriteFile creates it with permissions perm; +// otherwise WriteFile truncates it before writing. +func WriteFile(filename string, data []byte, perm uint32) os.Error { + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + n, err := f.Write(data) + f.Close() + if err == nil && n < len(data) { + err = io.ErrShortWrite + } + return err +} + +// A fileInfoList implements sort.Interface. +type fileInfoList []*os.FileInfo + +func (f fileInfoList) Len() int { return len(f) } +func (f fileInfoList) Less(i, j int) bool { return f[i].Name < f[j].Name } +func (f fileInfoList) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// ReadDir reads the directory named by dirname and returns +// a list of sorted directory entries. +func ReadDir(dirname string) ([]*os.FileInfo, os.Error) { + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + fi := make(fileInfoList, len(list)) + for i := range list { + fi[i] = &list[i] + } + sort.Sort(fi) + return fi, nil +} + +type nopCloser struct { + io.Reader +} + +func (nopCloser) Close() os.Error { return nil } + +// NopCloser returns a ReadCloser with a no-op Close method wrapping +// the provided Reader r. +func NopCloser(r io.Reader) io.ReadCloser { + return nopCloser{r} +} + +type devNull int + +func (devNull) Write(p []byte) (int, os.Error) { + return len(p), nil +} + +var blackHole = make([]byte, 8192) + +func (devNull) ReadFrom(r io.Reader) (n int64, err os.Error) { + readSize := 0 + for { + readSize, err = r.Read(blackHole) + n += int64(readSize) + if err != nil { + if err == os.EOF { + return n, nil + } + return + } + } + panic("unreachable") +} + +// Discard is an io.Writer on which all Write calls succeed +// without doing anything. +var Discard io.Writer = devNull(0) diff --git a/src/pkg/io/ioutil/ioutil_test.go b/src/pkg/io/ioutil/ioutil_test.go new file mode 100644 index 000000000..55e4b2c2b --- /dev/null +++ b/src/pkg/io/ioutil/ioutil_test.go @@ -0,0 +1,91 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ioutil_test + +import ( + . "io/ioutil" + "os" + "testing" +) + +func checkSize(t *testing.T, path string, size int64) { + dir, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat %q (looking for size %d): %s", path, size, err) + } + if dir.Size != size { + t.Errorf("Stat %q: size %d want %d", path, dir.Size, size) + } +} + +func TestReadFile(t *testing.T) { + filename := "rumpelstilzchen" + contents, err := ReadFile(filename) + if err == nil { + t.Fatalf("ReadFile %s: error expected, none found", filename) + } + + filename = "ioutil_test.go" + contents, err = ReadFile(filename) + if err != nil { + t.Fatalf("ReadFile %s: %v", filename, err) + } + + checkSize(t, filename, int64(len(contents))) +} + +func TestWriteFile(t *testing.T) { + filename := "_test/rumpelstilzchen" + data := "Programming today is a race between software engineers striving to " + + "build bigger and better idiot-proof programs, and the Universe trying " + + "to produce bigger and better idiots. So far, the Universe is winning." + + if err := WriteFile(filename, []byte(data), 0644); err != nil { + t.Fatalf("WriteFile %s: %v", filename, err) + } + + contents, err := ReadFile(filename) + if err != nil { + t.Fatalf("ReadFile %s: %v", filename, err) + } + + if string(contents) != data { + t.Fatalf("contents = %q\nexpected = %q", string(contents), data) + } + + // cleanup + os.Remove(filename) // ignore error +} + +func TestReadDir(t *testing.T) { + dirname := "rumpelstilzchen" + _, err := ReadDir(dirname) + if err == nil { + t.Fatalf("ReadDir %s: error expected, none found", dirname) + } + + dirname = "." + list, err := ReadDir(dirname) + if err != nil { + t.Fatalf("ReadDir %s: %v", dirname, err) + } + + foundTest := false + foundTestDir := false + for _, dir := range list { + switch { + case dir.IsRegular() && dir.Name == "ioutil_test.go": + foundTest = true + case dir.IsDirectory() && dir.Name == "_test": + foundTestDir = true + } + } + if !foundTest { + t.Fatalf("ReadDir %s: test file not found", dirname) + } + if !foundTestDir { + t.Fatalf("ReadDir %s: _test directory not found", dirname) + } +} diff --git a/src/pkg/io/ioutil/tempfile.go b/src/pkg/io/ioutil/tempfile.go new file mode 100644 index 000000000..8e681bdc3 --- /dev/null +++ b/src/pkg/io/ioutil/tempfile.go @@ -0,0 +1,91 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ioutil + +import ( + "os" + "path/filepath" + "strconv" +) + +// Random number state, accessed without lock; racy but harmless. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var rand uint32 + +func reseed() uint32 { + sec, nsec, _ := os.Time() + return uint32(sec*1e9 + nsec + int64(os.Getpid())) +} + +func nextSuffix() string { + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the name of the file. It is the caller's responsibility to +// remove the file when no longer needed. +func TempFile(dir, prefix string) (f *os.File, err os.Error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + break + } + return +} + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func TempDir(dir, prefix string) (name string, err os.Error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + try := filepath.Join(dir, prefix+nextSuffix()) + err = os.Mkdir(try, 0700) + if pe, ok := err.(*os.PathError); ok && pe.Error == os.EEXIST { + if nconflict++; nconflict > 10 { + rand = reseed() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/src/pkg/io/ioutil/tempfile_test.go b/src/pkg/io/ioutil/tempfile_test.go new file mode 100644 index 000000000..80c62f672 --- /dev/null +++ b/src/pkg/io/ioutil/tempfile_test.go @@ -0,0 +1,54 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ioutil_test + +import ( + . "io/ioutil" + "os" + "path/filepath" + "regexp" + "testing" +) + +func TestTempFile(t *testing.T) { + f, err := TempFile("/_not_exists_", "foo") + if f != nil || err == nil { + t.Errorf("TempFile(`/_not_exists_`, `foo`) = %v, %v", f, err) + } + + dir := os.TempDir() + f, err = TempFile(dir, "ioutil_test") + if f == nil || err != nil { + t.Errorf("TempFile(dir, `ioutil_test`) = %v, %v", f, err) + } + if f != nil { + f.Close() + os.Remove(f.Name()) + re := regexp.MustCompile("^" + regexp.QuoteMeta(filepath.Join(dir, "ioutil_test")) + "[0-9]+$") + if !re.MatchString(f.Name()) { + t.Errorf("TempFile(`"+dir+"`, `ioutil_test`) created bad name %s", f.Name()) + } + } +} + +func TestTempDir(t *testing.T) { + name, err := TempDir("/_not_exists_", "foo") + if name != "" || err == nil { + t.Errorf("TempDir(`/_not_exists_`, `foo`) = %v, %v", name, err) + } + + dir := os.TempDir() + name, err = TempDir(dir, "ioutil_test") + if name == "" || err != nil { + t.Errorf("TempDir(dir, `ioutil_test`) = %v, %v", name, err) + } + if name != "" { + os.Remove(name) + re := regexp.MustCompile("^" + regexp.QuoteMeta(filepath.Join(dir, "ioutil_test")) + "[0-9]+$") + if !re.MatchString(name) { + t.Errorf("TempDir(`"+dir+"`, `ioutil_test`) created bad name %s", name) + } + } +} diff --git a/src/pkg/io/multi.go b/src/pkg/io/multi.go new file mode 100644 index 000000000..d702d46c7 --- /dev/null +++ b/src/pkg/io/multi.go @@ -0,0 +1,58 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package io + +import "os" + +type multiReader struct { + readers []Reader +} + +func (mr *multiReader) Read(p []byte) (n int, err os.Error) { + for len(mr.readers) > 0 { + n, err = mr.readers[0].Read(p) + if n > 0 || err != os.EOF { + if err == os.EOF { + // Don't return EOF yet. There may be more bytes + // in the remaining readers. + err = nil + } + return + } + mr.readers = mr.readers[1:] + } + return 0, os.EOF +} + +// MultiReader returns a Reader that's the logical concatenation of +// the provided input readers. They're read sequentially. Once all +// inputs are drained, Read will return os.EOF. +func MultiReader(readers ...Reader) Reader { + return &multiReader{readers} +} + +type multiWriter struct { + writers []Writer +} + +func (t *multiWriter) Write(p []byte) (n int, err os.Error) { + for _, w := range t.writers { + n, err = w.Write(p) + if err != nil { + return + } + if n != len(p) { + err = ErrShortWrite + return + } + } + return len(p), nil +} + +// MultiWriter creates a writer that duplicates its writes to all the +// provided writers, similar to the Unix tee(1) command. +func MultiWriter(writers ...Writer) Writer { + return &multiWriter{writers} +} diff --git a/src/pkg/io/multi_test.go b/src/pkg/io/multi_test.go new file mode 100644 index 000000000..1b3589dde --- /dev/null +++ b/src/pkg/io/multi_test.go @@ -0,0 +1,89 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package io_test + +import ( + . "io" + "bytes" + "crypto/sha1" + "fmt" + "os" + "strings" + "testing" +) + +func TestMultiReader(t *testing.T) { + var mr Reader + var buf []byte + nread := 0 + withFooBar := func(tests func()) { + r1 := strings.NewReader("foo ") + r2 := strings.NewReader("") + r3 := strings.NewReader("bar") + mr = MultiReader(r1, r2, r3) + buf = make([]byte, 20) + tests() + } + expectRead := func(size int, expected string, eerr os.Error) { + nread++ + n, gerr := mr.Read(buf[0:size]) + if n != len(expected) { + t.Errorf("#%d, expected %d bytes; got %d", + nread, len(expected), n) + } + got := string(buf[0:n]) + if got != expected { + t.Errorf("#%d, expected %q; got %q", + nread, expected, got) + } + if gerr != eerr { + t.Errorf("#%d, expected error %v; got %v", + nread, eerr, gerr) + } + buf = buf[n:] + } + withFooBar(func() { + expectRead(2, "fo", nil) + expectRead(5, "o ", nil) + expectRead(5, "bar", nil) + expectRead(5, "", os.EOF) + }) + withFooBar(func() { + expectRead(4, "foo ", nil) + expectRead(1, "b", nil) + expectRead(3, "ar", nil) + expectRead(1, "", os.EOF) + }) + withFooBar(func() { + expectRead(5, "foo ", nil) + }) +} + +func TestMultiWriter(t *testing.T) { + sha1 := sha1.New() + sink := new(bytes.Buffer) + mw := MultiWriter(sha1, sink) + + sourceString := "My input text." + source := strings.NewReader(sourceString) + written, err := Copy(mw, source) + + if written != int64(len(sourceString)) { + t.Errorf("short write of %d, not %d", written, len(sourceString)) + } + + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + sha1hex := fmt.Sprintf("%x", sha1.Sum()) + if sha1hex != "01cb303fa8c30a64123067c5aa6284ba7ec2d31b" { + t.Error("incorrect sha1 value") + } + + if sink.String() != sourceString { + t.Errorf("expected %q; got %q", sourceString, sink.String()) + } +} diff --git a/src/pkg/io/pipe.go b/src/pkg/io/pipe.go new file mode 100644 index 000000000..00be8efa2 --- /dev/null +++ b/src/pkg/io/pipe.go @@ -0,0 +1,182 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Pipe adapter to connect code expecting an io.Reader +// with code expecting an io.Writer. + +package io + +import ( + "os" + "sync" +) + +type pipeResult struct { + n int + err os.Error +} + +// A pipe is the shared pipe structure underlying PipeReader and PipeWriter. +type pipe struct { + rl sync.Mutex // gates readers one at a time + wl sync.Mutex // gates writers one at a time + l sync.Mutex // protects remaining fields + data []byte // data remaining in pending write + rwait sync.Cond // waiting reader + wwait sync.Cond // waiting writer + rerr os.Error // if reader closed, error to give writes + werr os.Error // if writer closed, error to give reads +} + +func (p *pipe) read(b []byte) (n int, err os.Error) { + // One reader at a time. + p.rl.Lock() + defer p.rl.Unlock() + + p.l.Lock() + defer p.l.Unlock() + for { + if p.rerr != nil { + return 0, os.EINVAL + } + if p.data != nil { + break + } + if p.werr != nil { + return 0, p.werr + } + p.rwait.Wait() + } + n = copy(b, p.data) + p.data = p.data[n:] + if len(p.data) == 0 { + p.data = nil + p.wwait.Signal() + } + return +} + +var zero [0]byte + +func (p *pipe) write(b []byte) (n int, err os.Error) { + // pipe uses nil to mean not available + if b == nil { + b = zero[:] + } + + // One writer at a time. + p.wl.Lock() + defer p.wl.Unlock() + + p.l.Lock() + defer p.l.Unlock() + p.data = b + p.rwait.Signal() + for { + if p.data == nil { + break + } + if p.rerr != nil { + err = p.rerr + break + } + if p.werr != nil { + err = os.EINVAL + } + p.wwait.Wait() + } + n = len(b) - len(p.data) + p.data = nil // in case of rerr or werr + return +} + +func (p *pipe) rclose(err os.Error) { + if err == nil { + err = os.EPIPE + } + p.l.Lock() + defer p.l.Unlock() + p.rerr = err + p.rwait.Signal() + p.wwait.Signal() +} + +func (p *pipe) wclose(err os.Error) { + if err == nil { + err = os.EOF + } + p.l.Lock() + defer p.l.Unlock() + p.werr = err + p.rwait.Signal() + p.wwait.Signal() +} + +// A PipeReader is the read half of a pipe. +type PipeReader struct { + p *pipe +} + +// Read implements the standard Read interface: +// it reads data from the pipe, blocking until a writer +// arrives or the write end is closed. +// If the write end is closed with an error, that error is +// returned as err; otherwise err is os.EOF. +func (r *PipeReader) Read(data []byte) (n int, err os.Error) { + return r.p.read(data) +} + +// Close closes the reader; subsequent writes to the +// write half of the pipe will return the error os.EPIPE. +func (r *PipeReader) Close() os.Error { + return r.CloseWithError(nil) +} + +// CloseWithError closes the reader; subsequent writes +// to the write half of the pipe will return the error err. +func (r *PipeReader) CloseWithError(err os.Error) os.Error { + r.p.rclose(err) + return nil +} + +// A PipeWriter is the write half of a pipe. +type PipeWriter struct { + p *pipe +} + +// Write implements the standard Write interface: +// it writes data to the pipe, blocking until readers +// have consumed all the data or the read end is closed. +// If the read end is closed with an error, that err is +// returned as err; otherwise err is os.EPIPE. +func (w *PipeWriter) Write(data []byte) (n int, err os.Error) { + return w.p.write(data) +} + +// Close closes the writer; subsequent reads from the +// read half of the pipe will return no bytes and os.EOF. +func (w *PipeWriter) Close() os.Error { + return w.CloseWithError(nil) +} + +// CloseWithError closes the writer; subsequent reads from the +// read half of the pipe will return no bytes and the error err. +func (w *PipeWriter) CloseWithError(err os.Error) os.Error { + w.p.wclose(err) + return nil +} + +// Pipe creates a synchronous in-memory pipe. +// It can be used to connect code expecting an io.Reader +// with code expecting an io.Writer. +// Reads on one end are matched with writes on the other, +// copying data directly between the two; there is no internal buffering. +func Pipe() (*PipeReader, *PipeWriter) { + p := new(pipe) + p.rwait.L = &p.l + p.wwait.L = &p.l + r := &PipeReader{p} + w := &PipeWriter{p} + return r, w +} diff --git a/src/pkg/io/pipe_test.go b/src/pkg/io/pipe_test.go new file mode 100644 index 000000000..bd4b94f0a --- /dev/null +++ b/src/pkg/io/pipe_test.go @@ -0,0 +1,271 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package io_test + +import ( + "fmt" + . "io" + "os" + "testing" + "time" +) + +func checkWrite(t *testing.T, w Writer, data []byte, c chan int) { + n, err := w.Write(data) + if err != nil { + t.Errorf("write: %v", err) + } + if n != len(data) { + t.Errorf("short write: %d != %d", n, len(data)) + } + c <- 0 +} + +// Test a single read/write pair. +func TestPipe1(t *testing.T) { + c := make(chan int) + r, w := Pipe() + var buf = make([]byte, 64) + go checkWrite(t, w, []byte("hello, world"), c) + n, err := r.Read(buf) + if err != nil { + t.Errorf("read: %v", err) + } else if n != 12 || string(buf[0:12]) != "hello, world" { + t.Errorf("bad read: got %q", buf[0:n]) + } + <-c + r.Close() + w.Close() +} + +func reader(t *testing.T, r Reader, c chan int) { + var buf = make([]byte, 64) + for { + n, err := r.Read(buf) + if err == os.EOF { + c <- 0 + break + } + if err != nil { + t.Errorf("read: %v", err) + } + c <- n + } +} + +// Test a sequence of read/write pairs. +func TestPipe2(t *testing.T) { + c := make(chan int) + r, w := Pipe() + go reader(t, r, c) + var buf = make([]byte, 64) + for i := 0; i < 5; i++ { + p := buf[0 : 5+i*10] + n, err := w.Write(p) + if n != len(p) { + t.Errorf("wrote %d, got %d", len(p), n) + } + if err != nil { + t.Errorf("write: %v", err) + } + nn := <-c + if nn != n { + t.Errorf("wrote %d, read got %d", n, nn) + } + } + w.Close() + nn := <-c + if nn != 0 { + t.Errorf("final read got %d", nn) + } +} + +type pipeReturn struct { + n int + err os.Error +} + +// Test a large write that requires multiple reads to satisfy. +func writer(w WriteCloser, buf []byte, c chan pipeReturn) { + n, err := w.Write(buf) + w.Close() + c <- pipeReturn{n, err} +} + +func TestPipe3(t *testing.T) { + c := make(chan pipeReturn) + r, w := Pipe() + var wdat = make([]byte, 128) + for i := 0; i < len(wdat); i++ { + wdat[i] = byte(i) + } + go writer(w, wdat, c) + var rdat = make([]byte, 1024) + tot := 0 + for n := 1; n <= 256; n *= 2 { + nn, err := r.Read(rdat[tot : tot+n]) + if err != nil && err != os.EOF { + t.Fatalf("read: %v", err) + } + + // only final two reads should be short - 1 byte, then 0 + expect := n + if n == 128 { + expect = 1 + } else if n == 256 { + expect = 0 + if err != os.EOF { + t.Fatalf("read at end: %v", err) + } + } + if nn != expect { + t.Fatalf("read %d, expected %d, got %d", n, expect, nn) + } + tot += nn + } + pr := <-c + if pr.n != 128 || pr.err != nil { + t.Fatalf("write 128: %d, %v", pr.n, pr.err) + } + if tot != 128 { + t.Fatalf("total read %d != 128", tot) + } + for i := 0; i < 128; i++ { + if rdat[i] != byte(i) { + t.Fatalf("rdat[%d] = %d", i, rdat[i]) + } + } +} + +// Test read after/before writer close. + +type closer interface { + CloseWithError(os.Error) os.Error + Close() os.Error +} + +type pipeTest struct { + async bool + err os.Error + closeWithError bool +} + +func (p pipeTest) String() string { + return fmt.Sprintf("async=%v err=%v closeWithError=%v", p.async, p.err, p.closeWithError) +} + +var pipeTests = []pipeTest{ + {true, nil, false}, + {true, nil, true}, + {true, ErrShortWrite, true}, + {false, nil, false}, + {false, nil, true}, + {false, ErrShortWrite, true}, +} + +func delayClose(t *testing.T, cl closer, ch chan int, tt pipeTest) { + time.Sleep(1e6) // 1 ms + var err os.Error + if tt.closeWithError { + err = cl.CloseWithError(tt.err) + } else { + err = cl.Close() + } + if err != nil { + t.Errorf("delayClose: %v", err) + } + ch <- 0 +} + +func TestPipeReadClose(t *testing.T) { + for _, tt := range pipeTests { + c := make(chan int, 1) + r, w := Pipe() + if tt.async { + go delayClose(t, w, c, tt) + } else { + delayClose(t, w, c, tt) + } + var buf = make([]byte, 64) + n, err := r.Read(buf) + <-c + want := tt.err + if want == nil { + want = os.EOF + } + if err != want { + t.Errorf("read from closed pipe: %v want %v", err, want) + } + if n != 0 { + t.Errorf("read on closed pipe returned %d", n) + } + if err = r.Close(); err != nil { + t.Errorf("r.Close: %v", err) + } + } +} + +// Test close on Read side during Read. +func TestPipeReadClose2(t *testing.T) { + c := make(chan int, 1) + r, _ := Pipe() + go delayClose(t, r, c, pipeTest{}) + n, err := r.Read(make([]byte, 64)) + <-c + if n != 0 || err != os.EINVAL { + t.Errorf("read from closed pipe: %v, %v want %v, %v", n, err, 0, os.EINVAL) + } +} + +// Test write after/before reader close. + +func TestPipeWriteClose(t *testing.T) { + for _, tt := range pipeTests { + c := make(chan int, 1) + r, w := Pipe() + if tt.async { + go delayClose(t, r, c, tt) + } else { + delayClose(t, r, c, tt) + } + n, err := WriteString(w, "hello, world") + <-c + expect := tt.err + if expect == nil { + expect = os.EPIPE + } + if err != expect { + t.Errorf("write on closed pipe: %v want %v", err, expect) + } + if n != 0 { + t.Errorf("write on closed pipe returned %d", n) + } + if err = w.Close(); err != nil { + t.Errorf("w.Close: %v", err) + } + } +} + +func TestWriteEmpty(t *testing.T) { + r, w := Pipe() + go func() { + w.Write([]byte{}) + w.Close() + }() + var b [2]byte + ReadFull(r, b[0:2]) + r.Close() +} + +func TestWriteNil(t *testing.T) { + r, w := Pipe() + go func() { + w.Write(nil) + w.Close() + }() + var b [2]byte + ReadFull(r, b[0:2]) + r.Close() +} |