diff options
Diffstat (limited to 'src/pkg/archive')
45 files changed, 0 insertions, 5488 deletions
diff --git a/src/pkg/archive/tar/common.go b/src/pkg/archive/tar/common.go deleted file mode 100644 index e363aa793..000000000 --- a/src/pkg/archive/tar/common.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package tar implements access to tar archives. -// It aims to cover most of the variations, including those produced -// by GNU and BSD tars. -// -// References: -// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 -// http://www.gnu.org/software/tar/manual/html_node/Standard.html -// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html -package tar - -import ( - "bytes" - "errors" - "fmt" - "os" - "path" - "time" -) - -const ( - blockSize = 512 - - // Types - TypeReg = '0' // regular file - TypeRegA = '\x00' // regular file - TypeLink = '1' // hard link - TypeSymlink = '2' // symbolic link - TypeChar = '3' // character device node - TypeBlock = '4' // block device node - TypeDir = '5' // directory - TypeFifo = '6' // fifo node - TypeCont = '7' // reserved - TypeXHeader = 'x' // extended header - TypeXGlobalHeader = 'g' // global extended header - TypeGNULongName = 'L' // Next file has a long name - TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name - TypeGNUSparse = 'S' // sparse file -) - -// A Header represents a single header in a tar archive. -// Some fields may not be populated. -type Header struct { - Name string // name of header file entry - Mode int64 // permission and mode bits - Uid int // user id of owner - Gid int // group id of owner - Size int64 // length in bytes - ModTime time.Time // modified time - Typeflag byte // type of header entry - Linkname string // target name of link - Uname string // user name of owner - Gname string // group name of owner - Devmajor int64 // major number of character or block device - Devminor int64 // minor number of character or block device - AccessTime time.Time // access time - ChangeTime time.Time // status change time - Xattrs map[string]string -} - -// File name constants from the tar spec. -const ( - fileNameSize = 100 // Maximum number of bytes in a standard tar name. - fileNamePrefixSize = 155 // Maximum number of ustar extension bytes. -) - -// FileInfo returns an os.FileInfo for the Header. -func (h *Header) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - h *Header -} - -func (fi headerFileInfo) Size() int64 { return fi.h.Size } -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } -func (fi headerFileInfo) Sys() interface{} { return fi.h } - -// Name returns the base name of the file. -func (fi headerFileInfo) Name() string { - if fi.IsDir() { - return path.Base(path.Clean(fi.h.Name)) - } - return path.Base(fi.h.Name) -} - -// Mode returns the permission and mode bits for the headerFileInfo. -func (fi headerFileInfo) Mode() (mode os.FileMode) { - // Set file permission bits. - mode = os.FileMode(fi.h.Mode).Perm() - - // Set setuid, setgid and sticky bits. - if fi.h.Mode&c_ISUID != 0 { - // setuid - mode |= os.ModeSetuid - } - if fi.h.Mode&c_ISGID != 0 { - // setgid - mode |= os.ModeSetgid - } - if fi.h.Mode&c_ISVTX != 0 { - // sticky - mode |= os.ModeSticky - } - - // Set file mode bits. - // clear perm, setuid, setgid and sticky bits. - m := os.FileMode(fi.h.Mode) &^ 07777 - if m == c_ISDIR { - // directory - mode |= os.ModeDir - } - if m == c_ISFIFO { - // named pipe (FIFO) - mode |= os.ModeNamedPipe - } - if m == c_ISLNK { - // symbolic link - mode |= os.ModeSymlink - } - if m == c_ISBLK { - // device file - mode |= os.ModeDevice - } - if m == c_ISCHR { - // Unix character device - mode |= os.ModeDevice - mode |= os.ModeCharDevice - } - if m == c_ISSOCK { - // Unix domain socket - mode |= os.ModeSocket - } - - switch fi.h.Typeflag { - case TypeLink, TypeSymlink: - // hard link, symbolic link - mode |= os.ModeSymlink - case TypeChar: - // character device node - mode |= os.ModeDevice - mode |= os.ModeCharDevice - case TypeBlock: - // block device node - mode |= os.ModeDevice - case TypeDir: - // directory - mode |= os.ModeDir - case TypeFifo: - // fifo node - mode |= os.ModeNamedPipe - } - - return mode -} - -// sysStat, if non-nil, populates h from system-dependent fields of fi. -var sysStat func(fi os.FileInfo, h *Header) error - -// Mode constants from the tar spec. -const ( - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) - c_ISDIR = 040000 // Directory - c_ISFIFO = 010000 // FIFO - c_ISREG = 0100000 // Regular file - c_ISLNK = 0120000 // Symbolic link - c_ISBLK = 060000 // Block special file - c_ISCHR = 020000 // Character special file - c_ISSOCK = 0140000 // Socket -) - -// Keywords for the PAX Extended Header -const ( - paxAtime = "atime" - paxCharset = "charset" - paxComment = "comment" - paxCtime = "ctime" // please note that ctime is not a valid pax header. - paxGid = "gid" - paxGname = "gname" - paxLinkpath = "linkpath" - paxMtime = "mtime" - paxPath = "path" - paxSize = "size" - paxUid = "uid" - paxUname = "uname" - paxXattr = "SCHILY.xattr." - paxNone = "" -) - -// FileInfoHeader creates a partially-populated Header from fi. -// If fi describes a symlink, FileInfoHeader records link as the link target. -// If fi describes a directory, a slash is appended to the name. -// Because os.FileInfo's Name method returns only the base name of -// the file it describes, it may be necessary to modify the Name field -// of the returned header to provide the full path name of the file. -func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { - if fi == nil { - return nil, errors.New("tar: FileInfo is nil") - } - fm := fi.Mode() - h := &Header{ - Name: fi.Name(), - ModTime: fi.ModTime(), - Mode: int64(fm.Perm()), // or'd with c_IS* constants later - } - switch { - case fm.IsRegular(): - h.Mode |= c_ISREG - h.Typeflag = TypeReg - h.Size = fi.Size() - case fi.IsDir(): - h.Typeflag = TypeDir - h.Mode |= c_ISDIR - h.Name += "/" - case fm&os.ModeSymlink != 0: - h.Typeflag = TypeSymlink - h.Mode |= c_ISLNK - h.Linkname = link - case fm&os.ModeDevice != 0: - if fm&os.ModeCharDevice != 0 { - h.Mode |= c_ISCHR - h.Typeflag = TypeChar - } else { - h.Mode |= c_ISBLK - h.Typeflag = TypeBlock - } - case fm&os.ModeNamedPipe != 0: - h.Typeflag = TypeFifo - h.Mode |= c_ISFIFO - case fm&os.ModeSocket != 0: - h.Mode |= c_ISSOCK - default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) - } - if fm&os.ModeSetuid != 0 { - h.Mode |= c_ISUID - } - if fm&os.ModeSetgid != 0 { - h.Mode |= c_ISGID - } - if fm&os.ModeSticky != 0 { - h.Mode |= c_ISVTX - } - if sysStat != nil { - return h, sysStat(fi, h) - } - return h, nil -} - -var zeroBlock = make([]byte, blockSize) - -// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. -// We compute and return both. -func checksum(header []byte) (unsigned int64, signed int64) { - for i := 0; i < len(header); i++ { - if i == 148 { - // The chksum field (header[148:156]) is special: it should be treated as space bytes. - unsigned += ' ' * 8 - signed += ' ' * 8 - i += 7 - continue - } - unsigned += int64(header[i]) - signed += int64(int8(header[i])) - } - return -} - -type slicer []byte - -func (sp *slicer) next(n int) (b []byte) { - s := *sp - b, *sp = s[0:n], s[n:] - return -} - -func isASCII(s string) bool { - for _, c := range s { - if c >= 0x80 { - return false - } - } - return true -} - -func toASCII(s string) string { - if isASCII(s) { - return s - } - var buf bytes.Buffer - for _, c := range s { - if c < 0x80 { - buf.WriteByte(byte(c)) - } - } - return buf.String() -} diff --git a/src/pkg/archive/tar/example_test.go b/src/pkg/archive/tar/example_test.go deleted file mode 100644 index 351eaa0e6..000000000 --- a/src/pkg/archive/tar/example_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar_test - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "log" - "os" -) - -func Example() { - // Create a buffer to write our archive to. - buf := new(bytes.Buffer) - - // Create a new tar archive. - tw := tar.NewWriter(buf) - - // Add some files to the archive. - var files = []struct { - Name, Body string - }{ - {"readme.txt", "This archive contains some text files."}, - {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, - {"todo.txt", "Get animal handling licence."}, - } - for _, file := range files { - hdr := &tar.Header{ - Name: file.Name, - Size: int64(len(file.Body)), - } - if err := tw.WriteHeader(hdr); err != nil { - log.Fatalln(err) - } - if _, err := tw.Write([]byte(file.Body)); err != nil { - log.Fatalln(err) - } - } - // Make sure to check the error on Close. - if err := tw.Close(); err != nil { - log.Fatalln(err) - } - - // Open the tar archive for reading. - r := bytes.NewReader(buf.Bytes()) - tr := tar.NewReader(r) - - // Iterate through the files in the archive. - for { - hdr, err := tr.Next() - if err == io.EOF { - // end of tar archive - break - } - if err != nil { - log.Fatalln(err) - } - fmt.Printf("Contents of %s:\n", hdr.Name) - if _, err := io.Copy(os.Stdout, tr); err != nil { - log.Fatalln(err) - } - fmt.Println() - } - - // Output: - // Contents of readme.txt: - // This archive contains some text files. - // Contents of gopher.txt: - // Gopher names: - // George - // Geoffrey - // Gonzo - // Contents of todo.txt: - // Get animal handling licence. -} diff --git a/src/pkg/archive/tar/reader.go b/src/pkg/archive/tar/reader.go deleted file mode 100644 index 920a9b08f..000000000 --- a/src/pkg/archive/tar/reader.go +++ /dev/null @@ -1,817 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -// TODO(dsymonds): -// - pax extensions - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "os" - "strconv" - "strings" - "time" -) - -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") -) - -const maxNanoSecondIntSize = 9 - -// A Reader provides sequential access to the contents of a tar archive. -// A tar archive consists of a sequence of files. -// The Next method advances to the next file in the archive (including the first), -// and then it can be treated as an io.Reader to access the file's data. -type Reader struct { - r io.Reader - err error - pad int64 // amount of padding (ignored) after current file entry - curr numBytesReader // reader for current file entry -} - -// A numBytesReader is an io.Reader with a numBytes method, returning the number -// of bytes remaining in the underlying encoded data. -type numBytesReader interface { - io.Reader - numBytes() int64 -} - -// A regFileReader is a numBytesReader for reading file data from a tar archive. -type regFileReader struct { - r io.Reader // underlying reader - nb int64 // number of unread bytes for current file entry -} - -// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive. -type sparseFileReader struct { - rfr *regFileReader // reads the sparse-encoded file data - sp []sparseEntry // the sparse map for the file - pos int64 // keeps track of file position - tot int64 // total size of the file -} - -// Keywords for GNU sparse files in a PAX extended header -const ( - paxGNUSparseNumBlocks = "GNU.sparse.numblocks" - paxGNUSparseOffset = "GNU.sparse.offset" - paxGNUSparseNumBytes = "GNU.sparse.numbytes" - paxGNUSparseMap = "GNU.sparse.map" - paxGNUSparseName = "GNU.sparse.name" - paxGNUSparseMajor = "GNU.sparse.major" - paxGNUSparseMinor = "GNU.sparse.minor" - paxGNUSparseSize = "GNU.sparse.size" - paxGNUSparseRealSize = "GNU.sparse.realsize" -) - -// Keywords for old GNU sparse headers -const ( - oldGNUSparseMainHeaderOffset = 386 - oldGNUSparseMainHeaderIsExtendedOffset = 482 - oldGNUSparseMainHeaderNumEntries = 4 - oldGNUSparseExtendedHeaderIsExtendedOffset = 504 - oldGNUSparseExtendedHeaderNumEntries = 21 - oldGNUSparseOffsetSize = 12 - oldGNUSparseNumBytesSize = 12 -) - -// NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { return &Reader{r: r} } - -// Next advances to the next entry in the tar archive. -func (tr *Reader) Next() (*Header, error) { - var hdr *Header - if tr.err == nil { - tr.skipUnread() - } - if tr.err != nil { - return hdr, tr.err - } - hdr = tr.readHeader() - if hdr == nil { - return hdr, tr.err - } - // Check for PAX/GNU header. - switch hdr.Typeflag { - case TypeXHeader: - // PAX extended header - headers, err := parsePAX(tr) - if err != nil { - return nil, err - } - // We actually read the whole file, - // but this skips alignment padding - tr.skipUnread() - hdr = tr.readHeader() - mergePAX(hdr, headers) - - // Check for a PAX format sparse file - sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers) - if err != nil { - tr.err = err - return nil, err - } - if sp != nil { - // Current file is a PAX format GNU sparse file. - // Set the current file reader to a sparse file reader. - tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size} - } - return hdr, nil - case TypeGNULongName: - // We have a GNU long name header. Its contents are the real file name. - realname, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - hdr, err := tr.Next() - hdr.Name = cString(realname) - return hdr, err - case TypeGNULongLink: - // We have a GNU long link header. - realname, err := ioutil.ReadAll(tr) - if err != nil { - return nil, err - } - hdr, err := tr.Next() - hdr.Linkname = cString(realname) - return hdr, err - } - return hdr, tr.err -} - -// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then -// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to -// be treated as a regular file. -func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) { - var sparseFormat string - - // Check for sparse format indicators - major, majorOk := headers[paxGNUSparseMajor] - minor, minorOk := headers[paxGNUSparseMinor] - sparseName, sparseNameOk := headers[paxGNUSparseName] - _, sparseMapOk := headers[paxGNUSparseMap] - sparseSize, sparseSizeOk := headers[paxGNUSparseSize] - sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize] - - // Identify which, if any, sparse format applies from which PAX headers are set - if majorOk && minorOk { - sparseFormat = major + "." + minor - } else if sparseNameOk && sparseMapOk { - sparseFormat = "0.1" - } else if sparseSizeOk { - sparseFormat = "0.0" - } else { - // Not a PAX format GNU sparse file. - return nil, nil - } - - // Check for unknown sparse format - if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" { - return nil, nil - } - - // Update hdr from GNU sparse PAX headers - if sparseNameOk { - hdr.Name = sparseName - } - if sparseSizeOk { - realSize, err := strconv.ParseInt(sparseSize, 10, 0) - if err != nil { - return nil, ErrHeader - } - hdr.Size = realSize - } else if sparseRealSizeOk { - realSize, err := strconv.ParseInt(sparseRealSize, 10, 0) - if err != nil { - return nil, ErrHeader - } - hdr.Size = realSize - } - - // Set up the sparse map, according to the particular sparse format in use - var sp []sparseEntry - var err error - switch sparseFormat { - case "0.0", "0.1": - sp, err = readGNUSparseMap0x1(headers) - case "1.0": - sp, err = readGNUSparseMap1x0(tr.curr) - } - return sp, err -} - -// mergePAX merges well known headers according to PAX standard. -// In general headers with the same name as those found -// in the header struct overwrite those found in the header -// struct with higher precision or longer values. Esp. useful -// for name and linkname fields. -func mergePAX(hdr *Header, headers map[string]string) error { - for k, v := range headers { - switch k { - case paxPath: - hdr.Name = v - case paxLinkpath: - hdr.Linkname = v - case paxGname: - hdr.Gname = v - case paxUname: - hdr.Uname = v - case paxUid: - uid, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Uid = int(uid) - case paxGid: - gid, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Gid = int(gid) - case paxAtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.AccessTime = t - case paxMtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.ModTime = t - case paxCtime: - t, err := parsePAXTime(v) - if err != nil { - return err - } - hdr.ChangeTime = t - case paxSize: - size, err := strconv.ParseInt(v, 10, 0) - if err != nil { - return err - } - hdr.Size = int64(size) - default: - if strings.HasPrefix(k, paxXattr) { - if hdr.Xattrs == nil { - hdr.Xattrs = make(map[string]string) - } - hdr.Xattrs[k[len(paxXattr):]] = v - } - } - } - return nil -} - -// parsePAXTime takes a string of the form %d.%d as described in -// the PAX specification. -func parsePAXTime(t string) (time.Time, error) { - buf := []byte(t) - pos := bytes.IndexByte(buf, '.') - var seconds, nanoseconds int64 - var err error - if pos == -1 { - seconds, err = strconv.ParseInt(t, 10, 0) - if err != nil { - return time.Time{}, err - } - } else { - seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0) - if err != nil { - return time.Time{}, err - } - nano_buf := string(buf[pos+1:]) - // Pad as needed before converting to a decimal. - // For example .030 -> .030000000 -> 30000000 nanoseconds - if len(nano_buf) < maxNanoSecondIntSize { - // Right pad - nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf)) - } else if len(nano_buf) > maxNanoSecondIntSize { - // Right truncate - nano_buf = nano_buf[:maxNanoSecondIntSize] - } - nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0) - if err != nil { - return time.Time{}, err - } - } - ts := time.Unix(seconds, nanoseconds) - return ts, nil -} - -// parsePAX parses PAX headers. -// If an extended header (type 'x') is invalid, ErrHeader is returned -func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - // For GNU PAX sparse format 0.0 support. - // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers. - var sparseMap bytes.Buffer - - headers := make(map[string]string) - // Each record is constructed as - // "%d %s=%s\n", length, keyword, value - for len(buf) > 0 { - // or the header was empty to start with. - var sp int - // The size field ends at the first space. - sp = bytes.IndexByte(buf, ' ') - if sp == -1 { - return nil, ErrHeader - } - // Parse the first token as a decimal integer. - n, err := strconv.ParseInt(string(buf[:sp]), 10, 0) - if err != nil { - return nil, ErrHeader - } - // Extract everything between the decimal and the n -1 on the - // beginning to eat the ' ', -1 on the end to skip the newline. - var record []byte - record, buf = buf[sp+1:n-1], buf[n:] - // The first equals is guaranteed to mark the end of the key. - // Everything else is value. - eq := bytes.IndexByte(record, '=') - if eq == -1 { - return nil, ErrHeader - } - key, value := record[:eq], record[eq+1:] - - keyStr := string(key) - if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes { - // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map. - sparseMap.Write(value) - sparseMap.Write([]byte{','}) - } else { - // Normal key. Set the value in the headers map. - headers[keyStr] = string(value) - } - } - if sparseMap.Len() != 0 { - // Add sparse info to headers, chopping off the extra comma - sparseMap.Truncate(sparseMap.Len() - 1) - headers[paxGNUSparseMap] = sparseMap.String() - } - return headers, nil -} - -// cString parses bytes as a NUL-terminated C-style string. -// If a NUL byte is not found then the whole slice is returned as a string. -func cString(b []byte) string { - n := 0 - for n < len(b) && b[n] != 0 { - n++ - } - return string(b[0:n]) -} - -func (tr *Reader) octal(b []byte) int64 { - // Check for binary format first. - if len(b) > 0 && b[0]&0x80 != 0 { - var x int64 - for i, c := range b { - if i == 0 { - c &= 0x7f // ignore signal bit in first byte - } - x = x<<8 | int64(c) - } - return x - } - - // Because unused fields are filled with NULs, we need - // to skip leading NULs. Fields may also be padded with - // spaces or NULs. - // So we remove leading and trailing NULs and spaces to - // be sure. - b = bytes.Trim(b, " \x00") - - if len(b) == 0 { - return 0 - } - x, err := strconv.ParseUint(cString(b), 8, 64) - if err != nil { - tr.err = err - } - return int64(x) -} - -// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding. -func (tr *Reader) skipUnread() { - nr := tr.numBytes() + tr.pad // number of bytes to skip - tr.curr, tr.pad = nil, 0 - if sr, ok := tr.r.(io.Seeker); ok { - if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil { - return - } - } - _, tr.err = io.CopyN(ioutil.Discard, tr.r, nr) -} - -func (tr *Reader) verifyChecksum(header []byte) bool { - if tr.err != nil { - return false - } - - given := tr.octal(header[148:156]) - unsigned, signed := checksum(header) - return given == unsigned || given == signed -} - -func (tr *Reader) readHeader() *Header { - header := make([]byte, blockSize) - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { - return nil - } - - // Two blocks of zero bytes marks the end of the archive. - if bytes.Equal(header, zeroBlock[0:blockSize]) { - if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { - return nil - } - if bytes.Equal(header, zeroBlock[0:blockSize]) { - tr.err = io.EOF - } else { - tr.err = ErrHeader // zero block and then non-zero block - } - return nil - } - - if !tr.verifyChecksum(header) { - tr.err = ErrHeader - return nil - } - - // Unpack - hdr := new(Header) - s := slicer(header) - - hdr.Name = cString(s.next(100)) - hdr.Mode = tr.octal(s.next(8)) - hdr.Uid = int(tr.octal(s.next(8))) - hdr.Gid = int(tr.octal(s.next(8))) - hdr.Size = tr.octal(s.next(12)) - hdr.ModTime = time.Unix(tr.octal(s.next(12)), 0) - s.next(8) // chksum - hdr.Typeflag = s.next(1)[0] - hdr.Linkname = cString(s.next(100)) - - // The remainder of the header depends on the value of magic. - // The original (v7) version of tar had no explicit magic field, - // so its magic bytes, like the rest of the block, are NULs. - magic := string(s.next(8)) // contains version field as well. - var format string - switch { - case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988) - if string(header[508:512]) == "tar\x00" { - format = "star" - } else { - format = "posix" - } - case magic == "ustar \x00": // old GNU tar - format = "gnu" - } - - switch format { - case "posix", "gnu", "star": - hdr.Uname = cString(s.next(32)) - hdr.Gname = cString(s.next(32)) - devmajor := s.next(8) - devminor := s.next(8) - if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { - hdr.Devmajor = tr.octal(devmajor) - hdr.Devminor = tr.octal(devminor) - } - var prefix string - switch format { - case "posix", "gnu": - prefix = cString(s.next(155)) - case "star": - prefix = cString(s.next(131)) - hdr.AccessTime = time.Unix(tr.octal(s.next(12)), 0) - hdr.ChangeTime = time.Unix(tr.octal(s.next(12)), 0) - } - if len(prefix) > 0 { - hdr.Name = prefix + "/" + hdr.Name - } - } - - if tr.err != nil { - tr.err = ErrHeader - return nil - } - - // Maximum value of hdr.Size is 64 GB (12 octal digits), - // so there's no risk of int64 overflowing. - nb := int64(hdr.Size) - tr.pad = -nb & (blockSize - 1) // blockSize is a power of two - - // Set the current file reader. - tr.curr = ®FileReader{r: tr.r, nb: nb} - - // Check for old GNU sparse format entry. - if hdr.Typeflag == TypeGNUSparse { - // Get the real size of the file. - hdr.Size = tr.octal(header[483:495]) - - // Read the sparse map. - sp := tr.readOldGNUSparseMap(header) - if tr.err != nil { - return nil - } - // Current file is a GNU sparse file. Update the current file reader. - tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size} - } - - return hdr -} - -// A sparseEntry holds a single entry in a sparse file's sparse map. -// A sparse entry indicates the offset and size in a sparse file of a -// block of data. -type sparseEntry struct { - offset int64 - numBytes int64 -} - -// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format. -// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries, -// then one or more extension headers are used to store the rest of the sparse map. -func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry { - isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0 - spCap := oldGNUSparseMainHeaderNumEntries - if isExtended { - spCap += oldGNUSparseExtendedHeaderNumEntries - } - sp := make([]sparseEntry, 0, spCap) - s := slicer(header[oldGNUSparseMainHeaderOffset:]) - - // Read the four entries from the main tar header - for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ { - offset := tr.octal(s.next(oldGNUSparseOffsetSize)) - numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize)) - if tr.err != nil { - tr.err = ErrHeader - return nil - } - if offset == 0 && numBytes == 0 { - break - } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) - } - - for isExtended { - // There are more entries. Read an extension header and parse its entries. - sparseHeader := make([]byte, blockSize) - if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil { - return nil - } - isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0 - s = slicer(sparseHeader) - for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ { - offset := tr.octal(s.next(oldGNUSparseOffsetSize)) - numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize)) - if tr.err != nil { - tr.err = ErrHeader - return nil - } - if offset == 0 && numBytes == 0 { - break - } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) - } - } - return sp -} - -// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0. -// The sparse map is stored just before the file data and padded out to the nearest block boundary. -func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { - buf := make([]byte, 2*blockSize) - sparseHeader := buf[:blockSize] - - // readDecimal is a helper function to read a decimal integer from the sparse map - // while making sure to read from the file in blocks of size blockSize - readDecimal := func() (int64, error) { - // Look for newline - nl := bytes.IndexByte(sparseHeader, '\n') - if nl == -1 { - if len(sparseHeader) >= blockSize { - // This is an error - return 0, ErrHeader - } - oldLen := len(sparseHeader) - newLen := oldLen + blockSize - if cap(sparseHeader) < newLen { - // There's more header, but we need to make room for the next block - copy(buf, sparseHeader) - sparseHeader = buf[:newLen] - } else { - // There's more header, and we can just reslice - sparseHeader = sparseHeader[:newLen] - } - - // Now that sparseHeader is large enough, read next block - if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil { - return 0, err - } - - // Look for a newline in the new data - nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n') - if nl == -1 { - // This is an error - return 0, ErrHeader - } - nl += oldLen // We want the position from the beginning - } - // Now that we've found a newline, read a number - n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0) - if err != nil { - return 0, ErrHeader - } - - // Update sparseHeader to consume this number - sparseHeader = sparseHeader[nl+1:] - return n, nil - } - - // Read the first block - if _, err := io.ReadFull(r, sparseHeader); err != nil { - return nil, err - } - - // The first line contains the number of entries - numEntries, err := readDecimal() - if err != nil { - return nil, err - } - - // Read all the entries - sp := make([]sparseEntry, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - // Read the offset - offset, err := readDecimal() - if err != nil { - return nil, err - } - // Read numBytes - numBytes, err := readDecimal() - if err != nil { - return nil, err - } - - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) - } - - return sp, nil -} - -// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1. -// The sparse map is stored in the PAX headers. -func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) { - // Get number of entries - numEntriesStr, ok := headers[paxGNUSparseNumBlocks] - if !ok { - return nil, ErrHeader - } - numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) - if err != nil { - return nil, ErrHeader - } - - sparseMap := strings.Split(headers[paxGNUSparseMap], ",") - - // There should be two numbers in sparseMap for each entry - if int64(len(sparseMap)) != 2*numEntries { - return nil, ErrHeader - } - - // Loop through the entries in the sparse map - sp := make([]sparseEntry, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0) - if err != nil { - return nil, ErrHeader - } - numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0) - if err != nil { - return nil, ErrHeader - } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) - } - - return sp, nil -} - -// numBytes returns the number of bytes left to read in the current file's entry -// in the tar archive, or 0 if there is no current file. -func (tr *Reader) numBytes() int64 { - if tr.curr == nil { - // No current file, so no bytes - return 0 - } - return tr.curr.numBytes() -} - -// Read reads from the current entry in the tar archive. -// It returns 0, io.EOF when it reaches the end of that entry, -// until Next is called to advance to the next entry. -func (tr *Reader) Read(b []byte) (n int, err error) { - if tr.curr == nil { - return 0, io.EOF - } - n, err = tr.curr.Read(b) - if err != nil && err != io.EOF { - tr.err = err - } - return -} - -func (rfr *regFileReader) Read(b []byte) (n int, err error) { - if rfr.nb == 0 { - // file consumed - return 0, io.EOF - } - if int64(len(b)) > rfr.nb { - b = b[0:rfr.nb] - } - n, err = rfr.r.Read(b) - rfr.nb -= int64(n) - - if err == io.EOF && rfr.nb > 0 { - err = io.ErrUnexpectedEOF - } - return -} - -// numBytes returns the number of bytes left to read in the file's data in the tar archive. -func (rfr *regFileReader) numBytes() int64 { - return rfr.nb -} - -// readHole reads a sparse file hole ending at offset toOffset -func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int { - n64 := toOffset - sfr.pos - if n64 > int64(len(b)) { - n64 = int64(len(b)) - } - n := int(n64) - for i := 0; i < n; i++ { - b[i] = 0 - } - sfr.pos += n64 - return n -} - -// Read reads the sparse file data in expanded form. -func (sfr *sparseFileReader) Read(b []byte) (n int, err error) { - if len(sfr.sp) == 0 { - // No more data fragments to read from. - if sfr.pos < sfr.tot { - // We're in the last hole - n = sfr.readHole(b, sfr.tot) - return - } - // Otherwise, we're at the end of the file - return 0, io.EOF - } - if sfr.pos < sfr.sp[0].offset { - // We're in a hole - n = sfr.readHole(b, sfr.sp[0].offset) - return - } - - // We're not in a hole, so we'll read from the next data fragment - posInFragment := sfr.pos - sfr.sp[0].offset - bytesLeft := sfr.sp[0].numBytes - posInFragment - if int64(len(b)) > bytesLeft { - b = b[0:bytesLeft] - } - - n, err = sfr.rfr.Read(b) - sfr.pos += int64(n) - - if int64(n) == bytesLeft { - // We're done with this fragment - sfr.sp = sfr.sp[1:] - } - - if err == io.EOF && sfr.pos < sfr.tot { - // We reached the end of the last fragment's data, but there's a final hole - err = nil - } - return -} - -// numBytes returns the number of bytes left to read in the sparse file's -// sparse-encoded data in the tar archive. -func (sfr *sparseFileReader) numBytes() int64 { - return sfr.rfr.nb -} diff --git a/src/pkg/archive/tar/reader_test.go b/src/pkg/archive/tar/reader_test.go deleted file mode 100644 index 9601ffe45..000000000 --- a/src/pkg/archive/tar/reader_test.go +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "crypto/md5" - "fmt" - "io" - "io/ioutil" - "os" - "reflect" - "strings" - "testing" - "time" -) - -type untarTest struct { - file string - headers []*Header - cksums []string -} - -var gnuTarTest = &untarTest{ - file: "testdata/gnu.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244428340, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - { - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244436044, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - }, - cksums: []string{ - "e38b27eaccb4391bdec553a7f3ae6b2f", - "c65bd2e50a56a2138bf1716f2fd56fe9", - }, -} - -var sparseTarTest = &untarTest{ - file: "testdata/sparse-formats.tar", - headers: []*Header{ - { - Name: "sparse-gnu", - Mode: 420, - Uid: 1000, - Gid: 1000, - Size: 200, - ModTime: time.Unix(1392395740, 0), - Typeflag: 0x53, - Linkname: "", - Uname: "david", - Gname: "david", - Devmajor: 0, - Devminor: 0, - }, - { - Name: "sparse-posix-0.0", - Mode: 420, - Uid: 1000, - Gid: 1000, - Size: 200, - ModTime: time.Unix(1392342187, 0), - Typeflag: 0x30, - Linkname: "", - Uname: "david", - Gname: "david", - Devmajor: 0, - Devminor: 0, - }, - { - Name: "sparse-posix-0.1", - Mode: 420, - Uid: 1000, - Gid: 1000, - Size: 200, - ModTime: time.Unix(1392340456, 0), - Typeflag: 0x30, - Linkname: "", - Uname: "david", - Gname: "david", - Devmajor: 0, - Devminor: 0, - }, - { - Name: "sparse-posix-1.0", - Mode: 420, - Uid: 1000, - Gid: 1000, - Size: 200, - ModTime: time.Unix(1392337404, 0), - Typeflag: 0x30, - Linkname: "", - Uname: "david", - Gname: "david", - Devmajor: 0, - Devminor: 0, - }, - { - Name: "end", - Mode: 420, - Uid: 1000, - Gid: 1000, - Size: 4, - ModTime: time.Unix(1392398319, 0), - Typeflag: 0x30, - Linkname: "", - Uname: "david", - Gname: "david", - Devmajor: 0, - Devminor: 0, - }, - }, - cksums: []string{ - "6f53234398c2449fe67c1812d993012f", - "6f53234398c2449fe67c1812d993012f", - "6f53234398c2449fe67c1812d993012f", - "6f53234398c2449fe67c1812d993012f", - "b0061974914468de549a2af8ced10316", - }, -} - -var untarTests = []*untarTest{ - gnuTarTest, - sparseTarTest, - { - file: "testdata/star.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244592783, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - AccessTime: time.Unix(1244592783, 0), - ChangeTime: time.Unix(1244592783, 0), - }, - { - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244592783, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - AccessTime: time.Unix(1244592783, 0), - ChangeTime: time.Unix(1244592783, 0), - }, - }, - }, - { - file: "testdata/v7.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0444, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1244593104, 0), - Typeflag: '\x00', - }, - { - Name: "small2.txt", - Mode: 0444, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1244593104, 0), - Typeflag: '\x00', - }, - }, - }, - { - file: "testdata/pax.tar", - headers: []*Header{ - { - Name: "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", - Mode: 0664, - Uid: 1000, - Gid: 1000, - Uname: "shane", - Gname: "shane", - Size: 7, - ModTime: time.Unix(1350244992, 23960108), - ChangeTime: time.Unix(1350244992, 23960108), - AccessTime: time.Unix(1350244992, 23960108), - Typeflag: TypeReg, - }, - { - Name: "a/b", - Mode: 0777, - Uid: 1000, - Gid: 1000, - Uname: "shane", - Gname: "shane", - Size: 0, - ModTime: time.Unix(1350266320, 910238425), - ChangeTime: time.Unix(1350266320, 910238425), - AccessTime: time.Unix(1350266320, 910238425), - Typeflag: TypeSymlink, - Linkname: "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", - }, - }, - }, - { - file: "testdata/nil-uid.tar", // golang.org/issue/5290 - headers: []*Header{ - { - Name: "P1050238.JPG.log", - Mode: 0664, - Uid: 0, - Gid: 0, - Size: 14, - ModTime: time.Unix(1365454838, 0), - Typeflag: TypeReg, - Linkname: "", - Uname: "eyefi", - Gname: "eyefi", - Devmajor: 0, - Devminor: 0, - }, - }, - }, - { - file: "testdata/xattrs.tar", - headers: []*Header{ - { - Name: "small.txt", - Mode: 0644, - Uid: 1000, - Gid: 10, - Size: 5, - ModTime: time.Unix(1386065770, 448252320), - Typeflag: '0', - Uname: "alex", - Gname: "wheel", - AccessTime: time.Unix(1389782991, 419875220), - ChangeTime: time.Unix(1389782956, 794414986), - Xattrs: map[string]string{ - "user.key": "value", - "user.key2": "value2", - // Interestingly, selinux encodes the terminating null inside the xattr - "security.selinux": "unconfined_u:object_r:default_t:s0\x00", - }, - }, - { - Name: "small2.txt", - Mode: 0644, - Uid: 1000, - Gid: 10, - Size: 11, - ModTime: time.Unix(1386065770, 449252304), - Typeflag: '0', - Uname: "alex", - Gname: "wheel", - AccessTime: time.Unix(1389782991, 419875220), - ChangeTime: time.Unix(1386065770, 449252304), - Xattrs: map[string]string{ - "security.selinux": "unconfined_u:object_r:default_t:s0\x00", - }, - }, - }, - }, -} - -func TestReader(t *testing.T) { -testLoop: - for i, test := range untarTests { - f, err := os.Open(test.file) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - continue - } - defer f.Close() - tr := NewReader(f) - for j, header := range test.headers { - hdr, err := tr.Next() - if err != nil || hdr == nil { - t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) - f.Close() - continue testLoop - } - if !reflect.DeepEqual(*hdr, *header) { - t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", - i, j, *hdr, *header) - } - } - hdr, err := tr.Next() - if err == io.EOF { - continue testLoop - } - if hdr != nil || err != nil { - t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err) - } - } -} - -func TestPartialRead(t *testing.T) { - f, err := os.Open("testdata/gnu.tar") - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - - // Read the first four bytes; Next() should skip the last byte. - hdr, err := tr.Next() - if err != nil || hdr == nil { - t.Fatalf("Didn't get first file: %v", err) - } - buf := make([]byte, 4) - if _, err := io.ReadFull(tr, buf); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if expected := []byte("Kilt"); !bytes.Equal(buf, expected) { - t.Errorf("Contents = %v, want %v", buf, expected) - } - - // Second file - hdr, err = tr.Next() - if err != nil || hdr == nil { - t.Fatalf("Didn't get second file: %v", err) - } - buf = make([]byte, 6) - if _, err := io.ReadFull(tr, buf); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if expected := []byte("Google"); !bytes.Equal(buf, expected) { - t.Errorf("Contents = %v, want %v", buf, expected) - } -} - -func TestIncrementalRead(t *testing.T) { - test := gnuTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - - headers := test.headers - cksums := test.cksums - nread := 0 - - // loop over all files - for ; ; nread++ { - hdr, err := tr.Next() - if hdr == nil || err == io.EOF { - break - } - - // check the header - if !reflect.DeepEqual(*hdr, *headers[nread]) { - t.Errorf("Incorrect header:\nhave %+v\nwant %+v", - *hdr, headers[nread]) - } - - // read file contents in little chunks EOF, - // checksumming all the way - h := md5.New() - rdbuf := make([]uint8, 8) - for { - nr, err := tr.Read(rdbuf) - if err == io.EOF { - break - } - if err != nil { - t.Errorf("Read: unexpected error %v\n", err) - break - } - h.Write(rdbuf[0:nr]) - } - // verify checksum - have := fmt.Sprintf("%x", h.Sum(nil)) - want := cksums[nread] - if want != have { - t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) - } - } - if nread != len(headers) { - t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) - } -} - -func TestNonSeekable(t *testing.T) { - test := gnuTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - type readerOnly struct { - io.Reader - } - tr := NewReader(readerOnly{f}) - nread := 0 - - for ; ; nread++ { - _, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - } - - if nread != len(test.headers) { - t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread) - } -} - -func TestParsePAXHeader(t *testing.T) { - paxTests := [][3]string{ - {"a", "a=name", "10 a=name\n"}, // Test case involving multiple acceptable lengths - {"a", "a=name", "9 a=name\n"}, // Test case involving multiple acceptable length - {"mtime", "mtime=1350244992.023960108", "30 mtime=1350244992.023960108\n"}} - for _, test := range paxTests { - key, expected, raw := test[0], test[1], test[2] - reader := bytes.NewReader([]byte(raw)) - headers, err := parsePAX(reader) - if err != nil { - t.Errorf("Couldn't parse correctly formatted headers: %v", err) - continue - } - if strings.EqualFold(headers[key], expected) { - t.Errorf("mtime header incorrectly parsed: got %s, wanted %s", headers[key], expected) - continue - } - trailer := make([]byte, 100) - n, err := reader.Read(trailer) - if err != io.EOF || n != 0 { - t.Error("Buffer wasn't consumed") - } - } - badHeader := bytes.NewReader([]byte("3 somelongkey=")) - if _, err := parsePAX(badHeader); err != ErrHeader { - t.Fatal("Unexpected success when parsing bad header") - } -} - -func TestParsePAXTime(t *testing.T) { - // Some valid PAX time values - timestamps := map[string]time.Time{ - "1350244992.023960108": time.Unix(1350244992, 23960108), // The common case - "1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value - "1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value - "1350244992": time.Unix(1350244992, 0), // Low precision value - } - for input, expected := range timestamps { - ts, err := parsePAXTime(input) - if err != nil { - t.Fatal(err) - } - if !ts.Equal(expected) { - t.Fatalf("Time parsing failure %s %s", ts, expected) - } - } -} - -func TestMergePAX(t *testing.T) { - hdr := new(Header) - // Test a string, integer, and time based value. - headers := map[string]string{ - "path": "a/b/c", - "uid": "1000", - "mtime": "1350244992.023960108", - } - err := mergePAX(hdr, headers) - if err != nil { - t.Fatal(err) - } - want := &Header{ - Name: "a/b/c", - Uid: 1000, - ModTime: time.Unix(1350244992, 23960108), - } - if !reflect.DeepEqual(hdr, want) { - t.Errorf("incorrect merge: got %+v, want %+v", hdr, want) - } -} - -func TestSparseEndToEnd(t *testing.T) { - test := sparseTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - - headers := test.headers - cksums := test.cksums - nread := 0 - - // loop over all files - for ; ; nread++ { - hdr, err := tr.Next() - if hdr == nil || err == io.EOF { - break - } - - // check the header - if !reflect.DeepEqual(*hdr, *headers[nread]) { - t.Errorf("Incorrect header:\nhave %+v\nwant %+v", - *hdr, headers[nread]) - } - - // read and checksum the file data - h := md5.New() - _, err = io.Copy(h, tr) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - // verify checksum - have := fmt.Sprintf("%x", h.Sum(nil)) - want := cksums[nread] - if want != have { - t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) - } - } - if nread != len(headers) { - t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) - } -} - -type sparseFileReadTest struct { - sparseData []byte - sparseMap []sparseEntry - realSize int64 - expected []byte -} - -var sparseFileReadTests = []sparseFileReadTest{ - { - sparseData: []byte("abcde"), - sparseMap: []sparseEntry{ - {offset: 0, numBytes: 2}, - {offset: 5, numBytes: 3}, - }, - realSize: 8, - expected: []byte("ab\x00\x00\x00cde"), - }, - { - sparseData: []byte("abcde"), - sparseMap: []sparseEntry{ - {offset: 0, numBytes: 2}, - {offset: 5, numBytes: 3}, - }, - realSize: 10, - expected: []byte("ab\x00\x00\x00cde\x00\x00"), - }, - { - sparseData: []byte("abcde"), - sparseMap: []sparseEntry{ - {offset: 1, numBytes: 3}, - {offset: 6, numBytes: 2}, - }, - realSize: 8, - expected: []byte("\x00abc\x00\x00de"), - }, - { - sparseData: []byte("abcde"), - sparseMap: []sparseEntry{ - {offset: 1, numBytes: 3}, - {offset: 6, numBytes: 2}, - }, - realSize: 10, - expected: []byte("\x00abc\x00\x00de\x00\x00"), - }, - { - sparseData: []byte(""), - sparseMap: nil, - realSize: 2, - expected: []byte("\x00\x00"), - }, -} - -func TestSparseFileReader(t *testing.T) { - for i, test := range sparseFileReadTests { - r := bytes.NewReader(test.sparseData) - nb := int64(r.Len()) - sfr := &sparseFileReader{ - rfr: ®FileReader{r: r, nb: nb}, - sp: test.sparseMap, - pos: 0, - tot: test.realSize, - } - if sfr.numBytes() != nb { - t.Errorf("test %d: Before reading, sfr.numBytes() = %d, want %d", i, sfr.numBytes(), nb) - } - buf, err := ioutil.ReadAll(sfr) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - } - if e := test.expected; !bytes.Equal(buf, e) { - t.Errorf("test %d: Contents = %v, want %v", i, buf, e) - } - if sfr.numBytes() != 0 { - t.Errorf("test %d: After draining the reader, numBytes() was nonzero", i) - } - } -} - -func TestSparseIncrementalRead(t *testing.T) { - sparseMap := []sparseEntry{{10, 2}} - sparseData := []byte("Go") - expected := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Go\x00\x00\x00\x00\x00\x00\x00\x00" - - r := bytes.NewReader(sparseData) - nb := int64(r.Len()) - sfr := &sparseFileReader{ - rfr: ®FileReader{r: r, nb: nb}, - sp: sparseMap, - pos: 0, - tot: int64(len(expected)), - } - - // We'll read the data 6 bytes at a time, with a hole of size 10 at - // the beginning and one of size 8 at the end. - var outputBuf bytes.Buffer - buf := make([]byte, 6) - for { - n, err := sfr.Read(buf) - if err == io.EOF { - break - } - if err != nil { - t.Errorf("Read: unexpected error %v\n", err) - } - if n > 0 { - _, err := outputBuf.Write(buf[:n]) - if err != nil { - t.Errorf("Write: unexpected error %v\n", err) - } - } - } - got := outputBuf.String() - if got != expected { - t.Errorf("Contents = %v, want %v", got, expected) - } -} - -func TestReadGNUSparseMap0x1(t *testing.T) { - headers := map[string]string{ - paxGNUSparseNumBlocks: "4", - paxGNUSparseMap: "0,5,10,5,20,5,30,5", - } - expected := []sparseEntry{ - {offset: 0, numBytes: 5}, - {offset: 10, numBytes: 5}, - {offset: 20, numBytes: 5}, - {offset: 30, numBytes: 5}, - } - - sp, err := readGNUSparseMap0x1(headers) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !reflect.DeepEqual(sp, expected) { - t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected) - } -} - -func TestReadGNUSparseMap1x0(t *testing.T) { - // This test uses lots of holes so the sparse header takes up more than two blocks - numEntries := 100 - expected := make([]sparseEntry, 0, numEntries) - sparseMap := new(bytes.Buffer) - - fmt.Fprintf(sparseMap, "%d\n", numEntries) - for i := 0; i < numEntries; i++ { - offset := int64(2048 * i) - numBytes := int64(1024) - expected = append(expected, sparseEntry{offset: offset, numBytes: numBytes}) - fmt.Fprintf(sparseMap, "%d\n%d\n", offset, numBytes) - } - - // Make the header the smallest multiple of blockSize that fits the sparseMap - headerBlocks := (sparseMap.Len() + blockSize - 1) / blockSize - bufLen := blockSize * headerBlocks - buf := make([]byte, bufLen) - copy(buf, sparseMap.Bytes()) - - // Get an reader to read the sparse map - r := bytes.NewReader(buf) - - // Read the sparse map - sp, err := readGNUSparseMap1x0(r) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !reflect.DeepEqual(sp, expected) { - t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected) - } -} - -func TestUninitializedRead(t *testing.T) { - test := gnuTarTest - f, err := os.Open(test.file) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - defer f.Close() - - tr := NewReader(f) - _, err = tr.Read([]byte{}) - if err == nil || err != io.EOF { - t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF) - } - -} diff --git a/src/pkg/archive/tar/stat_atim.go b/src/pkg/archive/tar/stat_atim.go deleted file mode 100644 index cf9cc79c5..000000000 --- a/src/pkg/archive/tar/stat_atim.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux dragonfly openbsd solaris - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atim.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctim.Unix()) -} diff --git a/src/pkg/archive/tar/stat_atimespec.go b/src/pkg/archive/tar/stat_atimespec.go deleted file mode 100644 index 6f17dbe30..000000000 --- a/src/pkg/archive/tar/stat_atimespec.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin freebsd netbsd - -package tar - -import ( - "syscall" - "time" -) - -func statAtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Atimespec.Unix()) -} - -func statCtime(st *syscall.Stat_t) time.Time { - return time.Unix(st.Ctimespec.Unix()) -} diff --git a/src/pkg/archive/tar/stat_unix.go b/src/pkg/archive/tar/stat_unix.go deleted file mode 100644 index cb843db4c..000000000 --- a/src/pkg/archive/tar/stat_unix.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux darwin dragonfly freebsd openbsd netbsd solaris - -package tar - -import ( - "os" - "syscall" -) - -func init() { - sysStat = statUnix -} - -func statUnix(fi os.FileInfo, h *Header) error { - sys, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return nil - } - h.Uid = int(sys.Uid) - h.Gid = int(sys.Gid) - // TODO(bradfitz): populate username & group. os/user - // doesn't cache LookupId lookups, and lacks group - // lookup functions. - h.AccessTime = statAtime(sys) - h.ChangeTime = statCtime(sys) - // TODO(bradfitz): major/minor device numbers? - return nil -} diff --git a/src/pkg/archive/tar/tar_test.go b/src/pkg/archive/tar/tar_test.go deleted file mode 100644 index ed333f3ea..000000000 --- a/src/pkg/archive/tar/tar_test.go +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "io/ioutil" - "os" - "path" - "reflect" - "strings" - "testing" - "time" -) - -func TestFileInfoHeader(t *testing.T) { - fi, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - h, err := FileInfoHeader(fi, "") - if err != nil { - t.Fatalf("FileInfoHeader: %v", err) - } - if g, e := h.Name, "small.txt"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e { - t.Errorf("Mode = %#o; want %#o", g, e) - } - if g, e := h.Size, int64(5); g != e { - t.Errorf("Size = %v; want %v", g, e) - } - if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { - t.Errorf("ModTime = %v; want %v", g, e) - } - // FileInfoHeader should error when passing nil FileInfo - if _, err := FileInfoHeader(nil, ""); err == nil { - t.Fatalf("Expected error when passing nil to FileInfoHeader") - } -} - -func TestFileInfoHeaderDir(t *testing.T) { - fi, err := os.Stat("testdata") - if err != nil { - t.Fatal(err) - } - h, err := FileInfoHeader(fi, "") - if err != nil { - t.Fatalf("FileInfoHeader: %v", err) - } - if g, e := h.Name, "testdata/"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - // Ignoring c_ISGID for golang.org/issue/4867 - if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e { - t.Errorf("Mode = %#o; want %#o", g, e) - } - if g, e := h.Size, int64(0); g != e { - t.Errorf("Size = %v; want %v", g, e) - } - if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { - t.Errorf("ModTime = %v; want %v", g, e) - } -} - -func TestFileInfoHeaderSymlink(t *testing.T) { - h, err := FileInfoHeader(symlink{}, "some-target") - if err != nil { - t.Fatal(err) - } - if g, e := h.Name, "some-symlink"; g != e { - t.Errorf("Name = %q; want %q", g, e) - } - if g, e := h.Linkname, "some-target"; g != e { - t.Errorf("Linkname = %q; want %q", g, e) - } -} - -type symlink struct{} - -func (symlink) Name() string { return "some-symlink" } -func (symlink) Size() int64 { return 0 } -func (symlink) Mode() os.FileMode { return os.ModeSymlink } -func (symlink) ModTime() time.Time { return time.Time{} } -func (symlink) IsDir() bool { return false } -func (symlink) Sys() interface{} { return nil } - -func TestRoundTrip(t *testing.T) { - data := []byte("some file contents") - - var b bytes.Buffer - tw := NewWriter(&b) - hdr := &Header{ - Name: "file.txt", - Uid: 1 << 21, // too big for 8 octal digits - Size: int64(len(data)), - ModTime: time.Now(), - } - // tar only supports second precision. - hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) - if err := tw.WriteHeader(hdr); err != nil { - t.Fatalf("tw.WriteHeader: %v", err) - } - if _, err := tw.Write(data); err != nil { - t.Fatalf("tw.Write: %v", err) - } - if err := tw.Close(); err != nil { - t.Fatalf("tw.Close: %v", err) - } - - // Read it back. - tr := NewReader(&b) - rHdr, err := tr.Next() - if err != nil { - t.Fatalf("tr.Next: %v", err) - } - if !reflect.DeepEqual(rHdr, hdr) { - t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr) - } - rData, err := ioutil.ReadAll(tr) - if err != nil { - t.Fatalf("Read: %v", err) - } - if !bytes.Equal(rData, data) { - t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data) - } -} - -type headerRoundTripTest struct { - h *Header - fm os.FileMode -} - -func TestHeaderRoundTrip(t *testing.T) { - golden := []headerRoundTripTest{ - // regular file. - { - h: &Header{ - Name: "test.txt", - Mode: 0644 | c_ISREG, - Size: 12, - ModTime: time.Unix(1360600916, 0), - Typeflag: TypeReg, - }, - fm: 0644, - }, - // hard link. - { - h: &Header{ - Name: "hard.txt", - Mode: 0644 | c_ISLNK, - Size: 0, - ModTime: time.Unix(1360600916, 0), - Typeflag: TypeLink, - }, - fm: 0644 | os.ModeSymlink, - }, - // symbolic link. - { - h: &Header{ - Name: "link.txt", - Mode: 0777 | c_ISLNK, - Size: 0, - ModTime: time.Unix(1360600852, 0), - Typeflag: TypeSymlink, - }, - fm: 0777 | os.ModeSymlink, - }, - // character device node. - { - h: &Header{ - Name: "dev/null", - Mode: 0666 | c_ISCHR, - Size: 0, - ModTime: time.Unix(1360578951, 0), - Typeflag: TypeChar, - }, - fm: 0666 | os.ModeDevice | os.ModeCharDevice, - }, - // block device node. - { - h: &Header{ - Name: "dev/sda", - Mode: 0660 | c_ISBLK, - Size: 0, - ModTime: time.Unix(1360578954, 0), - Typeflag: TypeBlock, - }, - fm: 0660 | os.ModeDevice, - }, - // directory. - { - h: &Header{ - Name: "dir/", - Mode: 0755 | c_ISDIR, - Size: 0, - ModTime: time.Unix(1360601116, 0), - Typeflag: TypeDir, - }, - fm: 0755 | os.ModeDir, - }, - // fifo node. - { - h: &Header{ - Name: "dev/initctl", - Mode: 0600 | c_ISFIFO, - Size: 0, - ModTime: time.Unix(1360578949, 0), - Typeflag: TypeFifo, - }, - fm: 0600 | os.ModeNamedPipe, - }, - // setuid. - { - h: &Header{ - Name: "bin/su", - Mode: 0755 | c_ISREG | c_ISUID, - Size: 23232, - ModTime: time.Unix(1355405093, 0), - Typeflag: TypeReg, - }, - fm: 0755 | os.ModeSetuid, - }, - // setguid. - { - h: &Header{ - Name: "group.txt", - Mode: 0750 | c_ISREG | c_ISGID, - Size: 0, - ModTime: time.Unix(1360602346, 0), - Typeflag: TypeReg, - }, - fm: 0750 | os.ModeSetgid, - }, - // sticky. - { - h: &Header{ - Name: "sticky.txt", - Mode: 0600 | c_ISREG | c_ISVTX, - Size: 7, - ModTime: time.Unix(1360602540, 0), - Typeflag: TypeReg, - }, - fm: 0600 | os.ModeSticky, - }, - } - - for i, g := range golden { - fi := g.h.FileInfo() - h2, err := FileInfoHeader(fi, "") - if err != nil { - t.Error(err) - continue - } - if strings.Contains(fi.Name(), "/") { - t.Errorf("FileInfo of %q contains slash: %q", g.h.Name, fi.Name()) - } - name := path.Base(g.h.Name) - if fi.IsDir() { - name += "/" - } - if got, want := h2.Name, name; got != want { - t.Errorf("i=%d: Name: got %v, want %v", i, got, want) - } - if got, want := h2.Size, g.h.Size; got != want { - t.Errorf("i=%d: Size: got %v, want %v", i, got, want) - } - if got, want := h2.Mode, g.h.Mode; got != want { - t.Errorf("i=%d: Mode: got %o, want %o", i, got, want) - } - if got, want := fi.Mode(), g.fm; got != want { - t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want) - } - if got, want := h2.ModTime, g.h.ModTime; got != want { - t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want) - } - if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h { - t.Errorf("i=%d: Sys didn't return original *Header", i) - } - } -} diff --git a/src/pkg/archive/tar/testdata/gnu.tar b/src/pkg/archive/tar/testdata/gnu.tar Binary files differdeleted file mode 100644 index fc899dc8d..000000000 --- a/src/pkg/archive/tar/testdata/gnu.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/nil-uid.tar b/src/pkg/archive/tar/testdata/nil-uid.tar Binary files differdeleted file mode 100644 index cc9cfaa33..000000000 --- a/src/pkg/archive/tar/testdata/nil-uid.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/pax.tar b/src/pkg/archive/tar/testdata/pax.tar Binary files differdeleted file mode 100644 index 9bc24b658..000000000 --- a/src/pkg/archive/tar/testdata/pax.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/small.txt b/src/pkg/archive/tar/testdata/small.txt deleted file mode 100644 index b249bfc51..000000000 --- a/src/pkg/archive/tar/testdata/small.txt +++ /dev/null @@ -1 +0,0 @@ -Kilts
\ No newline at end of file diff --git a/src/pkg/archive/tar/testdata/small2.txt b/src/pkg/archive/tar/testdata/small2.txt deleted file mode 100644 index 394ee3ecd..000000000 --- a/src/pkg/archive/tar/testdata/small2.txt +++ /dev/null @@ -1 +0,0 @@ -Google.com diff --git a/src/pkg/archive/tar/testdata/sparse-formats.tar b/src/pkg/archive/tar/testdata/sparse-formats.tar Binary files differdeleted file mode 100644 index 8bd4e74d5..000000000 --- a/src/pkg/archive/tar/testdata/sparse-formats.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/star.tar b/src/pkg/archive/tar/testdata/star.tar Binary files differdeleted file mode 100644 index 59e2d4e60..000000000 --- a/src/pkg/archive/tar/testdata/star.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/ustar.tar b/src/pkg/archive/tar/testdata/ustar.tar Binary files differdeleted file mode 100644 index 29679d9a3..000000000 --- a/src/pkg/archive/tar/testdata/ustar.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/v7.tar b/src/pkg/archive/tar/testdata/v7.tar Binary files differdeleted file mode 100644 index eb65fc941..000000000 --- a/src/pkg/archive/tar/testdata/v7.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/writer-big-long.tar b/src/pkg/archive/tar/testdata/writer-big-long.tar Binary files differdeleted file mode 100644 index 5960ee824..000000000 --- a/src/pkg/archive/tar/testdata/writer-big-long.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/writer-big.tar b/src/pkg/archive/tar/testdata/writer-big.tar Binary files differdeleted file mode 100644 index 753e883ce..000000000 --- a/src/pkg/archive/tar/testdata/writer-big.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/writer.tar b/src/pkg/archive/tar/testdata/writer.tar Binary files differdeleted file mode 100644 index e6d816ad0..000000000 --- a/src/pkg/archive/tar/testdata/writer.tar +++ /dev/null diff --git a/src/pkg/archive/tar/testdata/xattrs.tar b/src/pkg/archive/tar/testdata/xattrs.tar Binary files differdeleted file mode 100644 index 9701950ed..000000000 --- a/src/pkg/archive/tar/testdata/xattrs.tar +++ /dev/null diff --git a/src/pkg/archive/tar/writer.go b/src/pkg/archive/tar/writer.go deleted file mode 100644 index 6eff6f6f8..000000000 --- a/src/pkg/archive/tar/writer.go +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -// TODO(dsymonds): -// - catch more errors (no first header, etc.) - -import ( - "bytes" - "errors" - "fmt" - "io" - "os" - "path" - "strconv" - "strings" - "time" -) - -var ( - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errNameTooLong = errors.New("archive/tar: name too long") - errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") -) - -// A Writer provides sequential writing of a tar archive in POSIX.1 format. -// A tar archive consists of a sequence of files. -// Call WriteHeader to begin a new file, and then call Write to supply that file's data, -// writing at most hdr.Size bytes in total. -type Writer struct { - w io.Writer - err error - nb int64 // number of unwritten bytes for current file entry - pad int64 // amount of padding to write after current file entry - closed bool - usedBinary bool // whether the binary numeric field extension was used - preferPax bool // use pax header instead of binary numeric header -} - -// NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } - -// Flush finishes writing the current file (optional). -func (tw *Writer) Flush() error { - if tw.nb > 0 { - tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb) - return tw.err - } - - n := tw.nb + tw.pad - for n > 0 && tw.err == nil { - nr := n - if nr > blockSize { - nr = blockSize - } - var nw int - nw, tw.err = tw.w.Write(zeroBlock[0:nr]) - n -= int64(nw) - } - tw.nb = 0 - tw.pad = 0 - return tw.err -} - -// Write s into b, terminating it with a NUL if there is room. -// If the value is too long for the field and allowPax is true add a paxheader record instead -func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) { - needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s) - if needsPaxHeader { - paxHeaders[paxKeyword] = s - return - } - if len(s) > len(b) { - if tw.err == nil { - tw.err = ErrFieldTooLong - } - return - } - ascii := toASCII(s) - copy(b, ascii) - if len(ascii) < len(b) { - b[len(ascii)] = 0 - } -} - -// Encode x as an octal ASCII string and write it into b with leading zeros. -func (tw *Writer) octal(b []byte, x int64) { - s := strconv.FormatInt(x, 8) - // leading zeros, but leave room for a NUL. - for len(s)+1 < len(b) { - s = "0" + s - } - tw.cString(b, s, false, paxNone, nil) -} - -// Write x into b, either as octal or as binary (GNUtar/star extension). -// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead -func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) { - // Try octal first. - s := strconv.FormatInt(x, 8) - if len(s) < len(b) { - tw.octal(b, x) - return - } - - // If it is too long for octal, and pax is preferred, use a pax header - if allowPax && tw.preferPax { - tw.octal(b, 0) - s := strconv.FormatInt(x, 10) - paxHeaders[paxKeyword] = s - return - } - - // Too big: use binary (big-endian). - tw.usedBinary = true - for i := len(b) - 1; x > 0 && i >= 0; i-- { - b[i] = byte(x) - x >>= 8 - } - b[0] |= 0x80 // highest bit indicates binary format -} - -var ( - minTime = time.Unix(0, 0) - // There is room for 11 octal digits (33 bits) of mtime. - maxTime = minTime.Add((1<<33 - 1) * time.Second) -) - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -func (tw *Writer) WriteHeader(hdr *Header) error { - return tw.writeHeader(hdr, true) -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -// As this method is called internally by writePax header to allow it to -// suppress writing the pax header. -func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { - if tw.closed { - return ErrWriteAfterClose - } - if tw.err == nil { - tw.Flush() - } - if tw.err != nil { - return tw.err - } - - // a map to hold pax header records, if any are needed - paxHeaders := make(map[string]string) - - // TODO(shanemhansen): we might want to use PAX headers for - // subsecond time resolution, but for now let's just capture - // too long fields or non ascii characters - - header := make([]byte, blockSize) - s := slicer(header) - - // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax - pathHeaderBytes := s.next(fileNameSize) - - tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders) - - // Handle out of range ModTime carefully. - var modTime int64 - if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { - modTime = hdr.ModTime.Unix() - } - - tw.octal(s.next(8), hdr.Mode) // 100:108 - tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116 - tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124 - tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136 - tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity - s.next(8) // chksum (148:156) - s.next(1)[0] = hdr.Typeflag // 156:157 - - tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders) - - copy(s.next(8), []byte("ustar\x0000")) // 257:265 - tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297 - tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329 - tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337 - tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345 - - // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax - prefixHeaderBytes := s.next(155) - tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix - - // Use the GNU magic instead of POSIX magic if we used any GNU extensions. - if tw.usedBinary { - copy(header[257:265], []byte("ustar \x00")) - } - - _, paxPathUsed := paxHeaders[paxPath] - // try to use a ustar header when only the name is too long - if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { - suffix := hdr.Name - prefix := "" - if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) { - var err error - prefix, suffix, err = tw.splitUSTARLongName(hdr.Name) - if err == nil { - // ok we can use a ustar long name instead of pax, now correct the fields - - // remove the path field from the pax header. this will suppress the pax header - delete(paxHeaders, paxPath) - - // update the path fields - tw.cString(pathHeaderBytes, suffix, false, paxNone, nil) - tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil) - - // Use the ustar magic if we used ustar long names. - if len(prefix) > 0 && !tw.usedBinary { - copy(header[257:265], []byte("ustar\x00")) - } - } - } - } - - // The chksum field is terminated by a NUL and a space. - // This is different from the other octal fields. - chksum, _ := checksum(header) - tw.octal(header[148:155], chksum) - header[155] = ' ' - - if tw.err != nil { - // problem with header; probably integer too big for a field. - return tw.err - } - - if allowPax { - for k, v := range hdr.Xattrs { - paxHeaders[paxXattr+k] = v - } - } - - if len(paxHeaders) > 0 { - if !allowPax { - return errInvalidHeader - } - if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { - return err - } - } - tw.nb = int64(hdr.Size) - tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize - - _, tw.err = tw.w.Write(header) - return tw.err -} - -// writeUSTARLongName splits a USTAR long name hdr.Name. -// name must be < 256 characters. errNameTooLong is returned -// if hdr.Name can't be split. The splitting heuristic -// is compatible with gnu tar. -func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) { - length := len(name) - if length > fileNamePrefixSize+1 { - length = fileNamePrefixSize + 1 - } else if name[length-1] == '/' { - length-- - } - i := strings.LastIndex(name[:length], "/") - // nlen contains the resulting length in the name field. - // plen contains the resulting length in the prefix field. - nlen := len(name) - i - 1 - plen := i - if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize { - err = errNameTooLong - return - } - prefix, suffix = name[:i], name[i+1:] - return -} - -// writePaxHeader writes an extended pax header to the -// archive. -func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { - // Prepare extended header - ext := new(Header) - ext.Typeflag = TypeXHeader - // Setting ModTime is required for reader parsing to - // succeed, and seems harmless enough. - ext.ModTime = hdr.ModTime - // The spec asks that we namespace our pseudo files - // with the current pid. - pid := os.Getpid() - dir, file := path.Split(hdr.Name) - fullName := path.Join(dir, - fmt.Sprintf("PaxHeaders.%d", pid), file) - - ascii := toASCII(fullName) - if len(ascii) > 100 { - ascii = ascii[:100] - } - ext.Name = ascii - // Construct the body - var buf bytes.Buffer - - for k, v := range paxHeaders { - fmt.Fprint(&buf, paxHeader(k+"="+v)) - } - - ext.Size = int64(len(buf.Bytes())) - if err := tw.writeHeader(ext, false); err != nil { - return err - } - if _, err := tw.Write(buf.Bytes()); err != nil { - return err - } - if err := tw.Flush(); err != nil { - return err - } - return nil -} - -// paxHeader formats a single pax record, prefixing it with the appropriate length -func paxHeader(msg string) string { - const padding = 2 // Extra padding for space and newline - size := len(msg) + padding - size += len(strconv.Itoa(size)) - record := fmt.Sprintf("%d %s\n", size, msg) - if len(record) != size { - // Final adjustment if adding size increased - // the number of digits in size - size = len(record) - record = fmt.Sprintf("%d %s\n", size, msg) - } - return record -} - -// Write writes to the current entry in the tar archive. -// Write returns the error ErrWriteTooLong if more than -// hdr.Size bytes are written after WriteHeader. -func (tw *Writer) Write(b []byte) (n int, err error) { - if tw.closed { - err = ErrWriteTooLong - return - } - overwrite := false - if int64(len(b)) > tw.nb { - b = b[0:tw.nb] - overwrite = true - } - n, err = tw.w.Write(b) - tw.nb -= int64(n) - if err == nil && overwrite { - err = ErrWriteTooLong - return - } - tw.err = err - return -} - -// Close closes the tar archive, flushing any unwritten -// data to the underlying writer. -func (tw *Writer) Close() error { - if tw.err != nil || tw.closed { - return tw.err - } - tw.Flush() - tw.closed = true - if tw.err != nil { - return tw.err - } - - // trailer: two zero blocks - for i := 0; i < 2; i++ { - _, tw.err = tw.w.Write(zeroBlock) - if tw.err != nil { - break - } - } - return tw.err -} diff --git a/src/pkg/archive/tar/writer_test.go b/src/pkg/archive/tar/writer_test.go deleted file mode 100644 index 512fab1a6..000000000 --- a/src/pkg/archive/tar/writer_test.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tar - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "reflect" - "strings" - "testing" - "testing/iotest" - "time" -) - -type writerTestEntry struct { - header *Header - contents string -} - -type writerTest struct { - file string // filename of expected output - entries []*writerTestEntry -} - -var writerTests = []*writerTest{ - // The writer test file was produced with this command: - // tar (GNU tar) 1.26 - // ln -s small.txt link.txt - // tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt - { - file: "testdata/writer.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: "small.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 5, - ModTime: time.Unix(1246508266, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - contents: "Kilts", - }, - { - header: &Header{ - Name: "small2.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 11, - ModTime: time.Unix(1245217492, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - contents: "Google.com\n", - }, - { - header: &Header{ - Name: "link.txt", - Mode: 0777, - Uid: 1000, - Gid: 1000, - Size: 0, - ModTime: time.Unix(1314603082, 0), - Typeflag: '2', - Linkname: "small.txt", - Uname: "strings", - Gname: "strings", - }, - // no contents - }, - }, - }, - // The truncated test file was produced using these commands: - // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt - // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar - { - file: "testdata/writer-big.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: "tmp/16gig.txt", - Mode: 0640, - Uid: 73025, - Gid: 5000, - Size: 16 << 30, - ModTime: time.Unix(1254699560, 0), - Typeflag: '0', - Uname: "dsymonds", - Gname: "eng", - }, - // fake contents - contents: strings.Repeat("\x00", 4<<10), - }, - }, - }, - // The truncated test file was produced using these commands: - // dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt - // tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar - { - file: "testdata/writer-big-long.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: strings.Repeat("longname/", 15) + "16gig.txt", - Mode: 0644, - Uid: 1000, - Gid: 1000, - Size: 16 << 30, - ModTime: time.Unix(1399583047, 0), - Typeflag: '0', - Uname: "guillaume", - Gname: "guillaume", - }, - // fake contents - contents: strings.Repeat("\x00", 4<<10), - }, - }, - }, - // This file was produced using gnu tar 1.17 - // gnutar -b 4 --format=ustar (longname/)*15 + file.txt - { - file: "testdata/ustar.tar", - entries: []*writerTestEntry{ - { - header: &Header{ - Name: strings.Repeat("longname/", 15) + "file.txt", - Mode: 0644, - Uid: 0765, - Gid: 024, - Size: 06, - ModTime: time.Unix(1360135598, 0), - Typeflag: '0', - Uname: "shane", - Gname: "staff", - }, - contents: "hello\n", - }, - }, - }, -} - -// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection. -func bytestr(offset int, b []byte) string { - const rowLen = 32 - s := fmt.Sprintf("%04x ", offset) - for _, ch := range b { - switch { - case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z': - s += fmt.Sprintf(" %c", ch) - default: - s += fmt.Sprintf(" %02x", ch) - } - } - return s -} - -// Render a pseudo-diff between two blocks of bytes. -func bytediff(a []byte, b []byte) string { - const rowLen = 32 - s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b)) - for offset := 0; len(a)+len(b) > 0; offset += rowLen { - na, nb := rowLen, rowLen - if na > len(a) { - na = len(a) - } - if nb > len(b) { - nb = len(b) - } - sa := bytestr(offset, a[0:na]) - sb := bytestr(offset, b[0:nb]) - if sa != sb { - s += fmt.Sprintf("-%v\n+%v\n", sa, sb) - } - a = a[na:] - b = b[nb:] - } - return s -} - -func TestWriter(t *testing.T) { -testLoop: - for i, test := range writerTests { - expected, err := ioutil.ReadFile(test.file) - if err != nil { - t.Errorf("test %d: Unexpected error: %v", i, err) - continue - } - - buf := new(bytes.Buffer) - tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB - big := false - for j, entry := range test.entries { - big = big || entry.header.Size > 1<<10 - if err := tw.WriteHeader(entry.header); err != nil { - t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err) - continue testLoop - } - if _, err := io.WriteString(tw, entry.contents); err != nil { - t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err) - continue testLoop - } - } - // Only interested in Close failures for the small tests. - if err := tw.Close(); err != nil && !big { - t.Errorf("test %d: Failed closing archive: %v", i, err) - continue testLoop - } - - actual := buf.Bytes() - if !bytes.Equal(expected, actual) { - t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v", - i, bytediff(expected, actual)) - } - if testing.Short() { // The second test is expensive. - break - } - } -} - -func TestPax(t *testing.T) { - // Create an archive with a large name - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - if err != nil { - t.Fatalf("os.Stat: %v", err) - } - // Force a PAX long name to be written - longName := strings.Repeat("ab", 100) - contents := strings.Repeat(" ", int(hdr.Size)) - hdr.Name = longName - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err = writer.Write([]byte(contents)); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != longName { - t.Fatal("Couldn't recover long file name") - } -} - -func TestPaxSymlink(t *testing.T) { - // Create an archive with a large linkname - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - hdr.Typeflag = TypeSymlink - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - // Force a PAX long linkname to be written - longLinkname := strings.Repeat("1234567890/1234567890", 10) - hdr.Linkname = longLinkname - - hdr.Size = 0 - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Linkname != longLinkname { - t.Fatal("Couldn't recover long link name") - } -} - -func TestPaxNonAscii(t *testing.T) { - // Create an archive with non ascii. These should trigger a pax header - // because pax headers have a defined utf-8 encoding. - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - - hdr, err := FileInfoHeader(fileinfo, "") - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - - // some sample data - chineseFilename := "文件名" - chineseGroupname := "組" - chineseUsername := "用戶名" - - hdr.Name = chineseFilename - hdr.Gname = chineseGroupname - hdr.Uname = chineseUsername - - contents := strings.Repeat(" ", int(hdr.Size)) - - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err = writer.Write([]byte(contents)); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Simple test to make sure PAX extensions are in effect - if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { - t.Fatal("Expected at least one PAX header to be written.") - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != chineseFilename { - t.Fatal("Couldn't recover unicode name") - } - if hdr.Gname != chineseGroupname { - t.Fatal("Couldn't recover unicode group") - } - if hdr.Uname != chineseUsername { - t.Fatal("Couldn't recover unicode user") - } -} - -func TestPaxXattrs(t *testing.T) { - xattrs := map[string]string{ - "user.key": "value", - } - - // Create an archive with an xattr - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - if err != nil { - t.Fatalf("os.Stat: %v", err) - } - contents := "Kilts" - hdr.Xattrs = xattrs - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if _, err = writer.Write([]byte(contents)); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Test that we can get the xattrs back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(hdr.Xattrs, xattrs) { - t.Fatalf("xattrs did not survive round trip: got %+v, want %+v", - hdr.Xattrs, xattrs) - } -} - -func TestPAXHeader(t *testing.T) { - medName := strings.Repeat("CD", 50) - longName := strings.Repeat("AB", 100) - paxTests := [][2]string{ - {paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"}, - {"a=b", "6 a=b\n"}, // Single digit length - {"a=names", "11 a=names\n"}, // Test case involving carries - {paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)}, - {paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}} - - for _, test := range paxTests { - key, expected := test[0], test[1] - if result := paxHeader(key); result != expected { - t.Fatalf("paxHeader: got %s, expected %s", result, expected) - } - } -} - -func TestUSTARLongName(t *testing.T) { - // Create an archive with a path that failed to split with USTAR extension in previous versions. - fileinfo, err := os.Stat("testdata/small.txt") - if err != nil { - t.Fatal(err) - } - hdr, err := FileInfoHeader(fileinfo, "") - hdr.Typeflag = TypeDir - if err != nil { - t.Fatalf("os.Stat:1 %v", err) - } - // Force a PAX long name to be written. The name was taken from a practical example - // that fails and replaced ever char through numbers to anonymize the sample. - longName := "/0000_0000000/00000-000000000/0000_0000000/00000-0000000000000/0000_0000000/00000-0000000-00000000/0000_0000000/00000000/0000_0000000/000/0000_0000000/00000000v00/0000_0000000/000000/0000_0000000/0000000/0000_0000000/00000y-00/0000/0000/00000000/0x000000/" - hdr.Name = longName - - hdr.Size = 0 - var buf bytes.Buffer - writer := NewWriter(&buf) - if err := writer.WriteHeader(hdr); err != nil { - t.Fatal(err) - } - if err := writer.Close(); err != nil { - t.Fatal(err) - } - // Test that we can get a long name back out of the archive. - reader := NewReader(&buf) - hdr, err = reader.Next() - if err != nil { - t.Fatal(err) - } - if hdr.Name != longName { - t.Fatal("Couldn't recover long name") - } -} diff --git a/src/pkg/archive/zip/example_test.go b/src/pkg/archive/zip/example_test.go deleted file mode 100644 index c2ed9e79c..000000000 --- a/src/pkg/archive/zip/example_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip_test - -import ( - "archive/zip" - "bytes" - "fmt" - "io" - "log" - "os" -) - -func ExampleWriter() { - // Create a buffer to write our archive to. - buf := new(bytes.Buffer) - - // Create a new zip archive. - w := zip.NewWriter(buf) - - // Add some files to the archive. - var files = []struct { - Name, Body string - }{ - {"readme.txt", "This archive contains some text files."}, - {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, - {"todo.txt", "Get animal handling licence.\nWrite more examples."}, - } - for _, file := range files { - f, err := w.Create(file.Name) - if err != nil { - log.Fatal(err) - } - _, err = f.Write([]byte(file.Body)) - if err != nil { - log.Fatal(err) - } - } - - // Make sure to check the error on Close. - err := w.Close() - if err != nil { - log.Fatal(err) - } -} - -func ExampleReader() { - // Open a zip archive for reading. - r, err := zip.OpenReader("testdata/readme.zip") - if err != nil { - log.Fatal(err) - } - defer r.Close() - - // Iterate through the files in the archive, - // printing some of their contents. - for _, f := range r.File { - fmt.Printf("Contents of %s:\n", f.Name) - rc, err := f.Open() - if err != nil { - log.Fatal(err) - } - _, err = io.CopyN(os.Stdout, rc, 68) - if err != nil { - log.Fatal(err) - } - rc.Close() - fmt.Println() - } - // Output: - // Contents of README: - // This is the source code repository for the Go programming language. -} diff --git a/src/pkg/archive/zip/reader.go b/src/pkg/archive/zip/reader.go deleted file mode 100644 index 80ee03006..000000000 --- a/src/pkg/archive/zip/reader.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip - -import ( - "bufio" - "encoding/binary" - "errors" - "hash" - "hash/crc32" - "io" - "os" -) - -var ( - ErrFormat = errors.New("zip: not a valid zip file") - ErrAlgorithm = errors.New("zip: unsupported compression algorithm") - ErrChecksum = errors.New("zip: checksum error") -) - -type Reader struct { - r io.ReaderAt - File []*File - Comment string -} - -type ReadCloser struct { - f *os.File - Reader -} - -type File struct { - FileHeader - zipr io.ReaderAt - zipsize int64 - headerOffset int64 -} - -func (f *File) hasDataDescriptor() bool { - return f.Flags&0x8 != 0 -} - -// OpenReader will open the Zip file specified by name and return a ReadCloser. -func OpenReader(name string) (*ReadCloser, error) { - f, err := os.Open(name) - if err != nil { - return nil, err - } - fi, err := f.Stat() - if err != nil { - f.Close() - return nil, err - } - r := new(ReadCloser) - if err := r.init(f, fi.Size()); err != nil { - f.Close() - return nil, err - } - r.f = f - return r, nil -} - -// NewReader returns a new Reader reading from r, which is assumed to -// have the given size in bytes. -func NewReader(r io.ReaderAt, size int64) (*Reader, error) { - zr := new(Reader) - if err := zr.init(r, size); err != nil { - return nil, err - } - return zr, nil -} - -func (z *Reader) init(r io.ReaderAt, size int64) error { - end, err := readDirectoryEnd(r, size) - if err != nil { - return err - } - z.r = r - z.File = make([]*File, 0, end.directoryRecords) - z.Comment = end.comment - rs := io.NewSectionReader(r, 0, size) - if _, err = rs.Seek(int64(end.directoryOffset), os.SEEK_SET); err != nil { - return err - } - buf := bufio.NewReader(rs) - - // The count of files inside a zip is truncated to fit in a uint16. - // Gloss over this by reading headers until we encounter - // a bad one, and then only report a ErrFormat or UnexpectedEOF if - // the file count modulo 65536 is incorrect. - for { - f := &File{zipr: r, zipsize: size} - err = readDirectoryHeader(f, buf) - if err == ErrFormat || err == io.ErrUnexpectedEOF { - break - } - if err != nil { - return err - } - z.File = append(z.File, f) - } - if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here - // Return the readDirectoryHeader error if we read - // the wrong number of directory entries. - return err - } - return nil -} - -// Close closes the Zip file, rendering it unusable for I/O. -func (rc *ReadCloser) Close() error { - return rc.f.Close() -} - -// DataOffset returns the offset of the file's possibly-compressed -// data, relative to the beginning of the zip file. -// -// Most callers should instead use Open, which transparently -// decompresses data and verifies checksums. -func (f *File) DataOffset() (offset int64, err error) { - bodyOffset, err := f.findBodyOffset() - if err != nil { - return - } - return f.headerOffset + bodyOffset, nil -} - -// Open returns a ReadCloser that provides access to the File's contents. -// Multiple files may be read concurrently. -func (f *File) Open() (rc io.ReadCloser, err error) { - bodyOffset, err := f.findBodyOffset() - if err != nil { - return - } - size := int64(f.CompressedSize64) - r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size) - dcomp := decompressor(f.Method) - if dcomp == nil { - err = ErrAlgorithm - return - } - rc = dcomp(r) - var desr io.Reader - if f.hasDataDescriptor() { - desr = io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset+size, dataDescriptorLen) - } - rc = &checksumReader{rc, crc32.NewIEEE(), f, desr, nil} - return -} - -type checksumReader struct { - rc io.ReadCloser - hash hash.Hash32 - f *File - desr io.Reader // if non-nil, where to read the data descriptor - err error // sticky error -} - -func (r *checksumReader) Read(b []byte) (n int, err error) { - if r.err != nil { - return 0, r.err - } - n, err = r.rc.Read(b) - r.hash.Write(b[:n]) - if err == nil { - return - } - if err == io.EOF { - if r.desr != nil { - if err1 := readDataDescriptor(r.desr, r.f); err1 != nil { - err = err1 - } else if r.hash.Sum32() != r.f.CRC32 { - err = ErrChecksum - } - } else { - // If there's not a data descriptor, we still compare - // the CRC32 of what we've read against the file header - // or TOC's CRC32, if it seems like it was set. - if r.f.CRC32 != 0 && r.hash.Sum32() != r.f.CRC32 { - err = ErrChecksum - } - } - } - r.err = err - return -} - -func (r *checksumReader) Close() error { return r.rc.Close() } - -// findBodyOffset does the minimum work to verify the file has a header -// and returns the file body offset. -func (f *File) findBodyOffset() (int64, error) { - var buf [fileHeaderLen]byte - if _, err := f.zipr.ReadAt(buf[:], f.headerOffset); err != nil { - return 0, err - } - b := readBuf(buf[:]) - if sig := b.uint32(); sig != fileHeaderSignature { - return 0, ErrFormat - } - b = b[22:] // skip over most of the header - filenameLen := int(b.uint16()) - extraLen := int(b.uint16()) - return int64(fileHeaderLen + filenameLen + extraLen), nil -} - -// readDirectoryHeader attempts to read a directory header from r. -// It returns io.ErrUnexpectedEOF if it cannot read a complete header, -// and ErrFormat if it doesn't find a valid header signature. -func readDirectoryHeader(f *File, r io.Reader) error { - var buf [directoryHeaderLen]byte - if _, err := io.ReadFull(r, buf[:]); err != nil { - return err - } - b := readBuf(buf[:]) - if sig := b.uint32(); sig != directoryHeaderSignature { - return ErrFormat - } - f.CreatorVersion = b.uint16() - f.ReaderVersion = b.uint16() - f.Flags = b.uint16() - f.Method = b.uint16() - f.ModifiedTime = b.uint16() - f.ModifiedDate = b.uint16() - f.CRC32 = b.uint32() - f.CompressedSize = b.uint32() - f.UncompressedSize = b.uint32() - f.CompressedSize64 = uint64(f.CompressedSize) - f.UncompressedSize64 = uint64(f.UncompressedSize) - filenameLen := int(b.uint16()) - extraLen := int(b.uint16()) - commentLen := int(b.uint16()) - b = b[4:] // skipped start disk number and internal attributes (2x uint16) - f.ExternalAttrs = b.uint32() - f.headerOffset = int64(b.uint32()) - d := make([]byte, filenameLen+extraLen+commentLen) - if _, err := io.ReadFull(r, d); err != nil { - return err - } - f.Name = string(d[:filenameLen]) - f.Extra = d[filenameLen : filenameLen+extraLen] - f.Comment = string(d[filenameLen+extraLen:]) - - if len(f.Extra) > 0 { - b := readBuf(f.Extra) - for len(b) >= 4 { // need at least tag and size - tag := b.uint16() - size := b.uint16() - if int(size) > len(b) { - return ErrFormat - } - if tag == zip64ExtraId { - // update directory values from the zip64 extra block - eb := readBuf(b[:size]) - if len(eb) >= 8 { - f.UncompressedSize64 = eb.uint64() - } - if len(eb) >= 8 { - f.CompressedSize64 = eb.uint64() - } - if len(eb) >= 8 { - f.headerOffset = int64(eb.uint64()) - } - } - b = b[size:] - } - // Should have consumed the whole header. - if len(b) != 0 { - return ErrFormat - } - } - return nil -} - -func readDataDescriptor(r io.Reader, f *File) error { - var buf [dataDescriptorLen]byte - - // The spec says: "Although not originally assigned a - // signature, the value 0x08074b50 has commonly been adopted - // as a signature value for the data descriptor record. - // Implementers should be aware that ZIP files may be - // encountered with or without this signature marking data - // descriptors and should account for either case when reading - // ZIP files to ensure compatibility." - // - // dataDescriptorLen includes the size of the signature but - // first read just those 4 bytes to see if it exists. - if _, err := io.ReadFull(r, buf[:4]); err != nil { - return err - } - off := 0 - maybeSig := readBuf(buf[:4]) - if maybeSig.uint32() != dataDescriptorSignature { - // No data descriptor signature. Keep these four - // bytes. - off += 4 - } - if _, err := io.ReadFull(r, buf[off:12]); err != nil { - return err - } - b := readBuf(buf[:12]) - if b.uint32() != f.CRC32 { - return ErrChecksum - } - - // The two sizes that follow here can be either 32 bits or 64 bits - // but the spec is not very clear on this and different - // interpretations has been made causing incompatibilities. We - // already have the sizes from the central directory so we can - // just ignore these. - - return nil -} - -func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) { - // look for directoryEndSignature in the last 1k, then in the last 65k - var buf []byte - var directoryEndOffset int64 - for i, bLen := range []int64{1024, 65 * 1024} { - if bLen > size { - bLen = size - } - buf = make([]byte, int(bLen)) - if _, err := r.ReadAt(buf, size-bLen); err != nil && err != io.EOF { - return nil, err - } - if p := findSignatureInBlock(buf); p >= 0 { - buf = buf[p:] - directoryEndOffset = size - bLen + int64(p) - break - } - if i == 1 || bLen == size { - return nil, ErrFormat - } - } - - // read header into struct - b := readBuf(buf[4:]) // skip signature - d := &directoryEnd{ - diskNbr: uint32(b.uint16()), - dirDiskNbr: uint32(b.uint16()), - dirRecordsThisDisk: uint64(b.uint16()), - directoryRecords: uint64(b.uint16()), - directorySize: uint64(b.uint32()), - directoryOffset: uint64(b.uint32()), - commentLen: b.uint16(), - } - l := int(d.commentLen) - if l > len(b) { - return nil, errors.New("zip: invalid comment length") - } - d.comment = string(b[:l]) - - p, err := findDirectory64End(r, directoryEndOffset) - if err == nil && p >= 0 { - err = readDirectory64End(r, p, d) - } - if err != nil { - return nil, err - } - - // Make sure directoryOffset points to somewhere in our file. - if o := int64(d.directoryOffset); o < 0 || o >= size { - return nil, ErrFormat - } - return d, nil -} - -// findDirectory64End tries to read the zip64 locator just before the -// directory end and returns the offset of the zip64 directory end if -// found. -func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) { - locOffset := directoryEndOffset - directory64LocLen - if locOffset < 0 { - return -1, nil // no need to look for a header outside the file - } - buf := make([]byte, directory64LocLen) - if _, err := r.ReadAt(buf, locOffset); err != nil { - return -1, err - } - b := readBuf(buf) - if sig := b.uint32(); sig != directory64LocSignature { - return -1, nil - } - b = b[4:] // skip number of the disk with the start of the zip64 end of central directory - p := b.uint64() // relative offset of the zip64 end of central directory record - return int64(p), nil -} - -// readDirectory64End reads the zip64 directory end and updates the -// directory end with the zip64 directory end values. -func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) { - buf := make([]byte, directory64EndLen) - if _, err := r.ReadAt(buf, offset); err != nil { - return err - } - - b := readBuf(buf) - if sig := b.uint32(); sig != directory64EndSignature { - return ErrFormat - } - - b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16) - d.diskNbr = b.uint32() // number of this disk - d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory - d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk - d.directoryRecords = b.uint64() // total number of entries in the central directory - d.directorySize = b.uint64() // size of the central directory - d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number - - return nil -} - -func findSignatureInBlock(b []byte) int { - for i := len(b) - directoryEndLen; i >= 0; i-- { - // defined from directoryEndSignature in struct.go - if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 { - // n is length of comment - n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8 - if n+directoryEndLen+i <= len(b) { - return i - } - } - } - return -1 -} - -type readBuf []byte - -func (b *readBuf) uint16() uint16 { - v := binary.LittleEndian.Uint16(*b) - *b = (*b)[2:] - return v -} - -func (b *readBuf) uint32() uint32 { - v := binary.LittleEndian.Uint32(*b) - *b = (*b)[4:] - return v -} - -func (b *readBuf) uint64() uint64 { - v := binary.LittleEndian.Uint64(*b) - *b = (*b)[8:] - return v -} diff --git a/src/pkg/archive/zip/reader_test.go b/src/pkg/archive/zip/reader_test.go deleted file mode 100644 index 5652f3a50..000000000 --- a/src/pkg/archive/zip/reader_test.go +++ /dev/null @@ -1,510 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "testing" - "time" -) - -type ZipTest struct { - Name string - Source func() (r io.ReaderAt, size int64) // if non-nil, used instead of testdata/<Name> file - Comment string - File []ZipTestFile - Error error // the error that Opening this file should return -} - -type ZipTestFile struct { - Name string - Content []byte // if blank, will attempt to compare against File - ContentErr error - File string // name of file to compare to (relative to testdata/) - Mtime string // modified time in format "mm-dd-yy hh:mm:ss" - Mode os.FileMode -} - -// Caution: The Mtime values found for the test files should correspond to -// the values listed with unzip -l <zipfile>. However, the values -// listed by unzip appear to be off by some hours. When creating -// fresh test files and testing them, this issue is not present. -// The test files were created in Sydney, so there might be a time -// zone issue. The time zone information does have to be encoded -// somewhere, because otherwise unzip -l could not provide a different -// time from what the archive/zip package provides, but there appears -// to be no documentation about this. - -var tests = []ZipTest{ - { - Name: "test.zip", - Comment: "This is a zipfile comment.", - File: []ZipTestFile{ - { - Name: "test.txt", - Content: []byte("This is a test text file.\n"), - Mtime: "09-05-10 12:12:02", - Mode: 0644, - }, - { - Name: "gophercolor16x16.png", - File: "gophercolor16x16.png", - Mtime: "09-05-10 15:52:58", - Mode: 0644, - }, - }, - }, - { - Name: "test-trailing-junk.zip", - Comment: "This is a zipfile comment.", - File: []ZipTestFile{ - { - Name: "test.txt", - Content: []byte("This is a test text file.\n"), - Mtime: "09-05-10 12:12:02", - Mode: 0644, - }, - { - Name: "gophercolor16x16.png", - File: "gophercolor16x16.png", - Mtime: "09-05-10 15:52:58", - Mode: 0644, - }, - }, - }, - { - Name: "r.zip", - Source: returnRecursiveZip, - File: []ZipTestFile{ - { - Name: "r/r.zip", - Content: rZipBytes(), - Mtime: "03-04-10 00:24:16", - Mode: 0666, - }, - }, - }, - { - Name: "symlink.zip", - File: []ZipTestFile{ - { - Name: "symlink", - Content: []byte("../target"), - Mode: 0777 | os.ModeSymlink, - }, - }, - }, - { - Name: "readme.zip", - }, - { - Name: "readme.notzip", - Error: ErrFormat, - }, - { - Name: "dd.zip", - File: []ZipTestFile{ - { - Name: "filename", - Content: []byte("This is a test textfile.\n"), - Mtime: "02-02-11 13:06:20", - Mode: 0666, - }, - }, - }, - { - // created in windows XP file manager. - Name: "winxp.zip", - File: crossPlatform, - }, - { - // created by Zip 3.0 under Linux - Name: "unix.zip", - File: crossPlatform, - }, - { - // created by Go, before we wrote the "optional" data - // descriptor signatures (which are required by OS X) - Name: "go-no-datadesc-sig.zip", - File: []ZipTestFile{ - { - Name: "foo.txt", - Content: []byte("foo\n"), - Mtime: "03-08-12 16:59:10", - Mode: 0644, - }, - { - Name: "bar.txt", - Content: []byte("bar\n"), - Mtime: "03-08-12 16:59:12", - Mode: 0644, - }, - }, - }, - { - // created by Go, after we wrote the "optional" data - // descriptor signatures (which are required by OS X) - Name: "go-with-datadesc-sig.zip", - File: []ZipTestFile{ - { - Name: "foo.txt", - Content: []byte("foo\n"), - Mode: 0666, - }, - { - Name: "bar.txt", - Content: []byte("bar\n"), - Mode: 0666, - }, - }, - }, - { - Name: "Bad-CRC32-in-data-descriptor", - Source: returnCorruptCRC32Zip, - File: []ZipTestFile{ - { - Name: "foo.txt", - Content: []byte("foo\n"), - Mode: 0666, - ContentErr: ErrChecksum, - }, - { - Name: "bar.txt", - Content: []byte("bar\n"), - Mode: 0666, - }, - }, - }, - // Tests that we verify (and accept valid) crc32s on files - // with crc32s in their file header (not in data descriptors) - { - Name: "crc32-not-streamed.zip", - File: []ZipTestFile{ - { - Name: "foo.txt", - Content: []byte("foo\n"), - Mtime: "03-08-12 16:59:10", - Mode: 0644, - }, - { - Name: "bar.txt", - Content: []byte("bar\n"), - Mtime: "03-08-12 16:59:12", - Mode: 0644, - }, - }, - }, - // Tests that we verify (and reject invalid) crc32s on files - // with crc32s in their file header (not in data descriptors) - { - Name: "crc32-not-streamed.zip", - Source: returnCorruptNotStreamedZip, - File: []ZipTestFile{ - { - Name: "foo.txt", - Content: []byte("foo\n"), - Mtime: "03-08-12 16:59:10", - Mode: 0644, - ContentErr: ErrChecksum, - }, - { - Name: "bar.txt", - Content: []byte("bar\n"), - Mtime: "03-08-12 16:59:12", - Mode: 0644, - }, - }, - }, - { - Name: "zip64.zip", - File: []ZipTestFile{ - { - Name: "README", - Content: []byte("This small file is in ZIP64 format.\n"), - Mtime: "08-10-12 14:33:32", - Mode: 0644, - }, - }, - }, - // Another zip64 file with different Extras fields. (golang.org/issue/7069) - { - Name: "zip64-2.zip", - File: []ZipTestFile{ - { - Name: "README", - Content: []byte("This small file is in ZIP64 format.\n"), - Mtime: "08-10-12 14:33:32", - Mode: 0644, - }, - }, - }, -} - -var crossPlatform = []ZipTestFile{ - { - Name: "hello", - Content: []byte("world \r\n"), - Mode: 0666, - }, - { - Name: "dir/bar", - Content: []byte("foo \r\n"), - Mode: 0666, - }, - { - Name: "dir/empty/", - Content: []byte{}, - Mode: os.ModeDir | 0777, - }, - { - Name: "readonly", - Content: []byte("important \r\n"), - Mode: 0444, - }, -} - -func TestReader(t *testing.T) { - for _, zt := range tests { - readTestZip(t, zt) - } -} - -func readTestZip(t *testing.T, zt ZipTest) { - var z *Reader - var err error - if zt.Source != nil { - rat, size := zt.Source() - z, err = NewReader(rat, size) - } else { - var rc *ReadCloser - rc, err = OpenReader(filepath.Join("testdata", zt.Name)) - if err == nil { - defer rc.Close() - z = &rc.Reader - } - } - if err != zt.Error { - t.Errorf("%s: error=%v, want %v", zt.Name, err, zt.Error) - return - } - - // bail if file is not zip - if err == ErrFormat { - return - } - - // bail here if no Files expected to be tested - // (there may actually be files in the zip, but we don't care) - if zt.File == nil { - return - } - - if z.Comment != zt.Comment { - t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment) - } - if len(z.File) != len(zt.File) { - t.Fatalf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File)) - } - - // test read of each file - for i, ft := range zt.File { - readTestFile(t, zt, ft, z.File[i]) - } - - // test simultaneous reads - n := 0 - done := make(chan bool) - for i := 0; i < 5; i++ { - for j, ft := range zt.File { - go func(j int, ft ZipTestFile) { - readTestFile(t, zt, ft, z.File[j]) - done <- true - }(j, ft) - n++ - } - } - for ; n > 0; n-- { - <-done - } -} - -func readTestFile(t *testing.T, zt ZipTest, ft ZipTestFile, f *File) { - if f.Name != ft.Name { - t.Errorf("%s: name=%q, want %q", zt.Name, f.Name, ft.Name) - } - - if ft.Mtime != "" { - mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime) - if err != nil { - t.Error(err) - return - } - if ft := f.ModTime(); !ft.Equal(mtime) { - t.Errorf("%s: %s: mtime=%s, want %s", zt.Name, f.Name, ft, mtime) - } - } - - testFileMode(t, zt.Name, f, ft.Mode) - - var b bytes.Buffer - r, err := f.Open() - if err != nil { - t.Errorf("%s: %v", zt.Name, err) - return - } - - _, err = io.Copy(&b, r) - if err != ft.ContentErr { - t.Errorf("%s: copying contents: %v (want %v)", zt.Name, err, ft.ContentErr) - } - if err != nil { - return - } - r.Close() - - size := uint64(f.UncompressedSize) - if size == uint32max { - size = f.UncompressedSize64 - } - if g := uint64(b.Len()); g != size { - t.Errorf("%v: read %v bytes but f.UncompressedSize == %v", f.Name, g, size) - } - - var c []byte - if ft.Content != nil { - c = ft.Content - } else if c, err = ioutil.ReadFile("testdata/" + ft.File); err != nil { - t.Error(err) - return - } - - if b.Len() != len(c) { - t.Errorf("%s: len=%d, want %d", f.Name, b.Len(), len(c)) - return - } - - for i, b := range b.Bytes() { - if b != c[i] { - t.Errorf("%s: content[%d]=%q want %q", f.Name, i, b, c[i]) - return - } - } -} - -func testFileMode(t *testing.T, zipName string, f *File, want os.FileMode) { - mode := f.Mode() - if want == 0 { - t.Errorf("%s: %s mode: got %v, want none", zipName, f.Name, mode) - } else if mode != want { - t.Errorf("%s: %s mode: want %v, got %v", zipName, f.Name, want, mode) - } -} - -func TestInvalidFiles(t *testing.T) { - const size = 1024 * 70 // 70kb - b := make([]byte, size) - - // zeroes - _, err := NewReader(bytes.NewReader(b), size) - if err != ErrFormat { - t.Errorf("zeroes: error=%v, want %v", err, ErrFormat) - } - - // repeated directoryEndSignatures - sig := make([]byte, 4) - binary.LittleEndian.PutUint32(sig, directoryEndSignature) - for i := 0; i < size-4; i += 4 { - copy(b[i:i+4], sig) - } - _, err = NewReader(bytes.NewReader(b), size) - if err != ErrFormat { - t.Errorf("sigs: error=%v, want %v", err, ErrFormat) - } -} - -func messWith(fileName string, corrupter func(b []byte)) (r io.ReaderAt, size int64) { - data, err := ioutil.ReadFile(filepath.Join("testdata", fileName)) - if err != nil { - panic("Error reading " + fileName + ": " + err.Error()) - } - corrupter(data) - return bytes.NewReader(data), int64(len(data)) -} - -func returnCorruptCRC32Zip() (r io.ReaderAt, size int64) { - return messWith("go-with-datadesc-sig.zip", func(b []byte) { - // Corrupt one of the CRC32s in the data descriptor: - b[0x2d]++ - }) -} - -func returnCorruptNotStreamedZip() (r io.ReaderAt, size int64) { - return messWith("crc32-not-streamed.zip", func(b []byte) { - // Corrupt foo.txt's final crc32 byte, in both - // the file header and TOC. (0x7e -> 0x7f) - b[0x11]++ - b[0x9d]++ - - // TODO(bradfitz): add a new test that only corrupts - // one of these values, and verify that that's also an - // error. Currently, the reader code doesn't verify the - // fileheader and TOC's crc32 match if they're both - // non-zero and only the second line above, the TOC, - // is what matters. - }) -} - -// rZipBytes returns the bytes of a recursive zip file, without -// putting it on disk and triggering certain virus scanners. -func rZipBytes() []byte { - s := ` -0000000 50 4b 03 04 14 00 00 00 08 00 08 03 64 3c f9 f4 -0000010 89 64 48 01 00 00 b8 01 00 00 07 00 00 00 72 2f -0000020 72 2e 7a 69 70 00 25 00 da ff 50 4b 03 04 14 00 -0000030 00 00 08 00 08 03 64 3c f9 f4 89 64 48 01 00 00 -0000040 b8 01 00 00 07 00 00 00 72 2f 72 2e 7a 69 70 00 -0000050 2f 00 d0 ff 00 25 00 da ff 50 4b 03 04 14 00 00 -0000060 00 08 00 08 03 64 3c f9 f4 89 64 48 01 00 00 b8 -0000070 01 00 00 07 00 00 00 72 2f 72 2e 7a 69 70 00 2f -0000080 00 d0 ff c2 54 8e 57 39 00 05 00 fa ff c2 54 8e -0000090 57 39 00 05 00 fa ff 00 05 00 fa ff 00 14 00 eb -00000a0 ff c2 54 8e 57 39 00 05 00 fa ff 00 05 00 fa ff -00000b0 00 14 00 eb ff 42 88 21 c4 00 00 14 00 eb ff 42 -00000c0 88 21 c4 00 00 14 00 eb ff 42 88 21 c4 00 00 14 -00000d0 00 eb ff 42 88 21 c4 00 00 14 00 eb ff 42 88 21 -00000e0 c4 00 00 00 00 ff ff 00 00 00 ff ff 00 34 00 cb -00000f0 ff 42 88 21 c4 00 00 00 00 ff ff 00 00 00 ff ff -0000100 00 34 00 cb ff 42 e8 21 5e 0f 00 00 00 ff ff 0a -0000110 f0 66 64 12 61 c0 15 dc e8 a0 48 bf 48 af 2a b3 -0000120 20 c0 9b 95 0d c4 67 04 42 53 06 06 06 40 00 06 -0000130 00 f9 ff 6d 01 00 00 00 00 42 e8 21 5e 0f 00 00 -0000140 00 ff ff 0a f0 66 64 12 61 c0 15 dc e8 a0 48 bf -0000150 48 af 2a b3 20 c0 9b 95 0d c4 67 04 42 53 06 06 -0000160 06 40 00 06 00 f9 ff 6d 01 00 00 00 00 50 4b 01 -0000170 02 14 00 14 00 00 00 08 00 08 03 64 3c f9 f4 89 -0000180 64 48 01 00 00 b8 01 00 00 07 00 00 00 00 00 00 -0000190 00 00 00 00 00 00 00 00 00 00 00 72 2f 72 2e 7a -00001a0 69 70 50 4b 05 06 00 00 00 00 01 00 01 00 35 00 -00001b0 00 00 6d 01 00 00 00 00` - s = regexp.MustCompile(`[0-9a-f]{7}`).ReplaceAllString(s, "") - s = regexp.MustCompile(`\s+`).ReplaceAllString(s, "") - b, err := hex.DecodeString(s) - if err != nil { - panic(err) - } - return b -} - -func returnRecursiveZip() (r io.ReaderAt, size int64) { - b := rZipBytes() - return bytes.NewReader(b), int64(len(b)) -} diff --git a/src/pkg/archive/zip/register.go b/src/pkg/archive/zip/register.go deleted file mode 100644 index 4211ec7af..000000000 --- a/src/pkg/archive/zip/register.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip - -import ( - "compress/flate" - "errors" - "io" - "io/ioutil" - "sync" -) - -// A Compressor returns a compressing writer, writing to the -// provided writer. On Close, any pending data should be flushed. -type Compressor func(io.Writer) (io.WriteCloser, error) - -// Decompressor is a function that wraps a Reader with a decompressing Reader. -// The decompressed ReadCloser is returned to callers who open files from -// within the archive. These callers are responsible for closing this reader -// when they're finished reading. -type Decompressor func(io.Reader) io.ReadCloser - -var flateWriterPool sync.Pool - -func newFlateWriter(w io.Writer) io.WriteCloser { - fw, ok := flateWriterPool.Get().(*flate.Writer) - if ok { - fw.Reset(w) - } else { - fw, _ = flate.NewWriter(w, 5) - } - return &pooledFlateWriter{fw: fw} -} - -type pooledFlateWriter struct { - mu sync.Mutex // guards Close and Write - fw *flate.Writer -} - -func (w *pooledFlateWriter) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() - if w.fw == nil { - return 0, errors.New("Write after Close") - } - return w.fw.Write(p) -} - -func (w *pooledFlateWriter) Close() error { - w.mu.Lock() - defer w.mu.Unlock() - var err error - if w.fw != nil { - err = w.fw.Close() - flateWriterPool.Put(w.fw) - w.fw = nil - } - return err -} - -var ( - mu sync.RWMutex // guards compressor and decompressor maps - - compressors = map[uint16]Compressor{ - Store: func(w io.Writer) (io.WriteCloser, error) { return &nopCloser{w}, nil }, - Deflate: func(w io.Writer) (io.WriteCloser, error) { return newFlateWriter(w), nil }, - } - - decompressors = map[uint16]Decompressor{ - Store: ioutil.NopCloser, - Deflate: flate.NewReader, - } -) - -// RegisterDecompressor allows custom decompressors for a specified method ID. -func RegisterDecompressor(method uint16, d Decompressor) { - mu.Lock() - defer mu.Unlock() - - if _, ok := decompressors[method]; ok { - panic("decompressor already registered") - } - decompressors[method] = d -} - -// RegisterCompressor registers custom compressors for a specified method ID. -// The common methods Store and Deflate are built in. -func RegisterCompressor(method uint16, comp Compressor) { - mu.Lock() - defer mu.Unlock() - - if _, ok := compressors[method]; ok { - panic("compressor already registered") - } - compressors[method] = comp -} - -func compressor(method uint16) Compressor { - mu.RLock() - defer mu.RUnlock() - return compressors[method] -} - -func decompressor(method uint16) Decompressor { - mu.RLock() - defer mu.RUnlock() - return decompressors[method] -} diff --git a/src/pkg/archive/zip/struct.go b/src/pkg/archive/zip/struct.go deleted file mode 100644 index cb28e8324..000000000 --- a/src/pkg/archive/zip/struct.go +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package zip provides support for reading and writing ZIP archives. - -See: http://www.pkware.com/documents/casestudies/APPNOTE.TXT - -This package does not support disk spanning. - -A note about ZIP64: - -To be backwards compatible the FileHeader has both 32 and 64 bit Size -fields. The 64 bit fields will always contain the correct value and -for normal archives both fields will be the same. For files requiring -the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit -fields must be used instead. -*/ -package zip - -import ( - "os" - "path" - "time" -) - -// Compression methods. -const ( - Store uint16 = 0 - Deflate uint16 = 8 -) - -const ( - fileHeaderSignature = 0x04034b50 - directoryHeaderSignature = 0x02014b50 - directoryEndSignature = 0x06054b50 - directory64LocSignature = 0x07064b50 - directory64EndSignature = 0x06064b50 - dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder - fileHeaderLen = 30 // + filename + extra - directoryHeaderLen = 46 // + filename + extra + comment - directoryEndLen = 22 // + comment - dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size - dataDescriptor64Len = 24 // descriptor with 8 byte sizes - directory64LocLen = 20 // - directory64EndLen = 56 // + extra - - // Constants for the first byte in CreatorVersion - creatorFAT = 0 - creatorUnix = 3 - creatorNTFS = 11 - creatorVFAT = 14 - creatorMacOSX = 19 - - // version numbers - zipVersion20 = 20 // 2.0 - zipVersion45 = 45 // 4.5 (reads and writes zip64 archives) - - // limits for non zip64 files - uint16max = (1 << 16) - 1 - uint32max = (1 << 32) - 1 - - // extra header id's - zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field -) - -// FileHeader describes a file within a zip file. -// See the zip spec for details. -type FileHeader struct { - // Name is the name of the file. - // It must be a relative path: it must not start with a drive - // letter (e.g. C:) or leading slash, and only forward slashes - // are allowed. - Name string - - CreatorVersion uint16 - ReaderVersion uint16 - Flags uint16 - Method uint16 - ModifiedTime uint16 // MS-DOS time - ModifiedDate uint16 // MS-DOS date - CRC32 uint32 - CompressedSize uint32 // deprecated; use CompressedSize64 - UncompressedSize uint32 // deprecated; use UncompressedSize64 - CompressedSize64 uint64 - UncompressedSize64 uint64 - Extra []byte - ExternalAttrs uint32 // Meaning depends on CreatorVersion - Comment string -} - -// FileInfo returns an os.FileInfo for the FileHeader. -func (h *FileHeader) FileInfo() os.FileInfo { - return headerFileInfo{h} -} - -// headerFileInfo implements os.FileInfo. -type headerFileInfo struct { - fh *FileHeader -} - -func (fi headerFileInfo) Name() string { return path.Base(fi.fh.Name) } -func (fi headerFileInfo) Size() int64 { - if fi.fh.UncompressedSize64 > 0 { - return int64(fi.fh.UncompressedSize64) - } - return int64(fi.fh.UncompressedSize) -} -func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } -func (fi headerFileInfo) ModTime() time.Time { return fi.fh.ModTime() } -func (fi headerFileInfo) Mode() os.FileMode { return fi.fh.Mode() } -func (fi headerFileInfo) Sys() interface{} { return fi.fh } - -// FileInfoHeader creates a partially-populated FileHeader from an -// os.FileInfo. -// Because os.FileInfo's Name method returns only the base name of -// the file it describes, it may be necessary to modify the Name field -// of the returned header to provide the full path name of the file. -func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) { - size := fi.Size() - fh := &FileHeader{ - Name: fi.Name(), - UncompressedSize64: uint64(size), - } - fh.SetModTime(fi.ModTime()) - fh.SetMode(fi.Mode()) - if fh.UncompressedSize64 > uint32max { - fh.UncompressedSize = uint32max - } else { - fh.UncompressedSize = uint32(fh.UncompressedSize64) - } - return fh, nil -} - -type directoryEnd struct { - diskNbr uint32 // unused - dirDiskNbr uint32 // unused - dirRecordsThisDisk uint64 // unused - directoryRecords uint64 - directorySize uint64 - directoryOffset uint64 // relative to file - commentLen uint16 - comment string -} - -// msDosTimeToTime converts an MS-DOS date and time into a time.Time. -// The resolution is 2s. -// See: http://msdn.microsoft.com/en-us/library/ms724247(v=VS.85).aspx -func msDosTimeToTime(dosDate, dosTime uint16) time.Time { - return time.Date( - // date bits 0-4: day of month; 5-8: month; 9-15: years since 1980 - int(dosDate>>9+1980), - time.Month(dosDate>>5&0xf), - int(dosDate&0x1f), - - // time bits 0-4: second/2; 5-10: minute; 11-15: hour - int(dosTime>>11), - int(dosTime>>5&0x3f), - int(dosTime&0x1f*2), - 0, // nanoseconds - - time.UTC, - ) -} - -// timeToMsDosTime converts a time.Time to an MS-DOS date and time. -// The resolution is 2s. -// See: http://msdn.microsoft.com/en-us/library/ms724274(v=VS.85).aspx -func timeToMsDosTime(t time.Time) (fDate uint16, fTime uint16) { - t = t.In(time.UTC) - fDate = uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9) - fTime = uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11) - return -} - -// ModTime returns the modification time in UTC. -// The resolution is 2s. -func (h *FileHeader) ModTime() time.Time { - return msDosTimeToTime(h.ModifiedDate, h.ModifiedTime) -} - -// SetModTime sets the ModifiedTime and ModifiedDate fields to the given time in UTC. -// The resolution is 2s. -func (h *FileHeader) SetModTime(t time.Time) { - h.ModifiedDate, h.ModifiedTime = timeToMsDosTime(t) -} - -const ( - // Unix constants. The specification doesn't mention them, - // but these seem to be the values agreed on by tools. - s_IFMT = 0xf000 - s_IFSOCK = 0xc000 - s_IFLNK = 0xa000 - s_IFREG = 0x8000 - s_IFBLK = 0x6000 - s_IFDIR = 0x4000 - s_IFCHR = 0x2000 - s_IFIFO = 0x1000 - s_ISUID = 0x800 - s_ISGID = 0x400 - s_ISVTX = 0x200 - - msdosDir = 0x10 - msdosReadOnly = 0x01 -) - -// Mode returns the permission and mode bits for the FileHeader. -func (h *FileHeader) Mode() (mode os.FileMode) { - switch h.CreatorVersion >> 8 { - case creatorUnix, creatorMacOSX: - mode = unixModeToFileMode(h.ExternalAttrs >> 16) - case creatorNTFS, creatorVFAT, creatorFAT: - mode = msdosModeToFileMode(h.ExternalAttrs) - } - if len(h.Name) > 0 && h.Name[len(h.Name)-1] == '/' { - mode |= os.ModeDir - } - return mode -} - -// SetMode changes the permission and mode bits for the FileHeader. -func (h *FileHeader) SetMode(mode os.FileMode) { - h.CreatorVersion = h.CreatorVersion&0xff | creatorUnix<<8 - h.ExternalAttrs = fileModeToUnixMode(mode) << 16 - - // set MSDOS attributes too, as the original zip does. - if mode&os.ModeDir != 0 { - h.ExternalAttrs |= msdosDir - } - if mode&0200 == 0 { - h.ExternalAttrs |= msdosReadOnly - } -} - -// isZip64 returns true if the file size exceeds the 32 bit limit -func (fh *FileHeader) isZip64() bool { - return fh.CompressedSize64 > uint32max || fh.UncompressedSize64 > uint32max -} - -func msdosModeToFileMode(m uint32) (mode os.FileMode) { - if m&msdosDir != 0 { - mode = os.ModeDir | 0777 - } else { - mode = 0666 - } - if m&msdosReadOnly != 0 { - mode &^= 0222 - } - return mode -} - -func fileModeToUnixMode(mode os.FileMode) uint32 { - var m uint32 - switch mode & os.ModeType { - default: - m = s_IFREG - case os.ModeDir: - m = s_IFDIR - case os.ModeSymlink: - m = s_IFLNK - case os.ModeNamedPipe: - m = s_IFIFO - case os.ModeSocket: - m = s_IFSOCK - case os.ModeDevice: - if mode&os.ModeCharDevice != 0 { - m = s_IFCHR - } else { - m = s_IFBLK - } - } - if mode&os.ModeSetuid != 0 { - m |= s_ISUID - } - if mode&os.ModeSetgid != 0 { - m |= s_ISGID - } - if mode&os.ModeSticky != 0 { - m |= s_ISVTX - } - return m | uint32(mode&0777) -} - -func unixModeToFileMode(m uint32) os.FileMode { - mode := os.FileMode(m & 0777) - switch m & s_IFMT { - case s_IFBLK: - mode |= os.ModeDevice - case s_IFCHR: - mode |= os.ModeDevice | os.ModeCharDevice - case s_IFDIR: - mode |= os.ModeDir - case s_IFIFO: - mode |= os.ModeNamedPipe - case s_IFLNK: - mode |= os.ModeSymlink - case s_IFREG: - // nothing to do - case s_IFSOCK: - mode |= os.ModeSocket - } - if m&s_ISGID != 0 { - mode |= os.ModeSetgid - } - if m&s_ISUID != 0 { - mode |= os.ModeSetuid - } - if m&s_ISVTX != 0 { - mode |= os.ModeSticky - } - return mode -} diff --git a/src/pkg/archive/zip/testdata/crc32-not-streamed.zip b/src/pkg/archive/zip/testdata/crc32-not-streamed.zip Binary files differdeleted file mode 100644 index f268d8873..000000000 --- a/src/pkg/archive/zip/testdata/crc32-not-streamed.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/dd.zip b/src/pkg/archive/zip/testdata/dd.zip Binary files differdeleted file mode 100644 index e53378b0b..000000000 --- a/src/pkg/archive/zip/testdata/dd.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/go-no-datadesc-sig.zip b/src/pkg/archive/zip/testdata/go-no-datadesc-sig.zip Binary files differdeleted file mode 100644 index c3d593f44..000000000 --- a/src/pkg/archive/zip/testdata/go-no-datadesc-sig.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/go-with-datadesc-sig.zip b/src/pkg/archive/zip/testdata/go-with-datadesc-sig.zip Binary files differdeleted file mode 100644 index bcfe121bb..000000000 --- a/src/pkg/archive/zip/testdata/go-with-datadesc-sig.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/gophercolor16x16.png b/src/pkg/archive/zip/testdata/gophercolor16x16.png Binary files differdeleted file mode 100644 index 48854ff3b..000000000 --- a/src/pkg/archive/zip/testdata/gophercolor16x16.png +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/readme.notzip b/src/pkg/archive/zip/testdata/readme.notzip Binary files differdeleted file mode 100644 index 06668c4c1..000000000 --- a/src/pkg/archive/zip/testdata/readme.notzip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/readme.zip b/src/pkg/archive/zip/testdata/readme.zip Binary files differdeleted file mode 100644 index db3bb900e..000000000 --- a/src/pkg/archive/zip/testdata/readme.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/symlink.zip b/src/pkg/archive/zip/testdata/symlink.zip Binary files differdeleted file mode 100644 index af846938c..000000000 --- a/src/pkg/archive/zip/testdata/symlink.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/test-trailing-junk.zip b/src/pkg/archive/zip/testdata/test-trailing-junk.zip Binary files differdeleted file mode 100644 index 42281b4e3..000000000 --- a/src/pkg/archive/zip/testdata/test-trailing-junk.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/test.zip b/src/pkg/archive/zip/testdata/test.zip Binary files differdeleted file mode 100644 index 03890c05d..000000000 --- a/src/pkg/archive/zip/testdata/test.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/unix.zip b/src/pkg/archive/zip/testdata/unix.zip Binary files differdeleted file mode 100644 index ce1a981b2..000000000 --- a/src/pkg/archive/zip/testdata/unix.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/winxp.zip b/src/pkg/archive/zip/testdata/winxp.zip Binary files differdeleted file mode 100644 index 3919322f0..000000000 --- a/src/pkg/archive/zip/testdata/winxp.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/zip64-2.zip b/src/pkg/archive/zip/testdata/zip64-2.zip Binary files differdeleted file mode 100644 index f844e3537..000000000 --- a/src/pkg/archive/zip/testdata/zip64-2.zip +++ /dev/null diff --git a/src/pkg/archive/zip/testdata/zip64.zip b/src/pkg/archive/zip/testdata/zip64.zip Binary files differdeleted file mode 100644 index a2ee1fa33..000000000 --- a/src/pkg/archive/zip/testdata/zip64.zip +++ /dev/null diff --git a/src/pkg/archive/zip/writer.go b/src/pkg/archive/zip/writer.go deleted file mode 100644 index 6c9800a78..000000000 --- a/src/pkg/archive/zip/writer.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip - -import ( - "bufio" - "encoding/binary" - "errors" - "hash" - "hash/crc32" - "io" -) - -// TODO(adg): support zip file comments -// TODO(adg): support specifying deflate level - -// Writer implements a zip file writer. -type Writer struct { - cw *countWriter - dir []*header - last *fileWriter - closed bool -} - -type header struct { - *FileHeader - offset uint64 -} - -// NewWriter returns a new Writer writing a zip file to w. -func NewWriter(w io.Writer) *Writer { - return &Writer{cw: &countWriter{w: bufio.NewWriter(w)}} -} - -// Close finishes writing the zip file by writing the central directory. -// It does not (and can not) close the underlying writer. -func (w *Writer) Close() error { - if w.last != nil && !w.last.closed { - if err := w.last.close(); err != nil { - return err - } - w.last = nil - } - if w.closed { - return errors.New("zip: writer closed twice") - } - w.closed = true - - // write central directory - start := w.cw.count - for _, h := range w.dir { - var buf [directoryHeaderLen]byte - b := writeBuf(buf[:]) - b.uint32(uint32(directoryHeaderSignature)) - b.uint16(h.CreatorVersion) - b.uint16(h.ReaderVersion) - b.uint16(h.Flags) - b.uint16(h.Method) - b.uint16(h.ModifiedTime) - b.uint16(h.ModifiedDate) - b.uint32(h.CRC32) - if h.isZip64() || h.offset > uint32max { - // the file needs a zip64 header. store maxint in both - // 32 bit size fields (and offset later) to signal that the - // zip64 extra header should be used. - b.uint32(uint32max) // compressed size - b.uint32(uint32max) // uncompressed size - - // append a zip64 extra block to Extra - var buf [28]byte // 2x uint16 + 3x uint64 - eb := writeBuf(buf[:]) - eb.uint16(zip64ExtraId) - eb.uint16(24) // size = 3x uint64 - eb.uint64(h.UncompressedSize64) - eb.uint64(h.CompressedSize64) - eb.uint64(h.offset) - h.Extra = append(h.Extra, buf[:]...) - } else { - b.uint32(h.CompressedSize) - b.uint32(h.UncompressedSize) - } - b.uint16(uint16(len(h.Name))) - b.uint16(uint16(len(h.Extra))) - b.uint16(uint16(len(h.Comment))) - b = b[4:] // skip disk number start and internal file attr (2x uint16) - b.uint32(h.ExternalAttrs) - if h.offset > uint32max { - b.uint32(uint32max) - } else { - b.uint32(uint32(h.offset)) - } - if _, err := w.cw.Write(buf[:]); err != nil { - return err - } - if _, err := io.WriteString(w.cw, h.Name); err != nil { - return err - } - if _, err := w.cw.Write(h.Extra); err != nil { - return err - } - if _, err := io.WriteString(w.cw, h.Comment); err != nil { - return err - } - } - end := w.cw.count - - records := uint64(len(w.dir)) - size := uint64(end - start) - offset := uint64(start) - - if records > uint16max || size > uint32max || offset > uint32max { - var buf [directory64EndLen + directory64LocLen]byte - b := writeBuf(buf[:]) - - // zip64 end of central directory record - b.uint32(directory64EndSignature) - b.uint64(directory64EndLen) - b.uint16(zipVersion45) // version made by - b.uint16(zipVersion45) // version needed to extract - b.uint32(0) // number of this disk - b.uint32(0) // number of the disk with the start of the central directory - b.uint64(records) // total number of entries in the central directory on this disk - b.uint64(records) // total number of entries in the central directory - b.uint64(size) // size of the central directory - b.uint64(offset) // offset of start of central directory with respect to the starting disk number - - // zip64 end of central directory locator - b.uint32(directory64LocSignature) - b.uint32(0) // number of the disk with the start of the zip64 end of central directory - b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record - b.uint32(1) // total number of disks - - if _, err := w.cw.Write(buf[:]); err != nil { - return err - } - - // store max values in the regular end record to signal that - // that the zip64 values should be used instead - records = uint16max - size = uint32max - offset = uint32max - } - - // write end record - var buf [directoryEndLen]byte - b := writeBuf(buf[:]) - b.uint32(uint32(directoryEndSignature)) - b = b[4:] // skip over disk number and first disk number (2x uint16) - b.uint16(uint16(records)) // number of entries this disk - b.uint16(uint16(records)) // number of entries total - b.uint32(uint32(size)) // size of directory - b.uint32(uint32(offset)) // start of directory - // skipped size of comment (always zero) - if _, err := w.cw.Write(buf[:]); err != nil { - return err - } - - return w.cw.w.(*bufio.Writer).Flush() -} - -// Create adds a file to the zip file using the provided name. -// It returns a Writer to which the file contents should be written. -// The name must be a relative path: it must not start with a drive -// letter (e.g. C:) or leading slash, and only forward slashes are -// allowed. -// The file's contents must be written to the io.Writer before the next -// call to Create, CreateHeader, or Close. -func (w *Writer) Create(name string) (io.Writer, error) { - header := &FileHeader{ - Name: name, - Method: Deflate, - } - return w.CreateHeader(header) -} - -// CreateHeader adds a file to the zip file using the provided FileHeader -// for the file metadata. -// It returns a Writer to which the file contents should be written. -// The file's contents must be written to the io.Writer before the next -// call to Create, CreateHeader, or Close. -func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) { - if w.last != nil && !w.last.closed { - if err := w.last.close(); err != nil { - return nil, err - } - } - - fh.Flags |= 0x8 // we will write a data descriptor - - fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte - fh.ReaderVersion = zipVersion20 - - fw := &fileWriter{ - zipw: w.cw, - compCount: &countWriter{w: w.cw}, - crc32: crc32.NewIEEE(), - } - comp := compressor(fh.Method) - if comp == nil { - return nil, ErrAlgorithm - } - var err error - fw.comp, err = comp(fw.compCount) - if err != nil { - return nil, err - } - fw.rawCount = &countWriter{w: fw.comp} - - h := &header{ - FileHeader: fh, - offset: uint64(w.cw.count), - } - w.dir = append(w.dir, h) - fw.header = h - - if err := writeHeader(w.cw, fh); err != nil { - return nil, err - } - - w.last = fw - return fw, nil -} - -func writeHeader(w io.Writer, h *FileHeader) error { - var buf [fileHeaderLen]byte - b := writeBuf(buf[:]) - b.uint32(uint32(fileHeaderSignature)) - b.uint16(h.ReaderVersion) - b.uint16(h.Flags) - b.uint16(h.Method) - b.uint16(h.ModifiedTime) - b.uint16(h.ModifiedDate) - b.uint32(0) // since we are writing a data descriptor crc32, - b.uint32(0) // compressed size, - b.uint32(0) // and uncompressed size should be zero - b.uint16(uint16(len(h.Name))) - b.uint16(uint16(len(h.Extra))) - if _, err := w.Write(buf[:]); err != nil { - return err - } - if _, err := io.WriteString(w, h.Name); err != nil { - return err - } - _, err := w.Write(h.Extra) - return err -} - -type fileWriter struct { - *header - zipw io.Writer - rawCount *countWriter - comp io.WriteCloser - compCount *countWriter - crc32 hash.Hash32 - closed bool -} - -func (w *fileWriter) Write(p []byte) (int, error) { - if w.closed { - return 0, errors.New("zip: write to closed file") - } - w.crc32.Write(p) - return w.rawCount.Write(p) -} - -func (w *fileWriter) close() error { - if w.closed { - return errors.New("zip: file closed twice") - } - w.closed = true - if err := w.comp.Close(); err != nil { - return err - } - - // update FileHeader - fh := w.header.FileHeader - fh.CRC32 = w.crc32.Sum32() - fh.CompressedSize64 = uint64(w.compCount.count) - fh.UncompressedSize64 = uint64(w.rawCount.count) - - if fh.isZip64() { - fh.CompressedSize = uint32max - fh.UncompressedSize = uint32max - fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions - } else { - fh.CompressedSize = uint32(fh.CompressedSize64) - fh.UncompressedSize = uint32(fh.UncompressedSize64) - } - - // Write data descriptor. This is more complicated than one would - // think, see e.g. comments in zipfile.c:putextended() and - // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588. - // The approach here is to write 8 byte sizes if needed without - // adding a zip64 extra in the local header (too late anyway). - var buf []byte - if fh.isZip64() { - buf = make([]byte, dataDescriptor64Len) - } else { - buf = make([]byte, dataDescriptorLen) - } - b := writeBuf(buf) - b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X - b.uint32(fh.CRC32) - if fh.isZip64() { - b.uint64(fh.CompressedSize64) - b.uint64(fh.UncompressedSize64) - } else { - b.uint32(fh.CompressedSize) - b.uint32(fh.UncompressedSize) - } - _, err := w.zipw.Write(buf) - return err -} - -type countWriter struct { - w io.Writer - count int64 -} - -func (w *countWriter) Write(p []byte) (int, error) { - n, err := w.w.Write(p) - w.count += int64(n) - return n, err -} - -type nopCloser struct { - io.Writer -} - -func (w nopCloser) Close() error { - return nil -} - -type writeBuf []byte - -func (b *writeBuf) uint16(v uint16) { - binary.LittleEndian.PutUint16(*b, v) - *b = (*b)[2:] -} - -func (b *writeBuf) uint32(v uint32) { - binary.LittleEndian.PutUint32(*b, v) - *b = (*b)[4:] -} - -func (b *writeBuf) uint64(v uint64) { - binary.LittleEndian.PutUint64(*b, v) - *b = (*b)[8:] -} diff --git a/src/pkg/archive/zip/writer_test.go b/src/pkg/archive/zip/writer_test.go deleted file mode 100644 index 4bfa87080..000000000 --- a/src/pkg/archive/zip/writer_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zip - -import ( - "bytes" - "io/ioutil" - "math/rand" - "os" - "testing" -) - -// TODO(adg): a more sophisticated test suite - -type WriteTest struct { - Name string - Data []byte - Method uint16 - Mode os.FileMode -} - -var writeTests = []WriteTest{ - { - Name: "foo", - Data: []byte("Rabbits, guinea pigs, gophers, marsupial rats, and quolls."), - Method: Store, - Mode: 0666, - }, - { - Name: "bar", - Data: nil, // large data set in the test - Method: Deflate, - Mode: 0644, - }, - { - Name: "setuid", - Data: []byte("setuid file"), - Method: Deflate, - Mode: 0755 | os.ModeSetuid, - }, - { - Name: "setgid", - Data: []byte("setgid file"), - Method: Deflate, - Mode: 0755 | os.ModeSetgid, - }, - { - Name: "symlink", - Data: []byte("../link/target"), - Method: Deflate, - Mode: 0755 | os.ModeSymlink, - }, -} - -func TestWriter(t *testing.T) { - largeData := make([]byte, 1<<17) - for i := range largeData { - largeData[i] = byte(rand.Int()) - } - writeTests[1].Data = largeData - defer func() { - writeTests[1].Data = nil - }() - - // write a zip file - buf := new(bytes.Buffer) - w := NewWriter(buf) - - for _, wt := range writeTests { - testCreate(t, w, &wt) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - // read it back - r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) - if err != nil { - t.Fatal(err) - } - for i, wt := range writeTests { - testReadFile(t, r.File[i], &wt) - } -} - -func testCreate(t *testing.T, w *Writer, wt *WriteTest) { - header := &FileHeader{ - Name: wt.Name, - Method: wt.Method, - } - if wt.Mode != 0 { - header.SetMode(wt.Mode) - } - f, err := w.CreateHeader(header) - if err != nil { - t.Fatal(err) - } - _, err = f.Write(wt.Data) - if err != nil { - t.Fatal(err) - } -} - -func testReadFile(t *testing.T, f *File, wt *WriteTest) { - if f.Name != wt.Name { - t.Fatalf("File name: got %q, want %q", f.Name, wt.Name) - } - testFileMode(t, wt.Name, f, wt.Mode) - rc, err := f.Open() - if err != nil { - t.Fatal("opening:", err) - } - b, err := ioutil.ReadAll(rc) - if err != nil { - t.Fatal("reading:", err) - } - err = rc.Close() - if err != nil { - t.Fatal("closing:", err) - } - if !bytes.Equal(b, wt.Data) { - t.Errorf("File contents %q, want %q", b, wt.Data) - } -} - -func BenchmarkCompressedZipGarbage(b *testing.B) { - b.ReportAllocs() - var buf bytes.Buffer - bigBuf := bytes.Repeat([]byte("a"), 1<<20) - for i := 0; i < b.N; i++ { - buf.Reset() - zw := NewWriter(&buf) - for j := 0; j < 3; j++ { - w, _ := zw.CreateHeader(&FileHeader{ - Name: "foo", - Method: Deflate, - }) - w.Write(bigBuf) - } - zw.Close() - } -} diff --git a/src/pkg/archive/zip/zip_test.go b/src/pkg/archive/zip/zip_test.go deleted file mode 100644 index 32a16a79e..000000000 --- a/src/pkg/archive/zip/zip_test.go +++ /dev/null @@ -1,395 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Tests that involve both reading and writing. - -package zip - -import ( - "bytes" - "fmt" - "hash" - "io" - "io/ioutil" - "sort" - "strings" - "testing" - "time" -) - -func TestOver65kFiles(t *testing.T) { - buf := new(bytes.Buffer) - w := NewWriter(buf) - const nFiles = (1 << 16) + 42 - for i := 0; i < nFiles; i++ { - _, err := w.CreateHeader(&FileHeader{ - Name: fmt.Sprintf("%d.dat", i), - Method: Store, // avoid Issue 6136 and Issue 6138 - }) - if err != nil { - t.Fatalf("creating file %d: %v", i, err) - } - } - if err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - s := buf.String() - zr, err := NewReader(strings.NewReader(s), int64(len(s))) - if err != nil { - t.Fatalf("NewReader: %v", err) - } - if got := len(zr.File); got != nFiles { - t.Fatalf("File contains %d files, want %d", got, nFiles) - } - for i := 0; i < nFiles; i++ { - want := fmt.Sprintf("%d.dat", i) - if zr.File[i].Name != want { - t.Fatalf("File(%d) = %q, want %q", i, zr.File[i].Name, want) - } - } -} - -func TestModTime(t *testing.T) { - var testTime = time.Date(2009, time.November, 10, 23, 45, 58, 0, time.UTC) - fh := new(FileHeader) - fh.SetModTime(testTime) - outTime := fh.ModTime() - if !outTime.Equal(testTime) { - t.Errorf("times don't match: got %s, want %s", outTime, testTime) - } -} - -func testHeaderRoundTrip(fh *FileHeader, wantUncompressedSize uint32, wantUncompressedSize64 uint64, t *testing.T) { - fi := fh.FileInfo() - fh2, err := FileInfoHeader(fi) - if err != nil { - t.Fatal(err) - } - if got, want := fh2.Name, fh.Name; got != want { - t.Errorf("Name: got %s, want %s\n", got, want) - } - if got, want := fh2.UncompressedSize, wantUncompressedSize; got != want { - t.Errorf("UncompressedSize: got %d, want %d\n", got, want) - } - if got, want := fh2.UncompressedSize64, wantUncompressedSize64; got != want { - t.Errorf("UncompressedSize64: got %d, want %d\n", got, want) - } - if got, want := fh2.ModifiedTime, fh.ModifiedTime; got != want { - t.Errorf("ModifiedTime: got %d, want %d\n", got, want) - } - if got, want := fh2.ModifiedDate, fh.ModifiedDate; got != want { - t.Errorf("ModifiedDate: got %d, want %d\n", got, want) - } - - if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh { - t.Errorf("Sys didn't return original *FileHeader") - } -} - -func TestFileHeaderRoundTrip(t *testing.T) { - fh := &FileHeader{ - Name: "foo.txt", - UncompressedSize: 987654321, - ModifiedTime: 1234, - ModifiedDate: 5678, - } - testHeaderRoundTrip(fh, fh.UncompressedSize, uint64(fh.UncompressedSize), t) -} - -func TestFileHeaderRoundTrip64(t *testing.T) { - fh := &FileHeader{ - Name: "foo.txt", - UncompressedSize64: 9876543210, - ModifiedTime: 1234, - ModifiedDate: 5678, - } - testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t) -} - -type repeatedByte struct { - off int64 - b byte - n int64 -} - -// rleBuffer is a run-length-encoded byte buffer. -// It's an io.Writer (like a bytes.Buffer) and also an io.ReaderAt, -// allowing random-access reads. -type rleBuffer struct { - buf []repeatedByte -} - -func (r *rleBuffer) Size() int64 { - if len(r.buf) == 0 { - return 0 - } - last := &r.buf[len(r.buf)-1] - return last.off + last.n -} - -func (r *rleBuffer) Write(p []byte) (n int, err error) { - var rp *repeatedByte - if len(r.buf) > 0 { - rp = &r.buf[len(r.buf)-1] - // Fast path, if p is entirely the same byte repeated. - if lastByte := rp.b; len(p) > 0 && p[0] == lastByte { - all := true - for _, b := range p { - if b != lastByte { - all = false - break - } - } - if all { - rp.n += int64(len(p)) - return len(p), nil - } - } - } - - for _, b := range p { - if rp == nil || rp.b != b { - r.buf = append(r.buf, repeatedByte{r.Size(), b, 1}) - rp = &r.buf[len(r.buf)-1] - } else { - rp.n++ - } - } - return len(p), nil -} - -func (r *rleBuffer) ReadAt(p []byte, off int64) (n int, err error) { - if len(p) == 0 { - return - } - skipParts := sort.Search(len(r.buf), func(i int) bool { - part := &r.buf[i] - return part.off+part.n > off - }) - parts := r.buf[skipParts:] - if len(parts) > 0 { - skipBytes := off - parts[0].off - for len(parts) > 0 { - part := parts[0] - for i := skipBytes; i < part.n; i++ { - if n == len(p) { - return - } - p[n] = part.b - n++ - } - parts = parts[1:] - skipBytes = 0 - } - } - if n != len(p) { - err = io.ErrUnexpectedEOF - } - return -} - -// Just testing the rleBuffer used in the Zip64 test above. Not used by the zip code. -func TestRLEBuffer(t *testing.T) { - b := new(rleBuffer) - var all []byte - writes := []string{"abcdeee", "eeeeeee", "eeeefghaaiii"} - for _, w := range writes { - b.Write([]byte(w)) - all = append(all, w...) - } - if len(b.buf) != 10 { - t.Fatalf("len(b.buf) = %d; want 10", len(b.buf)) - } - - for i := 0; i < len(all); i++ { - for j := 0; j < len(all)-i; j++ { - buf := make([]byte, j) - n, err := b.ReadAt(buf, int64(i)) - if err != nil || n != len(buf) { - t.Errorf("ReadAt(%d, %d) = %d, %v; want %d, nil", i, j, n, err, len(buf)) - } - if !bytes.Equal(buf, all[i:i+j]) { - t.Errorf("ReadAt(%d, %d) = %q; want %q", i, j, buf, all[i:i+j]) - } - } - } -} - -// fakeHash32 is a dummy Hash32 that always returns 0. -type fakeHash32 struct { - hash.Hash32 -} - -func (fakeHash32) Write(p []byte) (int, error) { return len(p), nil } -func (fakeHash32) Sum32() uint32 { return 0 } - -func TestZip64(t *testing.T) { - if testing.Short() { - t.Skip("slow test; skipping") - } - const size = 1 << 32 // before the "END\n" part - testZip64(t, size) -} - -func testZip64(t testing.TB, size int64) { - const chunkSize = 1024 - chunks := int(size / chunkSize) - // write 2^32 bytes plus "END\n" to a zip file - buf := new(rleBuffer) - w := NewWriter(buf) - f, err := w.CreateHeader(&FileHeader{ - Name: "huge.txt", - Method: Store, - }) - if err != nil { - t.Fatal(err) - } - f.(*fileWriter).crc32 = fakeHash32{} - chunk := make([]byte, chunkSize) - for i := range chunk { - chunk[i] = '.' - } - for i := 0; i < chunks; i++ { - _, err := f.Write(chunk) - if err != nil { - t.Fatal("write chunk:", err) - } - } - end := []byte("END\n") - _, err = f.Write(end) - if err != nil { - t.Fatal("write end:", err) - } - if err := w.Close(); err != nil { - t.Fatal(err) - } - - // read back zip file and check that we get to the end of it - r, err := NewReader(buf, int64(buf.Size())) - if err != nil { - t.Fatal("reader:", err) - } - f0 := r.File[0] - rc, err := f0.Open() - if err != nil { - t.Fatal("opening:", err) - } - rc.(*checksumReader).hash = fakeHash32{} - for i := 0; i < chunks; i++ { - _, err := io.ReadFull(rc, chunk) - if err != nil { - t.Fatal("read:", err) - } - } - gotEnd, err := ioutil.ReadAll(rc) - if err != nil { - t.Fatal("read end:", err) - } - if !bytes.Equal(gotEnd, end) { - t.Errorf("End of zip64 archive %q, want %q", gotEnd, end) - } - err = rc.Close() - if err != nil { - t.Fatal("closing:", err) - } - if size == 1<<32 { - if got, want := f0.UncompressedSize, uint32(uint32max); got != want { - t.Errorf("UncompressedSize %d, want %d", got, want) - } - } - - if got, want := f0.UncompressedSize64, uint64(size)+uint64(len(end)); got != want { - t.Errorf("UncompressedSize64 %d, want %d", got, want) - } -} - -func testInvalidHeader(h *FileHeader, t *testing.T) { - var buf bytes.Buffer - z := NewWriter(&buf) - - f, err := z.CreateHeader(h) - if err != nil { - t.Fatalf("error creating header: %v", err) - } - if _, err := f.Write([]byte("hi")); err != nil { - t.Fatalf("error writing content: %v", err) - } - if err := z.Close(); err != nil { - t.Fatalf("error closing zip writer: %v", err) - } - - b := buf.Bytes() - if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != ErrFormat { - t.Fatalf("got %v, expected ErrFormat", err) - } -} - -func testValidHeader(h *FileHeader, t *testing.T) { - var buf bytes.Buffer - z := NewWriter(&buf) - - f, err := z.CreateHeader(h) - if err != nil { - t.Fatalf("error creating header: %v", err) - } - if _, err := f.Write([]byte("hi")); err != nil { - t.Fatalf("error writing content: %v", err) - } - if err := z.Close(); err != nil { - t.Fatalf("error closing zip writer: %v", err) - } - - b := buf.Bytes() - if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil { - t.Fatalf("got %v, expected nil", err) - } -} - -// Issue 4302. -func TestHeaderInvalidTagAndSize(t *testing.T) { - const timeFormat = "20060102T150405.000.txt" - - ts := time.Now() - filename := ts.Format(timeFormat) - - h := FileHeader{ - Name: filename, - Method: Deflate, - Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len - } - h.SetModTime(ts) - - testInvalidHeader(&h, t) -} - -func TestHeaderTooShort(t *testing.T) { - h := FileHeader{ - Name: "foo.txt", - Method: Deflate, - Extra: []byte{zip64ExtraId}, // missing size - } - testInvalidHeader(&h, t) -} - -// Issue 4393. It is valid to have an extra data header -// which contains no body. -func TestZeroLengthHeader(t *testing.T) { - h := FileHeader{ - Name: "extadata.txt", - Method: Deflate, - Extra: []byte{ - 85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5 - 85, 120, 0, 0, // tag 30805 size 0 - }, - } - testValidHeader(&h, t) -} - -// Just benchmarking how fast the Zip64 test above is. Not related to -// our zip performance, since the test above disabled CRC32 and flate. -func BenchmarkZip64Test(b *testing.B) { - for i := 0; i < b.N; i++ { - testZip64(b, 1<<26) - } -} |
