summaryrefslogtreecommitdiff
path: root/src/pkg/archive
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/archive')
-rw-r--r--src/pkg/archive/tar/common.go184
-rw-r--r--src/pkg/archive/tar/example_test.go79
-rw-r--r--src/pkg/archive/tar/reader.go205
-rw-r--r--src/pkg/archive/tar/reader_test.go106
-rw-r--r--src/pkg/archive/tar/stat_atim.go20
-rw-r--r--src/pkg/archive/tar/stat_atimespec.go20
-rw-r--r--src/pkg/archive/tar/stat_unix.go32
-rw-r--r--src/pkg/archive/tar/tar_test.go271
-rw-r--r--src/pkg/archive/tar/testdata/pax.tarbin0 -> 10240 bytes
-rw-r--r--src/pkg/archive/tar/testdata/ustar.tarbin0 -> 2048 bytes
-rw-r--r--src/pkg/archive/tar/writer.go155
-rw-r--r--src/pkg/archive/tar/writer_test.go80
-rw-r--r--src/pkg/archive/zip/reader.go121
-rw-r--r--src/pkg/archive/zip/reader_test.go11
-rw-r--r--src/pkg/archive/zip/struct.go93
-rw-r--r--src/pkg/archive/zip/testdata/zip64.zipbin0 -> 242 bytes
-rw-r--r--src/pkg/archive/zip/writer.go136
-rw-r--r--src/pkg/archive/zip/zip_test.go198
18 files changed, 1589 insertions, 122 deletions
diff --git a/src/pkg/archive/tar/common.go b/src/pkg/archive/tar/common.go
index fc7a40923..60d207c48 100644
--- a/src/pkg/archive/tar/common.go
+++ b/src/pkg/archive/tar/common.go
@@ -9,9 +9,16 @@
// References:
// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
// http://www.gnu.org/software/tar/manual/html_node/Standard.html
+// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
package tar
-import "time"
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "time"
+)
const (
blockSize = 512
@@ -28,6 +35,8 @@ const (
TypeCont = '7' // reserved
TypeXHeader = 'x' // extended header
TypeXGlobalHeader = 'g' // global extended header
+ TypeGNULongName = 'L' // Next file has a long name
+ TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name
)
// A Header represents a single header in a tar archive.
@@ -49,6 +58,179 @@ type Header struct {
ChangeTime time.Time // status change time
}
+// File name constants from the tar spec.
+const (
+ fileNameSize = 100 // Maximum number of bytes in a standard tar name.
+ fileNamePrefixSize = 155 // Maximum number of ustar extension bytes.
+)
+
+// FileInfo returns an os.FileInfo for the Header.
+func (h *Header) FileInfo() os.FileInfo {
+ return headerFileInfo{h}
+}
+
+// headerFileInfo implements os.FileInfo.
+type headerFileInfo struct {
+ h *Header
+}
+
+func (fi headerFileInfo) Size() int64 { return fi.h.Size }
+func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() }
+func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime }
+func (fi headerFileInfo) Sys() interface{} { return fi.h }
+
+// Name returns the base name of the file.
+func (fi headerFileInfo) Name() string {
+ if fi.IsDir() {
+ return path.Clean(fi.h.Name)
+ }
+ return fi.h.Name
+}
+
+// Mode returns the permission and mode bits for the headerFileInfo.
+func (fi headerFileInfo) Mode() (mode os.FileMode) {
+ // Set file permission bits.
+ mode = os.FileMode(fi.h.Mode).Perm()
+
+ // Set setuid, setgid and sticky bits.
+ if fi.h.Mode&c_ISUID != 0 {
+ // setuid
+ mode |= os.ModeSetuid
+ }
+ if fi.h.Mode&c_ISGID != 0 {
+ // setgid
+ mode |= os.ModeSetgid
+ }
+ if fi.h.Mode&c_ISVTX != 0 {
+ // sticky
+ mode |= os.ModeSticky
+ }
+
+ // Set file mode bits.
+ // clear perm, setuid, setgid and sticky bits.
+ m := os.FileMode(fi.h.Mode) &^ 07777
+ if m == c_ISDIR {
+ // directory
+ mode |= os.ModeDir
+ }
+ if m == c_ISFIFO {
+ // named pipe (FIFO)
+ mode |= os.ModeNamedPipe
+ }
+ if m == c_ISLNK {
+ // symbolic link
+ mode |= os.ModeSymlink
+ }
+ if m == c_ISBLK {
+ // device file
+ mode |= os.ModeDevice
+ }
+ if m == c_ISCHR {
+ // Unix character device
+ mode |= os.ModeDevice
+ mode |= os.ModeCharDevice
+ }
+ if m == c_ISSOCK {
+ // Unix domain socket
+ mode |= os.ModeSocket
+ }
+
+ switch fi.h.Typeflag {
+ case TypeLink, TypeSymlink:
+ // hard link, symbolic link
+ mode |= os.ModeSymlink
+ case TypeChar:
+ // character device node
+ mode |= os.ModeDevice
+ mode |= os.ModeCharDevice
+ case TypeBlock:
+ // block device node
+ mode |= os.ModeDevice
+ case TypeDir:
+ // directory
+ mode |= os.ModeDir
+ case TypeFifo:
+ // fifo node
+ mode |= os.ModeNamedPipe
+ }
+
+ return mode
+}
+
+// sysStat, if non-nil, populates h from system-dependent fields of fi.
+var sysStat func(fi os.FileInfo, h *Header) error
+
+// Mode constants from the tar spec.
+const (
+ c_ISUID = 04000 // Set uid
+ c_ISGID = 02000 // Set gid
+ c_ISVTX = 01000 // Save text (sticky bit)
+ c_ISDIR = 040000 // Directory
+ c_ISFIFO = 010000 // FIFO
+ c_ISREG = 0100000 // Regular file
+ c_ISLNK = 0120000 // Symbolic link
+ c_ISBLK = 060000 // Block special file
+ c_ISCHR = 020000 // Character special file
+ c_ISSOCK = 0140000 // Socket
+)
+
+// FileInfoHeader creates a partially-populated Header from fi.
+// If fi describes a symlink, FileInfoHeader records link as the link target.
+// If fi describes a directory, a slash is appended to the name.
+func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
+ if fi == nil {
+ return nil, errors.New("tar: FileInfo is nil")
+ }
+ fm := fi.Mode()
+ h := &Header{
+ Name: fi.Name(),
+ ModTime: fi.ModTime(),
+ Mode: int64(fm.Perm()), // or'd with c_IS* constants later
+ }
+ switch {
+ case fm.IsRegular():
+ h.Mode |= c_ISREG
+ h.Typeflag = TypeReg
+ h.Size = fi.Size()
+ case fi.IsDir():
+ h.Typeflag = TypeDir
+ h.Mode |= c_ISDIR
+ h.Name += "/"
+ case fm&os.ModeSymlink != 0:
+ h.Typeflag = TypeSymlink
+ h.Mode |= c_ISLNK
+ h.Linkname = link
+ case fm&os.ModeDevice != 0:
+ if fm&os.ModeCharDevice != 0 {
+ h.Mode |= c_ISCHR
+ h.Typeflag = TypeChar
+ } else {
+ h.Mode |= c_ISBLK
+ h.Typeflag = TypeBlock
+ }
+ case fm&os.ModeNamedPipe != 0:
+ h.Typeflag = TypeFifo
+ h.Mode |= c_ISFIFO
+ case fm&os.ModeSocket != 0:
+ h.Mode |= c_ISSOCK
+ default:
+ return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm)
+ }
+ if fm&os.ModeSetuid != 0 {
+ h.Mode |= c_ISUID
+ }
+ if fm&os.ModeSetgid != 0 {
+ h.Mode |= c_ISGID
+ }
+ if fm&os.ModeSticky != 0 {
+ h.Mode |= c_ISVTX
+ }
+ if sysStat != nil {
+ return h, sysStat(fi, h)
+ }
+ return h, nil
+}
+
var zeroBlock = make([]byte, blockSize)
// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values.
diff --git a/src/pkg/archive/tar/example_test.go b/src/pkg/archive/tar/example_test.go
new file mode 100644
index 000000000..351eaa0e6
--- /dev/null
+++ b/src/pkg/archive/tar/example_test.go
@@ -0,0 +1,79 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar_test
+
+import (
+ "archive/tar"
+ "bytes"
+ "fmt"
+ "io"
+ "log"
+ "os"
+)
+
+func Example() {
+ // Create a buffer to write our archive to.
+ buf := new(bytes.Buffer)
+
+ // Create a new tar archive.
+ tw := tar.NewWriter(buf)
+
+ // Add some files to the archive.
+ var files = []struct {
+ Name, Body string
+ }{
+ {"readme.txt", "This archive contains some text files."},
+ {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
+ {"todo.txt", "Get animal handling licence."},
+ }
+ for _, file := range files {
+ hdr := &tar.Header{
+ Name: file.Name,
+ Size: int64(len(file.Body)),
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ log.Fatalln(err)
+ }
+ if _, err := tw.Write([]byte(file.Body)); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ // Make sure to check the error on Close.
+ if err := tw.Close(); err != nil {
+ log.Fatalln(err)
+ }
+
+ // Open the tar archive for reading.
+ r := bytes.NewReader(buf.Bytes())
+ tr := tar.NewReader(r)
+
+ // Iterate through the files in the archive.
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ // end of tar archive
+ break
+ }
+ if err != nil {
+ log.Fatalln(err)
+ }
+ fmt.Printf("Contents of %s:\n", hdr.Name)
+ if _, err := io.Copy(os.Stdout, tr); err != nil {
+ log.Fatalln(err)
+ }
+ fmt.Println()
+ }
+
+ // Output:
+ // Contents of readme.txt:
+ // This archive contains some text files.
+ // Contents of gopher.txt:
+ // Gopher names:
+ // George
+ // Geoffrey
+ // Gonzo
+ // Contents of todo.txt:
+ // Get animal handling licence.
+}
diff --git a/src/pkg/archive/tar/reader.go b/src/pkg/archive/tar/reader.go
index 1b40af812..05f82a40d 100644
--- a/src/pkg/archive/tar/reader.go
+++ b/src/pkg/archive/tar/reader.go
@@ -14,6 +14,7 @@ import (
"io/ioutil"
"os"
"strconv"
+ "strings"
"time"
)
@@ -21,24 +22,12 @@ var (
ErrHeader = errors.New("archive/tar: invalid tar header")
)
+const maxNanoSecondIntSize = 9
+
// A Reader provides sequential access to the contents of a tar archive.
// A tar archive consists of a sequence of files.
// The Next method advances to the next file in the archive (including the first),
// and then it can be treated as an io.Reader to access the file's data.
-//
-// Example:
-// tr := tar.NewReader(r)
-// for {
-// hdr, err := tr.Next()
-// if err == io.EOF {
-// // end of tar archive
-// break
-// }
-// if err != nil {
-// // handle error
-// }
-// io.Copy(data, tr)
-// }
type Reader struct {
r io.Reader
err error
@@ -55,13 +44,183 @@ func (tr *Reader) Next() (*Header, error) {
if tr.err == nil {
tr.skipUnread()
}
- if tr.err == nil {
+ if tr.err != nil {
+ return hdr, tr.err
+ }
+ hdr = tr.readHeader()
+ if hdr == nil {
+ return hdr, tr.err
+ }
+ // Check for PAX/GNU header.
+ switch hdr.Typeflag {
+ case TypeXHeader:
+ // PAX extended header
+ headers, err := parsePAX(tr)
+ if err != nil {
+ return nil, err
+ }
+ // We actually read the whole file,
+ // but this skips alignment padding
+ tr.skipUnread()
hdr = tr.readHeader()
+ mergePAX(hdr, headers)
+ return hdr, nil
+ case TypeGNULongName:
+ // We have a GNU long name header. Its contents are the real file name.
+ realname, err := ioutil.ReadAll(tr)
+ if err != nil {
+ return nil, err
+ }
+ hdr, err := tr.Next()
+ hdr.Name = cString(realname)
+ return hdr, err
+ case TypeGNULongLink:
+ // We have a GNU long link header.
+ realname, err := ioutil.ReadAll(tr)
+ if err != nil {
+ return nil, err
+ }
+ hdr, err := tr.Next()
+ hdr.Linkname = cString(realname)
+ return hdr, err
}
return hdr, tr.err
}
-// Parse bytes as a NUL-terminated C-style string.
+// mergePAX merges well known headers according to PAX standard.
+// In general headers with the same name as those found
+// in the header struct overwrite those found in the header
+// struct with higher precision or longer values. Esp. useful
+// for name and linkname fields.
+func mergePAX(hdr *Header, headers map[string]string) error {
+ for k, v := range headers {
+ switch k {
+ case "path":
+ hdr.Name = v
+ case "linkpath":
+ hdr.Linkname = v
+ case "gname":
+ hdr.Gname = v
+ case "uname":
+ hdr.Uname = v
+ case "uid":
+ uid, err := strconv.ParseInt(v, 10, 0)
+ if err != nil {
+ return err
+ }
+ hdr.Uid = int(uid)
+ case "gid":
+ gid, err := strconv.ParseInt(v, 10, 0)
+ if err != nil {
+ return err
+ }
+ hdr.Gid = int(gid)
+ case "atime":
+ t, err := parsePAXTime(v)
+ if err != nil {
+ return err
+ }
+ hdr.AccessTime = t
+ case "mtime":
+ t, err := parsePAXTime(v)
+ if err != nil {
+ return err
+ }
+ hdr.ModTime = t
+ case "ctime":
+ t, err := parsePAXTime(v)
+ if err != nil {
+ return err
+ }
+ hdr.ChangeTime = t
+ case "size":
+ size, err := strconv.ParseInt(v, 10, 0)
+ if err != nil {
+ return err
+ }
+ hdr.Size = int64(size)
+ }
+
+ }
+ return nil
+}
+
+// parsePAXTime takes a string of the form %d.%d as described in
+// the PAX specification.
+func parsePAXTime(t string) (time.Time, error) {
+ buf := []byte(t)
+ pos := bytes.IndexByte(buf, '.')
+ var seconds, nanoseconds int64
+ var err error
+ if pos == -1 {
+ seconds, err = strconv.ParseInt(t, 10, 0)
+ if err != nil {
+ return time.Time{}, err
+ }
+ } else {
+ seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0)
+ if err != nil {
+ return time.Time{}, err
+ }
+ nano_buf := string(buf[pos+1:])
+ // Pad as needed before converting to a decimal.
+ // For example .030 -> .030000000 -> 30000000 nanoseconds
+ if len(nano_buf) < maxNanoSecondIntSize {
+ // Right pad
+ nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf))
+ } else if len(nano_buf) > maxNanoSecondIntSize {
+ // Right truncate
+ nano_buf = nano_buf[:maxNanoSecondIntSize]
+ }
+ nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0)
+ if err != nil {
+ return time.Time{}, err
+ }
+ }
+ ts := time.Unix(seconds, nanoseconds)
+ return ts, nil
+}
+
+// parsePAX parses PAX headers.
+// If an extended header (type 'x') is invalid, ErrHeader is returned
+func parsePAX(r io.Reader) (map[string]string, error) {
+ buf, err := ioutil.ReadAll(r)
+ if err != nil {
+ return nil, err
+ }
+ headers := make(map[string]string)
+ // Each record is constructed as
+ // "%d %s=%s\n", length, keyword, value
+ for len(buf) > 0 {
+ // or the header was empty to start with.
+ var sp int
+ // The size field ends at the first space.
+ sp = bytes.IndexByte(buf, ' ')
+ if sp == -1 {
+ return nil, ErrHeader
+ }
+ // Parse the first token as a decimal integer.
+ n, err := strconv.ParseInt(string(buf[:sp]), 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+ // Extract everything between the decimal and the n -1 on the
+ // beginning to to eat the ' ', -1 on the end to skip the newline.
+ var record []byte
+ record, buf = buf[sp+1:n-1], buf[n:]
+ // The first equals is guaranteed to mark the end of the key.
+ // Everything else is value.
+ eq := bytes.IndexByte(record, '=')
+ if eq == -1 {
+ return nil, ErrHeader
+ }
+ key, value := record[:eq], record[eq+1:]
+ headers[string(key)] = string(value)
+ }
+ return headers, nil
+}
+
+// cString parses bytes as a NUL-terminated C-style string.
// If a NUL byte is not found then the whole slice is returned as a string.
func cString(b []byte) string {
n := 0
@@ -72,6 +231,18 @@ func cString(b []byte) string {
}
func (tr *Reader) octal(b []byte) int64 {
+ // Check for binary format first.
+ if len(b) > 0 && b[0]&0x80 != 0 {
+ var x int64
+ for i, c := range b {
+ if i == 0 {
+ c &= 0x7f // ignore signal bit in first byte
+ }
+ x = x<<8 | int64(c)
+ }
+ return x
+ }
+
// Removing leading spaces.
for len(b) > 0 && b[0] == ' ' {
b = b[1:]
@@ -87,7 +258,7 @@ func (tr *Reader) octal(b []byte) int64 {
return int64(x)
}
-// Skip any unread bytes in the existing file entry, as well as any alignment padding.
+// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
func (tr *Reader) skipUnread() {
nr := tr.nb + tr.pad // number of bytes to skip
tr.nb, tr.pad = 0, 0
diff --git a/src/pkg/archive/tar/reader_test.go b/src/pkg/archive/tar/reader_test.go
index 0a8646c39..9a1968237 100644
--- a/src/pkg/archive/tar/reader_test.go
+++ b/src/pkg/archive/tar/reader_test.go
@@ -10,6 +10,8 @@ import (
"fmt"
"io"
"os"
+ "reflect"
+ "strings"
"testing"
"time"
)
@@ -108,6 +110,38 @@ var untarTests = []*untarTest{
},
},
},
+ {
+ file: "testdata/pax.tar",
+ headers: []*Header{
+ {
+ Name: "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100",
+ Mode: 0664,
+ Uid: 1000,
+ Gid: 1000,
+ Uname: "shane",
+ Gname: "shane",
+ Size: 7,
+ ModTime: time.Unix(1350244992, 23960108),
+ ChangeTime: time.Unix(1350244992, 23960108),
+ AccessTime: time.Unix(1350244992, 23960108),
+ Typeflag: TypeReg,
+ },
+ {
+ Name: "a/b",
+ Mode: 0777,
+ Uid: 1000,
+ Gid: 1000,
+ Uname: "shane",
+ Gname: "shane",
+ Size: 0,
+ ModTime: time.Unix(1350266320, 910238425),
+ ChangeTime: time.Unix(1350266320, 910238425),
+ AccessTime: time.Unix(1350266320, 910238425),
+ Typeflag: TypeSymlink,
+ Linkname: "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100",
+ },
+ },
+ },
}
func TestReader(t *testing.T) {
@@ -133,7 +167,7 @@ testLoop:
}
hdr, err := tr.Next()
if err == io.EOF {
- break
+ continue testLoop
}
if hdr != nil || err != nil {
t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err)
@@ -260,3 +294,73 @@ func TestNonSeekable(t *testing.T) {
t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread)
}
}
+
+func TestParsePAXHeader(t *testing.T) {
+ paxTests := [][3]string{
+ {"a", "a=name", "10 a=name\n"}, // Test case involving multiple acceptable lengths
+ {"a", "a=name", "9 a=name\n"}, // Test case involving multiple acceptable length
+ {"mtime", "mtime=1350244992.023960108", "30 mtime=1350244992.023960108\n"}}
+ for _, test := range paxTests {
+ key, expected, raw := test[0], test[1], test[2]
+ reader := bytes.NewBuffer([]byte(raw))
+ headers, err := parsePAX(reader)
+ if err != nil {
+ t.Errorf("Couldn't parse correctly formatted headers: %v", err)
+ continue
+ }
+ if strings.EqualFold(headers[key], expected) {
+ t.Errorf("mtime header incorrectly parsed: got %s, wanted %s", headers[key], expected)
+ continue
+ }
+ trailer := make([]byte, 100)
+ n, err := reader.Read(trailer)
+ if err != io.EOF || n != 0 {
+ t.Error("Buffer wasn't consumed")
+ }
+ }
+ badHeader := bytes.NewBuffer([]byte("3 somelongkey="))
+ if _, err := parsePAX(badHeader); err != ErrHeader {
+ t.Fatal("Unexpected success when parsing bad header")
+ }
+}
+
+func TestParsePAXTime(t *testing.T) {
+ // Some valid PAX time values
+ timestamps := map[string]time.Time{
+ "1350244992.023960108": time.Unix(1350244992, 23960108), // The commoon case
+ "1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value
+ "1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value
+ "1350244992": time.Unix(1350244992, 0), // Low precision value
+ }
+ for input, expected := range timestamps {
+ ts, err := parsePAXTime(input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ts.Equal(expected) {
+ t.Fatalf("Time parsing failure %s %s", ts, expected)
+ }
+ }
+}
+
+func TestMergePAX(t *testing.T) {
+ hdr := new(Header)
+ // Test a string, integer, and time based value.
+ headers := map[string]string{
+ "path": "a/b/c",
+ "uid": "1000",
+ "mtime": "1350244992.023960108",
+ }
+ err := mergePAX(hdr, headers)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := &Header{
+ Name: "a/b/c",
+ Uid: 1000,
+ ModTime: time.Unix(1350244992, 23960108),
+ }
+ if !reflect.DeepEqual(hdr, want) {
+ t.Errorf("incorrect merge: got %+v, want %+v", hdr, want)
+ }
+}
diff --git a/src/pkg/archive/tar/stat_atim.go b/src/pkg/archive/tar/stat_atim.go
new file mode 100644
index 000000000..6029b0871
--- /dev/null
+++ b/src/pkg/archive/tar/stat_atim.go
@@ -0,0 +1,20 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux openbsd
+
+package tar
+
+import (
+ "syscall"
+ "time"
+)
+
+func statAtime(st *syscall.Stat_t) time.Time {
+ return time.Unix(st.Atim.Unix())
+}
+
+func statCtime(st *syscall.Stat_t) time.Time {
+ return time.Unix(st.Ctim.Unix())
+}
diff --git a/src/pkg/archive/tar/stat_atimespec.go b/src/pkg/archive/tar/stat_atimespec.go
new file mode 100644
index 000000000..6f17dbe30
--- /dev/null
+++ b/src/pkg/archive/tar/stat_atimespec.go
@@ -0,0 +1,20 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd netbsd
+
+package tar
+
+import (
+ "syscall"
+ "time"
+)
+
+func statAtime(st *syscall.Stat_t) time.Time {
+ return time.Unix(st.Atimespec.Unix())
+}
+
+func statCtime(st *syscall.Stat_t) time.Time {
+ return time.Unix(st.Ctimespec.Unix())
+}
diff --git a/src/pkg/archive/tar/stat_unix.go b/src/pkg/archive/tar/stat_unix.go
new file mode 100644
index 000000000..92bc92424
--- /dev/null
+++ b/src/pkg/archive/tar/stat_unix.go
@@ -0,0 +1,32 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin freebsd openbsd netbsd
+
+package tar
+
+import (
+ "os"
+ "syscall"
+)
+
+func init() {
+ sysStat = statUnix
+}
+
+func statUnix(fi os.FileInfo, h *Header) error {
+ sys, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok {
+ return nil
+ }
+ h.Uid = int(sys.Uid)
+ h.Gid = int(sys.Gid)
+ // TODO(bradfitz): populate username & group. os/user
+ // doesn't cache LookupId lookups, and lacks group
+ // lookup functions.
+ h.AccessTime = statAtime(sys)
+ h.ChangeTime = statCtime(sys)
+ // TODO(bradfitz): major/minor device numbers?
+ return nil
+}
diff --git a/src/pkg/archive/tar/tar_test.go b/src/pkg/archive/tar/tar_test.go
new file mode 100644
index 000000000..dd6310313
--- /dev/null
+++ b/src/pkg/archive/tar/tar_test.go
@@ -0,0 +1,271 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestFileInfoHeader(t *testing.T) {
+ fi, err := os.Stat("testdata/small.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := FileInfoHeader(fi, "")
+ if err != nil {
+ t.Fatalf("FileInfoHeader: %v", err)
+ }
+ if g, e := h.Name, "small.txt"; g != e {
+ t.Errorf("Name = %q; want %q", g, e)
+ }
+ if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e {
+ t.Errorf("Mode = %#o; want %#o", g, e)
+ }
+ if g, e := h.Size, int64(5); g != e {
+ t.Errorf("Size = %v; want %v", g, e)
+ }
+ if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
+ t.Errorf("ModTime = %v; want %v", g, e)
+ }
+}
+
+func TestFileInfoHeaderDir(t *testing.T) {
+ fi, err := os.Stat("testdata")
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := FileInfoHeader(fi, "")
+ if err != nil {
+ t.Fatalf("FileInfoHeader: %v", err)
+ }
+ if g, e := h.Name, "testdata/"; g != e {
+ t.Errorf("Name = %q; want %q", g, e)
+ }
+ // Ignoring c_ISGID for golang.org/issue/4867
+ if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e {
+ t.Errorf("Mode = %#o; want %#o", g, e)
+ }
+ if g, e := h.Size, int64(0); g != e {
+ t.Errorf("Size = %v; want %v", g, e)
+ }
+ if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
+ t.Errorf("ModTime = %v; want %v", g, e)
+ }
+}
+
+func TestFileInfoHeaderSymlink(t *testing.T) {
+ h, err := FileInfoHeader(symlink{}, "some-target")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if g, e := h.Name, "some-symlink"; g != e {
+ t.Errorf("Name = %q; want %q", g, e)
+ }
+ if g, e := h.Linkname, "some-target"; g != e {
+ t.Errorf("Linkname = %q; want %q", g, e)
+ }
+}
+
+type symlink struct{}
+
+func (symlink) Name() string { return "some-symlink" }
+func (symlink) Size() int64 { return 0 }
+func (symlink) Mode() os.FileMode { return os.ModeSymlink }
+func (symlink) ModTime() time.Time { return time.Time{} }
+func (symlink) IsDir() bool { return false }
+func (symlink) Sys() interface{} { return nil }
+
+func TestRoundTrip(t *testing.T) {
+ data := []byte("some file contents")
+
+ var b bytes.Buffer
+ tw := NewWriter(&b)
+ hdr := &Header{
+ Name: "file.txt",
+ Uid: 1 << 21, // too big for 8 octal digits
+ Size: int64(len(data)),
+ ModTime: time.Now(),
+ }
+ // tar only supports second precision.
+ hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
+ if err := tw.WriteHeader(hdr); err != nil {
+ t.Fatalf("tw.WriteHeader: %v", err)
+ }
+ if _, err := tw.Write(data); err != nil {
+ t.Fatalf("tw.Write: %v", err)
+ }
+ if err := tw.Close(); err != nil {
+ t.Fatalf("tw.Close: %v", err)
+ }
+
+ // Read it back.
+ tr := NewReader(&b)
+ rHdr, err := tr.Next()
+ if err != nil {
+ t.Fatalf("tr.Next: %v", err)
+ }
+ if !reflect.DeepEqual(rHdr, hdr) {
+ t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
+ }
+ rData, err := ioutil.ReadAll(tr)
+ if err != nil {
+ t.Fatalf("Read: %v", err)
+ }
+ if !bytes.Equal(rData, data) {
+ t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
+ }
+}
+
+type headerRoundTripTest struct {
+ h *Header
+ fm os.FileMode
+}
+
+func TestHeaderRoundTrip(t *testing.T) {
+ golden := []headerRoundTripTest{
+ // regular file.
+ {
+ h: &Header{
+ Name: "test.txt",
+ Mode: 0644 | c_ISREG,
+ Size: 12,
+ ModTime: time.Unix(1360600916, 0),
+ Typeflag: TypeReg,
+ },
+ fm: 0644,
+ },
+ // hard link.
+ {
+ h: &Header{
+ Name: "hard.txt",
+ Mode: 0644 | c_ISLNK,
+ Size: 0,
+ ModTime: time.Unix(1360600916, 0),
+ Typeflag: TypeLink,
+ },
+ fm: 0644 | os.ModeSymlink,
+ },
+ // symbolic link.
+ {
+ h: &Header{
+ Name: "link.txt",
+ Mode: 0777 | c_ISLNK,
+ Size: 0,
+ ModTime: time.Unix(1360600852, 0),
+ Typeflag: TypeSymlink,
+ },
+ fm: 0777 | os.ModeSymlink,
+ },
+ // character device node.
+ {
+ h: &Header{
+ Name: "dev/null",
+ Mode: 0666 | c_ISCHR,
+ Size: 0,
+ ModTime: time.Unix(1360578951, 0),
+ Typeflag: TypeChar,
+ },
+ fm: 0666 | os.ModeDevice | os.ModeCharDevice,
+ },
+ // block device node.
+ {
+ h: &Header{
+ Name: "dev/sda",
+ Mode: 0660 | c_ISBLK,
+ Size: 0,
+ ModTime: time.Unix(1360578954, 0),
+ Typeflag: TypeBlock,
+ },
+ fm: 0660 | os.ModeDevice,
+ },
+ // directory.
+ {
+ h: &Header{
+ Name: "dir/",
+ Mode: 0755 | c_ISDIR,
+ Size: 0,
+ ModTime: time.Unix(1360601116, 0),
+ Typeflag: TypeDir,
+ },
+ fm: 0755 | os.ModeDir,
+ },
+ // fifo node.
+ {
+ h: &Header{
+ Name: "dev/initctl",
+ Mode: 0600 | c_ISFIFO,
+ Size: 0,
+ ModTime: time.Unix(1360578949, 0),
+ Typeflag: TypeFifo,
+ },
+ fm: 0600 | os.ModeNamedPipe,
+ },
+ // setuid.
+ {
+ h: &Header{
+ Name: "bin/su",
+ Mode: 0755 | c_ISREG | c_ISUID,
+ Size: 23232,
+ ModTime: time.Unix(1355405093, 0),
+ Typeflag: TypeReg,
+ },
+ fm: 0755 | os.ModeSetuid,
+ },
+ // setguid.
+ {
+ h: &Header{
+ Name: "group.txt",
+ Mode: 0750 | c_ISREG | c_ISGID,
+ Size: 0,
+ ModTime: time.Unix(1360602346, 0),
+ Typeflag: TypeReg,
+ },
+ fm: 0750 | os.ModeSetgid,
+ },
+ // sticky.
+ {
+ h: &Header{
+ Name: "sticky.txt",
+ Mode: 0600 | c_ISREG | c_ISVTX,
+ Size: 7,
+ ModTime: time.Unix(1360602540, 0),
+ Typeflag: TypeReg,
+ },
+ fm: 0600 | os.ModeSticky,
+ },
+ }
+
+ for i, g := range golden {
+ fi := g.h.FileInfo()
+ h2, err := FileInfoHeader(fi, "")
+ if err != nil {
+ t.Error(err)
+ continue
+ }
+ if got, want := h2.Name, g.h.Name; got != want {
+ t.Errorf("i=%d: Name: got %v, want %v", i, got, want)
+ }
+ if got, want := h2.Size, g.h.Size; got != want {
+ t.Errorf("i=%d: Size: got %v, want %v", i, got, want)
+ }
+ if got, want := h2.Mode, g.h.Mode; got != want {
+ t.Errorf("i=%d: Mode: got %o, want %o", i, got, want)
+ }
+ if got, want := fi.Mode(), g.fm; got != want {
+ t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want)
+ }
+ if got, want := h2.ModTime, g.h.ModTime; got != want {
+ t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want)
+ }
+ if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h {
+ t.Errorf("i=%d: Sys didn't return original *Header", i)
+ }
+ }
+}
diff --git a/src/pkg/archive/tar/testdata/pax.tar b/src/pkg/archive/tar/testdata/pax.tar
new file mode 100644
index 000000000..9bc24b658
--- /dev/null
+++ b/src/pkg/archive/tar/testdata/pax.tar
Binary files differ
diff --git a/src/pkg/archive/tar/testdata/ustar.tar b/src/pkg/archive/tar/testdata/ustar.tar
new file mode 100644
index 000000000..29679d9a3
--- /dev/null
+++ b/src/pkg/archive/tar/testdata/ustar.tar
Binary files differ
diff --git a/src/pkg/archive/tar/writer.go b/src/pkg/archive/tar/writer.go
index b2b7a58a1..d92dd06ea 100644
--- a/src/pkg/archive/tar/writer.go
+++ b/src/pkg/archive/tar/writer.go
@@ -8,33 +8,28 @@ package tar
// - catch more errors (no first header, etc.)
import (
+ "bytes"
"errors"
"fmt"
"io"
+ "os"
+ "path"
"strconv"
+ "strings"
+ "time"
)
var (
ErrWriteTooLong = errors.New("archive/tar: write too long")
ErrFieldTooLong = errors.New("archive/tar: header field too long")
ErrWriteAfterClose = errors.New("archive/tar: write after close")
+ errNameTooLong = errors.New("archive/tar: name too long")
)
// A Writer provides sequential writing of a tar archive in POSIX.1 format.
// A tar archive consists of a sequence of files.
// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
// writing at most hdr.Size bytes in total.
-//
-// Example:
-// tw := tar.NewWriter(w)
-// hdr := new(Header)
-// hdr.Size = length of data in bytes
-// // populate other hdr fields as desired
-// if err := tw.WriteHeader(hdr); err != nil {
-// // handle error
-// }
-// io.Copy(tw, data)
-// tw.Close()
type Writer struct {
w io.Writer
err error
@@ -110,6 +105,12 @@ func (tw *Writer) numeric(b []byte, x int64) {
b[0] |= 0x80 // highest bit indicates binary format
}
+var (
+ minTime = time.Unix(0, 0)
+ // There is room for 11 octal digits (33 bits) of mtime.
+ maxTime = minTime.Add((1<<33 - 1) * time.Second)
+)
+
// WriteHeader writes hdr and prepares to accept the file's contents.
// WriteHeader calls Flush if it is not the first header.
// Calling after a Close will return ErrWriteAfterClose.
@@ -123,34 +124,60 @@ func (tw *Writer) WriteHeader(hdr *Header) error {
if tw.err != nil {
return tw.err
}
-
+ // Decide whether or not to use PAX extensions
+ // TODO(shanemhansen): we might want to use PAX headers for
+ // subsecond time resolution, but for now let's just capture
+ // the long name/long symlink use case.
+ suffix := hdr.Name
+ prefix := ""
+ if len(hdr.Name) > fileNameSize || len(hdr.Linkname) > fileNameSize {
+ var err error
+ prefix, suffix, err = tw.splitUSTARLongName(hdr.Name)
+ // Either we were unable to pack the long name into ustar format
+ // or the link name is too long; use PAX headers.
+ if err == errNameTooLong || len(hdr.Linkname) > fileNameSize {
+ if err := tw.writePAXHeader(hdr); err != nil {
+ return err
+ }
+ } else if err != nil {
+ return err
+ }
+ }
tw.nb = int64(hdr.Size)
tw.pad = -tw.nb & (blockSize - 1) // blockSize is a power of two
header := make([]byte, blockSize)
s := slicer(header)
+ tw.cString(s.next(fileNameSize), suffix)
- // TODO(dsymonds): handle names longer than 100 chars
- copy(s.next(100), []byte(hdr.Name))
-
- tw.octal(s.next(8), hdr.Mode) // 100:108
- tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116
- tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124
- tw.numeric(s.next(12), hdr.Size) // 124:136
- tw.numeric(s.next(12), hdr.ModTime.Unix()) // 136:148
- s.next(8) // chksum (148:156)
- s.next(1)[0] = hdr.Typeflag // 156:157
- tw.cString(s.next(100), hdr.Linkname) // linkname (157:257)
- copy(s.next(8), []byte("ustar\x0000")) // 257:265
- tw.cString(s.next(32), hdr.Uname) // 265:297
- tw.cString(s.next(32), hdr.Gname) // 297:329
- tw.numeric(s.next(8), hdr.Devmajor) // 329:337
- tw.numeric(s.next(8), hdr.Devminor) // 337:345
+ // Handle out of range ModTime carefully.
+ var modTime int64
+ if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
+ modTime = hdr.ModTime.Unix()
+ }
+ tw.octal(s.next(8), hdr.Mode) // 100:108
+ tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116
+ tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124
+ tw.numeric(s.next(12), hdr.Size) // 124:136
+ tw.numeric(s.next(12), modTime) // 136:148
+ s.next(8) // chksum (148:156)
+ s.next(1)[0] = hdr.Typeflag // 156:157
+ tw.cString(s.next(100), hdr.Linkname) // linkname (157:257)
+ copy(s.next(8), []byte("ustar\x0000")) // 257:265
+ tw.cString(s.next(32), hdr.Uname) // 265:297
+ tw.cString(s.next(32), hdr.Gname) // 297:329
+ tw.numeric(s.next(8), hdr.Devmajor) // 329:337
+ tw.numeric(s.next(8), hdr.Devminor) // 337:345
+ tw.cString(s.next(155), prefix) // 345:500
// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
if tw.usedBinary {
copy(header[257:265], []byte("ustar \x00"))
}
+ // Use the ustar magic if we used ustar long names.
+ if len(prefix) > 0 {
+ copy(header[257:265], []byte("ustar\000"))
+ }
// The chksum field is terminated by a NUL and a space.
// This is different from the other octal fields.
@@ -168,6 +195,78 @@ func (tw *Writer) WriteHeader(hdr *Header) error {
return tw.err
}
+// writeUSTARLongName splits a USTAR long name hdr.Name.
+// name must be < 256 characters. errNameTooLong is returned
+// if hdr.Name can't be split. The splitting heuristic
+// is compatible with gnu tar.
+func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) {
+ length := len(name)
+ if length > fileNamePrefixSize+1 {
+ length = fileNamePrefixSize + 1
+ } else if name[length-1] == '/' {
+ length--
+ }
+ i := strings.LastIndex(name[:length], "/")
+ nlen := length - i - 1
+ if i <= 0 || nlen > fileNameSize || nlen == 0 {
+ err = errNameTooLong
+ return
+ }
+ prefix, suffix = name[:i], name[i+1:]
+ return
+}
+
+// writePaxHeader writes an extended pax header to the
+// archive.
+func (tw *Writer) writePAXHeader(hdr *Header) error {
+ // Prepare extended header
+ ext := new(Header)
+ ext.Typeflag = TypeXHeader
+ // Setting ModTime is required for reader parsing to
+ // succeed, and seems harmless enough.
+ ext.ModTime = hdr.ModTime
+ // The spec asks that we namespace our pseudo files
+ // with the current pid.
+ pid := os.Getpid()
+ dir, file := path.Split(hdr.Name)
+ ext.Name = path.Join(dir,
+ fmt.Sprintf("PaxHeaders.%d", pid), file)[0:100]
+ // Construct the body
+ var buf bytes.Buffer
+ if len(hdr.Name) > fileNameSize {
+ fmt.Fprint(&buf, paxHeader("path="+hdr.Name))
+ }
+ if len(hdr.Linkname) > fileNameSize {
+ fmt.Fprint(&buf, paxHeader("linkpath="+hdr.Linkname))
+ }
+ ext.Size = int64(len(buf.Bytes()))
+ if err := tw.WriteHeader(ext); err != nil {
+ return err
+ }
+ if _, err := tw.Write(buf.Bytes()); err != nil {
+ return err
+ }
+ if err := tw.Flush(); err != nil {
+ return err
+ }
+ return nil
+}
+
+// paxHeader formats a single pax record, prefixing it with the appropriate length
+func paxHeader(msg string) string {
+ const padding = 2 // Extra padding for space and newline
+ size := len(msg) + padding
+ size += len(strconv.Itoa(size))
+ record := fmt.Sprintf("%d %s\n", size, msg)
+ if len(record) != size {
+ // Final adjustment if adding size increased
+ // the number of digits in size
+ size = len(record)
+ record = fmt.Sprintf("%d %s\n", size, msg)
+ }
+ return record
+}
+
// Write writes to the current entry in the tar archive.
// Write returns the error ErrWriteTooLong if more than
// hdr.Size bytes are written after WriteHeader.
diff --git a/src/pkg/archive/tar/writer_test.go b/src/pkg/archive/tar/writer_test.go
index a214e57b9..4cf7c72af 100644
--- a/src/pkg/archive/tar/writer_test.go
+++ b/src/pkg/archive/tar/writer_test.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"io/ioutil"
+ "os"
"strings"
"testing"
"testing/iotest"
@@ -101,6 +102,27 @@ var writerTests = []*writerTest{
},
},
},
+ // This file was produced using gnu tar 1.17
+ // gnutar -b 4 --format=ustar (longname/)*15 + file.txt
+ {
+ file: "testdata/ustar.tar",
+ entries: []*writerTestEntry{
+ {
+ header: &Header{
+ Name: strings.Repeat("longname/", 15) + "file.txt",
+ Mode: 0644,
+ Uid: 0765,
+ Gid: 024,
+ Size: 06,
+ ModTime: time.Unix(1360135598, 0),
+ Typeflag: '0',
+ Uname: "shane",
+ Gname: "staff",
+ },
+ contents: "hello\n",
+ },
+ },
+ },
}
// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection.
@@ -180,3 +202,61 @@ testLoop:
}
}
}
+
+func TestPax(t *testing.T) {
+ // Create an archive with a large name
+ fileinfo, err := os.Stat("testdata/small.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ hdr, err := FileInfoHeader(fileinfo, "")
+ if err != nil {
+ t.Fatalf("os.Stat: %v", err)
+ }
+ // Force a PAX long name to be written
+ longName := strings.Repeat("ab", 100)
+ contents := strings.Repeat(" ", int(hdr.Size))
+ hdr.Name = longName
+ var buf bytes.Buffer
+ writer := NewWriter(&buf)
+ if err := writer.WriteHeader(hdr); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = writer.Write([]byte(contents)); err != nil {
+ t.Fatal(err)
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ // Simple test to make sure PAX extensions are in effect
+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
+ t.Fatal("Expected at least one PAX header to be written.")
+ }
+ // Test that we can get a long name back out of the archive.
+ reader := NewReader(&buf)
+ hdr, err = reader.Next()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if hdr.Name != longName {
+ t.Fatal("Couldn't recover long file name")
+ }
+}
+
+func TestPAXHeader(t *testing.T) {
+ medName := strings.Repeat("CD", 50)
+ longName := strings.Repeat("AB", 100)
+ paxTests := [][2]string{
+ {"name=/etc/hosts", "19 name=/etc/hosts\n"},
+ {"a=b", "6 a=b\n"}, // Single digit length
+ {"a=names", "11 a=names\n"}, // Test case involving carries
+ {"name=" + longName, fmt.Sprintf("210 name=%s\n", longName)},
+ {"name=" + medName, fmt.Sprintf("110 name=%s\n", medName)}}
+
+ for _, test := range paxTests {
+ key, expected := test[0], test[1]
+ if result := paxHeader(key); result != expected {
+ t.Fatalf("paxHeader: got %s, expected %s", result, expected)
+ }
+ }
+}
diff --git a/src/pkg/archive/zip/reader.go b/src/pkg/archive/zip/reader.go
index ddd507538..c10f29a83 100644
--- a/src/pkg/archive/zip/reader.go
+++ b/src/pkg/archive/zip/reader.go
@@ -103,7 +103,7 @@ func (z *Reader) init(r io.ReaderAt, size int64) error {
}
z.File = append(z.File, f)
}
- if uint16(len(z.File)) != end.directoryRecords {
+ if uint16(len(z.File)) != uint16(end.directoryRecords) { // only compare 16 bits here
// Return the readDirectoryHeader error if we read
// the wrong number of directory entries.
return err
@@ -123,7 +123,7 @@ func (f *File) Open() (rc io.ReadCloser, err error) {
if err != nil {
return
}
- size := int64(f.CompressedSize)
+ size := int64(f.CompressedSize64)
r := io.NewSectionReader(f.zipr, f.headerOffset+bodyOffset, size)
switch f.Method {
case Store: // (no compression)
@@ -220,6 +220,8 @@ func readDirectoryHeader(f *File, r io.Reader) error {
f.CRC32 = b.uint32()
f.CompressedSize = b.uint32()
f.UncompressedSize = b.uint32()
+ f.CompressedSize64 = uint64(f.CompressedSize)
+ f.UncompressedSize64 = uint64(f.UncompressedSize)
filenameLen := int(b.uint16())
extraLen := int(b.uint16())
commentLen := int(b.uint16())
@@ -233,6 +235,35 @@ func readDirectoryHeader(f *File, r io.Reader) error {
f.Name = string(d[:filenameLen])
f.Extra = d[filenameLen : filenameLen+extraLen]
f.Comment = string(d[filenameLen+extraLen:])
+
+ if len(f.Extra) > 0 {
+ b := readBuf(f.Extra)
+ for len(b) >= 4 { // need at least tag and size
+ tag := b.uint16()
+ size := b.uint16()
+ if int(size) > len(b) {
+ return ErrFormat
+ }
+ if tag == zip64ExtraId {
+ // update directory values from the zip64 extra block
+ eb := readBuf(b)
+ if len(eb) >= 8 {
+ f.UncompressedSize64 = eb.uint64()
+ }
+ if len(eb) >= 8 {
+ f.CompressedSize64 = eb.uint64()
+ }
+ if len(eb) >= 8 {
+ f.headerOffset = int64(eb.uint64())
+ }
+ }
+ b = b[size:]
+ }
+ // Should have consumed the whole header.
+ if len(b) != 0 {
+ return ErrFormat
+ }
+ }
return nil
}
@@ -263,15 +294,23 @@ func readDataDescriptor(r io.Reader, f *File) error {
return err
}
b := readBuf(buf[:12])
- f.CRC32 = b.uint32()
- f.CompressedSize = b.uint32()
- f.UncompressedSize = b.uint32()
+ if b.uint32() != f.CRC32 {
+ return ErrChecksum
+ }
+
+ // The two sizes that follow here can be either 32 bits or 64 bits
+ // but the spec is not very clear on this and different
+ // interpretations has been made causing incompatibilities. We
+ // already have the sizes from the central directory so we can
+ // just ignore these.
+
return nil
}
func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error) {
// look for directoryEndSignature in the last 1k, then in the last 65k
var buf []byte
+ var directoryEndOffset int64
for i, bLen := range []int64{1024, 65 * 1024} {
if bLen > size {
bLen = size
@@ -282,6 +321,7 @@ func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error)
}
if p := findSignatureInBlock(buf); p >= 0 {
buf = buf[p:]
+ directoryEndOffset = size - bLen + int64(p)
break
}
if i == 1 || bLen == size {
@@ -292,12 +332,12 @@ func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error)
// read header into struct
b := readBuf(buf[4:]) // skip signature
d := &directoryEnd{
- diskNbr: b.uint16(),
- dirDiskNbr: b.uint16(),
- dirRecordsThisDisk: b.uint16(),
- directoryRecords: b.uint16(),
- directorySize: b.uint32(),
- directoryOffset: b.uint32(),
+ diskNbr: uint32(b.uint16()),
+ dirDiskNbr: uint32(b.uint16()),
+ dirRecordsThisDisk: uint64(b.uint16()),
+ directoryRecords: uint64(b.uint16()),
+ directorySize: uint64(b.uint32()),
+ directoryOffset: uint64(b.uint32()),
commentLen: b.uint16(),
}
l := int(d.commentLen)
@@ -305,9 +345,62 @@ func readDirectoryEnd(r io.ReaderAt, size int64) (dir *directoryEnd, err error)
return nil, errors.New("zip: invalid comment length")
}
d.comment = string(b[:l])
+
+ p, err := findDirectory64End(r, directoryEndOffset)
+ if err == nil && p >= 0 {
+ err = readDirectory64End(r, p, d)
+ }
+ if err != nil {
+ return nil, err
+ }
return d, nil
}
+// findDirectory64End tries to read the zip64 locator just before the
+// directory end and returns the offset of the zip64 directory end if
+// found.
+func findDirectory64End(r io.ReaderAt, directoryEndOffset int64) (int64, error) {
+ locOffset := directoryEndOffset - directory64LocLen
+ if locOffset < 0 {
+ return -1, nil // no need to look for a header outside the file
+ }
+ buf := make([]byte, directory64LocLen)
+ if _, err := r.ReadAt(buf, locOffset); err != nil {
+ return -1, err
+ }
+ b := readBuf(buf)
+ if sig := b.uint32(); sig != directory64LocSignature {
+ return -1, nil
+ }
+ b = b[4:] // skip number of the disk with the start of the zip64 end of central directory
+ p := b.uint64() // relative offset of the zip64 end of central directory record
+ return int64(p), nil
+}
+
+// readDirectory64End reads the zip64 directory end and updates the
+// directory end with the zip64 directory end values.
+func readDirectory64End(r io.ReaderAt, offset int64, d *directoryEnd) (err error) {
+ buf := make([]byte, directory64EndLen)
+ if _, err := r.ReadAt(buf, offset); err != nil {
+ return err
+ }
+
+ b := readBuf(buf)
+ if sig := b.uint32(); sig != directory64EndSignature {
+ return ErrFormat
+ }
+
+ b = b[12:] // skip dir size, version and version needed (uint64 + 2x uint16)
+ d.diskNbr = b.uint32() // number of this disk
+ d.dirDiskNbr = b.uint32() // number of the disk with the start of the central directory
+ d.dirRecordsThisDisk = b.uint64() // total number of entries in the central directory on this disk
+ d.directoryRecords = b.uint64() // total number of entries in the central directory
+ d.directorySize = b.uint64() // size of the central directory
+ d.directoryOffset = b.uint64() // offset of start of central directory with respect to the starting disk number
+
+ return nil
+}
+
func findSignatureInBlock(b []byte) int {
for i := len(b) - directoryEndLen; i >= 0; i-- {
// defined from directoryEndSignature in struct.go
@@ -335,3 +428,9 @@ func (b *readBuf) uint32() uint32 {
*b = (*b)[4:]
return v
}
+
+func (b *readBuf) uint64() uint64 {
+ v := binary.LittleEndian.Uint64(*b)
+ *b = (*b)[8:]
+ return v
+}
diff --git a/src/pkg/archive/zip/reader_test.go b/src/pkg/archive/zip/reader_test.go
index 5f1d1b28a..cf9c59c4b 100644
--- a/src/pkg/archive/zip/reader_test.go
+++ b/src/pkg/archive/zip/reader_test.go
@@ -206,6 +206,17 @@ var tests = []ZipTest{
},
},
},
+ {
+ Name: "zip64.zip",
+ File: []ZipTestFile{
+ {
+ Name: "README",
+ Content: []byte("This small file is in ZIP64 format.\n"),
+ Mtime: "08-10-12 14:33:32",
+ Mode: 0644,
+ },
+ },
+ },
}
var crossPlatform = []ZipTestFile{
diff --git a/src/pkg/archive/zip/struct.go b/src/pkg/archive/zip/struct.go
index 55f3dcfb8..ea067f355 100644
--- a/src/pkg/archive/zip/struct.go
+++ b/src/pkg/archive/zip/struct.go
@@ -7,12 +7,19 @@ Package zip provides support for reading and writing ZIP archives.
See: http://www.pkware.com/documents/casestudies/APPNOTE.TXT
-This package does not support ZIP64 or disk spanning.
+This package does not support disk spanning.
+
+A note about ZIP64:
+
+To be backwards compatible the FileHeader has both 32 and 64 bit Size
+fields. The 64 bit fields will always contain the correct value and
+for normal archives both fields will be the same. For files requiring
+the ZIP64 format the 32 bit fields will be 0xffffffff and the 64 bit
+fields must be used instead.
*/
package zip
import (
- "errors"
"os"
"time"
)
@@ -27,11 +34,16 @@ const (
fileHeaderSignature = 0x04034b50
directoryHeaderSignature = 0x02014b50
directoryEndSignature = 0x06054b50
+ directory64LocSignature = 0x07064b50
+ directory64EndSignature = 0x06064b50
dataDescriptorSignature = 0x08074b50 // de-facto standard; required by OS X Finder
fileHeaderLen = 30 // + filename + extra
directoryHeaderLen = 46 // + filename + extra + comment
directoryEndLen = 22 // + comment
dataDescriptorLen = 16 // four uint32: descriptor signature, crc32, compressed size, size
+ dataDescriptor64Len = 24 // descriptor with 8 byte sizes
+ directory64LocLen = 20 //
+ directory64EndLen = 56 // + extra
// Constants for the first byte in CreatorVersion
creatorFAT = 0
@@ -39,22 +51,35 @@ const (
creatorNTFS = 11
creatorVFAT = 14
creatorMacOSX = 19
+
+ // version numbers
+ zipVersion20 = 20 // 2.0
+ zipVersion45 = 45 // 4.5 (reads and writes zip64 archives)
+
+ // limits for non zip64 files
+ uint16max = (1 << 16) - 1
+ uint32max = (1 << 32) - 1
+
+ // extra header id's
+ zip64ExtraId = 0x0001 // zip64 Extended Information Extra Field
)
type FileHeader struct {
- Name string
- CreatorVersion uint16
- ReaderVersion uint16
- Flags uint16
- Method uint16
- ModifiedTime uint16 // MS-DOS time
- ModifiedDate uint16 // MS-DOS date
- CRC32 uint32
- CompressedSize uint32
- UncompressedSize uint32
- Extra []byte
- ExternalAttrs uint32 // Meaning depends on CreatorVersion
- Comment string
+ Name string
+ CreatorVersion uint16
+ ReaderVersion uint16
+ Flags uint16
+ Method uint16
+ ModifiedTime uint16 // MS-DOS time
+ ModifiedDate uint16 // MS-DOS date
+ CRC32 uint32
+ CompressedSize uint32 // deprecated; use CompressedSize64
+ UncompressedSize uint32 // deprecated; use UncompressedSize64
+ CompressedSize64 uint64
+ UncompressedSize64 uint64
+ Extra []byte
+ ExternalAttrs uint32 // Meaning depends on CreatorVersion
+ Comment string
}
// FileInfo returns an os.FileInfo for the FileHeader.
@@ -67,8 +92,13 @@ type headerFileInfo struct {
fh *FileHeader
}
-func (fi headerFileInfo) Name() string { return fi.fh.Name }
-func (fi headerFileInfo) Size() int64 { return int64(fi.fh.UncompressedSize) }
+func (fi headerFileInfo) Name() string { return fi.fh.Name }
+func (fi headerFileInfo) Size() int64 {
+ if fi.fh.UncompressedSize64 > 0 {
+ return int64(fi.fh.UncompressedSize64)
+ }
+ return int64(fi.fh.UncompressedSize)
+}
func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() }
func (fi headerFileInfo) ModTime() time.Time { return fi.fh.ModTime() }
func (fi headerFileInfo) Mode() os.FileMode { return fi.fh.Mode() }
@@ -78,25 +108,27 @@ func (fi headerFileInfo) Sys() interface{} { return fi.fh }
// os.FileInfo.
func FileInfoHeader(fi os.FileInfo) (*FileHeader, error) {
size := fi.Size()
- if size > (1<<32 - 1) {
- return nil, errors.New("zip: file over 4GB")
- }
fh := &FileHeader{
- Name: fi.Name(),
- UncompressedSize: uint32(size),
+ Name: fi.Name(),
+ UncompressedSize64: uint64(size),
}
fh.SetModTime(fi.ModTime())
fh.SetMode(fi.Mode())
+ if fh.UncompressedSize64 > uint32max {
+ fh.UncompressedSize = uint32max
+ } else {
+ fh.UncompressedSize = uint32(fh.UncompressedSize64)
+ }
return fh, nil
}
type directoryEnd struct {
- diskNbr uint16 // unused
- dirDiskNbr uint16 // unused
- dirRecordsThisDisk uint16 // unused
- directoryRecords uint16
- directorySize uint32
- directoryOffset uint32 // relative to file
+ diskNbr uint32 // unused
+ dirDiskNbr uint32 // unused
+ dirRecordsThisDisk uint64 // unused
+ directoryRecords uint64
+ directorySize uint64
+ directoryOffset uint64 // relative to file
commentLen uint16
comment string
}
@@ -190,6 +222,11 @@ func (h *FileHeader) SetMode(mode os.FileMode) {
}
}
+// isZip64 returns true if the file size exceeds the 32 bit limit
+func (fh *FileHeader) isZip64() bool {
+ return fh.CompressedSize64 > uint32max || fh.UncompressedSize64 > uint32max
+}
+
func msdosModeToFileMode(m uint32) (mode os.FileMode) {
if m&msdosDir != 0 {
mode = os.ModeDir | 0777
diff --git a/src/pkg/archive/zip/testdata/zip64.zip b/src/pkg/archive/zip/testdata/zip64.zip
new file mode 100644
index 000000000..a2ee1fa33
--- /dev/null
+++ b/src/pkg/archive/zip/testdata/zip64.zip
Binary files differ
diff --git a/src/pkg/archive/zip/writer.go b/src/pkg/archive/zip/writer.go
index 45eb6bd73..4c696e152 100644
--- a/src/pkg/archive/zip/writer.go
+++ b/src/pkg/archive/zip/writer.go
@@ -27,7 +27,7 @@ type Writer struct {
type header struct {
*FileHeader
- offset uint32
+ offset uint64
}
// NewWriter returns a new Writer writing a zip file to w.
@@ -62,14 +62,36 @@ func (w *Writer) Close() error {
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
b.uint32(h.CRC32)
- b.uint32(h.CompressedSize)
- b.uint32(h.UncompressedSize)
+ if h.isZip64() || h.offset > uint32max {
+ // the file needs a zip64 header. store maxint in both
+ // 32 bit size fields (and offset later) to signal that the
+ // zip64 extra header should be used.
+ b.uint32(uint32max) // compressed size
+ b.uint32(uint32max) // uncompressed size
+
+ // append a zip64 extra block to Extra
+ var buf [28]byte // 2x uint16 + 3x uint64
+ eb := writeBuf(buf[:])
+ eb.uint16(zip64ExtraId)
+ eb.uint16(24) // size = 3x uint64
+ eb.uint64(h.UncompressedSize64)
+ eb.uint64(h.CompressedSize64)
+ eb.uint64(h.offset)
+ h.Extra = append(h.Extra, buf[:]...)
+ } else {
+ b.uint32(h.CompressedSize)
+ b.uint32(h.UncompressedSize)
+ }
b.uint16(uint16(len(h.Name)))
b.uint16(uint16(len(h.Extra)))
b.uint16(uint16(len(h.Comment)))
b = b[4:] // skip disk number start and internal file attr (2x uint16)
b.uint32(h.ExternalAttrs)
- b.uint32(h.offset)
+ if h.offset > uint32max {
+ b.uint32(uint32max)
+ } else {
+ b.uint32(uint32(h.offset))
+ }
if _, err := w.cw.Write(buf[:]); err != nil {
return err
}
@@ -85,15 +107,52 @@ func (w *Writer) Close() error {
}
end := w.cw.count
+ records := uint64(len(w.dir))
+ size := uint64(end - start)
+ offset := uint64(start)
+
+ if records > uint16max || size > uint32max || offset > uint32max {
+ var buf [directory64EndLen + directory64LocLen]byte
+ b := writeBuf(buf[:])
+
+ // zip64 end of central directory record
+ b.uint32(directory64EndSignature)
+ b.uint64(directory64EndLen)
+ b.uint16(zipVersion45) // version made by
+ b.uint16(zipVersion45) // version needed to extract
+ b.uint32(0) // number of this disk
+ b.uint32(0) // number of the disk with the start of the central directory
+ b.uint64(records) // total number of entries in the central directory on this disk
+ b.uint64(records) // total number of entries in the central directory
+ b.uint64(size) // size of the central directory
+ b.uint64(offset) // offset of start of central directory with respect to the starting disk number
+
+ // zip64 end of central directory locator
+ b.uint32(directory64LocSignature)
+ b.uint32(0) // number of the disk with the start of the zip64 end of central directory
+ b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record
+ b.uint32(1) // total number of disks
+
+ if _, err := w.cw.Write(buf[:]); err != nil {
+ return err
+ }
+
+ // store max values in the regular end record to signal that
+ // that the zip64 values should be used instead
+ records = uint16max
+ size = uint32max
+ offset = uint32max
+ }
+
// write end record
var buf [directoryEndLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(directoryEndSignature))
- b = b[4:] // skip over disk number and first disk number (2x uint16)
- b.uint16(uint16(len(w.dir))) // number of entries this disk
- b.uint16(uint16(len(w.dir))) // number of entries total
- b.uint32(uint32(end - start)) // size of directory
- b.uint32(uint32(start)) // start of directory
+ b = b[4:] // skip over disk number and first disk number (2x uint16)
+ b.uint16(uint16(records)) // number of entries this disk
+ b.uint16(uint16(records)) // number of entries total
+ b.uint32(uint32(size)) // size of directory
+ b.uint32(uint32(offset)) // start of directory
// skipped size of comment (always zero)
if _, err := w.cw.Write(buf[:]); err != nil {
return err
@@ -115,7 +174,7 @@ func (w *Writer) Create(name string) (io.Writer, error) {
}
// CreateHeader adds a file to the zip file using the provided FileHeader
-// for the file metadata.
+// for the file metadata.
// It returns a Writer to which the file contents should be written.
// The file's contents must be written to the io.Writer before the next
// call to Create, CreateHeader, or Close.
@@ -127,8 +186,9 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
}
fh.Flags |= 0x8 // we will write a data descriptor
- fh.CreatorVersion = fh.CreatorVersion&0xff00 | 0x14
- fh.ReaderVersion = 0x14
+
+ fh.CreatorVersion = fh.CreatorVersion&0xff00 | zipVersion20 // preserve compatibility byte
+ fh.ReaderVersion = zipVersion20
fw := &fileWriter{
zipw: w.cw,
@@ -151,7 +211,7 @@ func (w *Writer) CreateHeader(fh *FileHeader) (io.Writer, error) {
h := &header{
FileHeader: fh,
- offset: uint32(w.cw.count),
+ offset: uint64(w.cw.count),
}
w.dir = append(w.dir, h)
fw.header = h
@@ -173,9 +233,9 @@ func writeHeader(w io.Writer, h *FileHeader) error {
b.uint16(h.Method)
b.uint16(h.ModifiedTime)
b.uint16(h.ModifiedDate)
- b.uint32(h.CRC32)
- b.uint32(h.CompressedSize)
- b.uint32(h.UncompressedSize)
+ b.uint32(0) // since we are writing a data descriptor crc32,
+ b.uint32(0) // compressed size,
+ b.uint32(0) // and uncompressed size should be zero
b.uint16(uint16(len(h.Name)))
b.uint16(uint16(len(h.Extra)))
if _, err := w.Write(buf[:]); err != nil {
@@ -218,17 +278,40 @@ func (w *fileWriter) close() error {
// update FileHeader
fh := w.header.FileHeader
fh.CRC32 = w.crc32.Sum32()
- fh.CompressedSize = uint32(w.compCount.count)
- fh.UncompressedSize = uint32(w.rawCount.count)
+ fh.CompressedSize64 = uint64(w.compCount.count)
+ fh.UncompressedSize64 = uint64(w.rawCount.count)
- // write data descriptor
- var buf [dataDescriptorLen]byte
- b := writeBuf(buf[:])
+ if fh.isZip64() {
+ fh.CompressedSize = uint32max
+ fh.UncompressedSize = uint32max
+ fh.ReaderVersion = zipVersion45 // requires 4.5 - File uses ZIP64 format extensions
+ } else {
+ fh.CompressedSize = uint32(fh.CompressedSize64)
+ fh.UncompressedSize = uint32(fh.UncompressedSize64)
+ }
+
+ // Write data descriptor. This is more complicated than one would
+ // think, see e.g. comments in zipfile.c:putextended() and
+ // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=7073588.
+ // The approach here is to write 8 byte sizes if needed without
+ // adding a zip64 extra in the local header (too late anyway).
+ var buf []byte
+ if fh.isZip64() {
+ buf = make([]byte, dataDescriptor64Len)
+ } else {
+ buf = make([]byte, dataDescriptorLen)
+ }
+ b := writeBuf(buf)
b.uint32(dataDescriptorSignature) // de-facto standard, required by OS X
b.uint32(fh.CRC32)
- b.uint32(fh.CompressedSize)
- b.uint32(fh.UncompressedSize)
- _, err := w.zipw.Write(buf[:])
+ if fh.isZip64() {
+ b.uint64(fh.CompressedSize64)
+ b.uint64(fh.UncompressedSize64)
+ } else {
+ b.uint32(fh.CompressedSize)
+ b.uint32(fh.UncompressedSize)
+ }
+ _, err := w.zipw.Write(buf)
return err
}
@@ -262,3 +345,8 @@ func (b *writeBuf) uint32(v uint32) {
binary.LittleEndian.PutUint32(*b, v)
*b = (*b)[4:]
}
+
+func (b *writeBuf) uint64(v uint64) {
+ binary.LittleEndian.PutUint64(*b, v)
+ *b = (*b)[8:]
+}
diff --git a/src/pkg/archive/zip/zip_test.go b/src/pkg/archive/zip/zip_test.go
index d6490c4cb..a8af206a8 100644
--- a/src/pkg/archive/zip/zip_test.go
+++ b/src/pkg/archive/zip/zip_test.go
@@ -9,7 +9,8 @@ package zip
import (
"bytes"
"fmt"
- "reflect"
+ "io"
+ "io/ioutil"
"strings"
"testing"
"time"
@@ -17,8 +18,7 @@ import (
func TestOver65kFiles(t *testing.T) {
if testing.Short() {
- t.Logf("slow test; skipping")
- return
+ t.Skip("slow test; skipping")
}
buf := new(bytes.Buffer)
w := NewWriter(buf)
@@ -58,6 +58,33 @@ func TestModTime(t *testing.T) {
}
}
+func testHeaderRoundTrip(fh *FileHeader, wantUncompressedSize uint32, wantUncompressedSize64 uint64, t *testing.T) {
+ fi := fh.FileInfo()
+ fh2, err := FileInfoHeader(fi)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, want := fh2.Name, fh.Name; got != want {
+ t.Errorf("Name: got %s, want %s\n", got, want)
+ }
+ if got, want := fh2.UncompressedSize, wantUncompressedSize; got != want {
+ t.Errorf("UncompressedSize: got %d, want %d\n", got, want)
+ }
+ if got, want := fh2.UncompressedSize64, wantUncompressedSize64; got != want {
+ t.Errorf("UncompressedSize64: got %d, want %d\n", got, want)
+ }
+ if got, want := fh2.ModifiedTime, fh.ModifiedTime; got != want {
+ t.Errorf("ModifiedTime: got %d, want %d\n", got, want)
+ }
+ if got, want := fh2.ModifiedDate, fh.ModifiedDate; got != want {
+ t.Errorf("ModifiedDate: got %d, want %d\n", got, want)
+ }
+
+ if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh {
+ t.Errorf("Sys didn't return original *FileHeader")
+ }
+}
+
func TestFileHeaderRoundTrip(t *testing.T) {
fh := &FileHeader{
Name: "foo.txt",
@@ -65,17 +92,164 @@ func TestFileHeaderRoundTrip(t *testing.T) {
ModifiedTime: 1234,
ModifiedDate: 5678,
}
- fi := fh.FileInfo()
- fh2, err := FileInfoHeader(fi)
+ testHeaderRoundTrip(fh, fh.UncompressedSize, uint64(fh.UncompressedSize), t)
+}
- // Ignore these fields:
- fh2.CreatorVersion = 0
- fh2.ExternalAttrs = 0
+func TestFileHeaderRoundTrip64(t *testing.T) {
+ fh := &FileHeader{
+ Name: "foo.txt",
+ UncompressedSize64: 9876543210,
+ ModifiedTime: 1234,
+ ModifiedDate: 5678,
+ }
+ testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t)
+}
- if !reflect.DeepEqual(fh, fh2) {
- t.Errorf("mismatch\n input=%#v\noutput=%#v\nerr=%v", fh, fh2, err)
+func TestZip64(t *testing.T) {
+ if testing.Short() {
+ t.Skip("slow test; skipping")
}
- if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh {
- t.Errorf("Sys didn't return original *FileHeader")
+ // write 2^32 bytes plus "END\n" to a zip file
+ buf := new(bytes.Buffer)
+ w := NewWriter(buf)
+ f, err := w.Create("huge.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunk := make([]byte, 1024)
+ for i := range chunk {
+ chunk[i] = '.'
+ }
+ chunk[len(chunk)-1] = '\n'
+ end := []byte("END\n")
+ for i := 0; i < (1<<32)/1024; i++ {
+ _, err := f.Write(chunk)
+ if err != nil {
+ t.Fatal("write chunk:", err)
+ }
+ }
+ _, err = f.Write(end)
+ if err != nil {
+ t.Fatal("write end:", err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ // read back zip file and check that we get to the end of it
+ r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
+ if err != nil {
+ t.Fatal("reader:", err)
+ }
+ f0 := r.File[0]
+ rc, err := f0.Open()
+ if err != nil {
+ t.Fatal("opening:", err)
+ }
+ for i := 0; i < (1<<32)/1024; i++ {
+ _, err := io.ReadFull(rc, chunk)
+ if err != nil {
+ t.Fatal("read:", err)
+ }
+ }
+ gotEnd, err := ioutil.ReadAll(rc)
+ if err != nil {
+ t.Fatal("read end:", err)
+ }
+ if !bytes.Equal(gotEnd, end) {
+ t.Errorf("End of zip64 archive %q, want %q", gotEnd, end)
+ }
+ err = rc.Close()
+ if err != nil {
+ t.Fatal("closing:", err)
+ }
+ if got, want := f0.UncompressedSize, uint32(uint32max); got != want {
+ t.Errorf("UncompressedSize %d, want %d", got, want)
+ }
+
+ if got, want := f0.UncompressedSize64, (1<<32)+uint64(len(end)); got != want {
+ t.Errorf("UncompressedSize64 %d, want %d", got, want)
+ }
+}
+
+func testInvalidHeader(h *FileHeader, t *testing.T) {
+ var buf bytes.Buffer
+ z := NewWriter(&buf)
+
+ f, err := z.CreateHeader(h)
+ if err != nil {
+ t.Fatalf("error creating header: %v", err)
+ }
+ if _, err := f.Write([]byte("hi")); err != nil {
+ t.Fatalf("error writing content: %v", err)
+ }
+ if err := z.Close(); err != nil {
+ t.Fatalf("error closing zip writer: %v", err)
+ }
+
+ b := buf.Bytes()
+ if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != ErrFormat {
+ t.Fatalf("got %v, expected ErrFormat", err)
+ }
+}
+
+func testValidHeader(h *FileHeader, t *testing.T) {
+ var buf bytes.Buffer
+ z := NewWriter(&buf)
+
+ f, err := z.CreateHeader(h)
+ if err != nil {
+ t.Fatalf("error creating header: %v", err)
+ }
+ if _, err := f.Write([]byte("hi")); err != nil {
+ t.Fatalf("error writing content: %v", err)
+ }
+ if err := z.Close(); err != nil {
+ t.Fatalf("error closing zip writer: %v", err)
+ }
+
+ b := buf.Bytes()
+ if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil {
+ t.Fatalf("got %v, expected nil", err)
+ }
+}
+
+// Issue 4302.
+func TestHeaderInvalidTagAndSize(t *testing.T) {
+ const timeFormat = "20060102T150405.000.txt"
+
+ ts := time.Now()
+ filename := ts.Format(timeFormat)
+
+ h := FileHeader{
+ Name: filename,
+ Method: Deflate,
+ Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len
+ }
+ h.SetModTime(ts)
+
+ testInvalidHeader(&h, t)
+}
+
+func TestHeaderTooShort(t *testing.T) {
+ h := FileHeader{
+ Name: "foo.txt",
+ Method: Deflate,
+ Extra: []byte{zip64ExtraId}, // missing size
+ }
+ testInvalidHeader(&h, t)
+}
+
+// Issue 4393. It is valid to have an extra data header
+// which contains no body.
+func TestZeroLengthHeader(t *testing.T) {
+ h := FileHeader{
+ Name: "extadata.txt",
+ Method: Deflate,
+ Extra: []byte{
+ 85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
+ 85, 120, 0, 0, // tag 30805 size 0
+ },
}
+ testValidHeader(&h, t)
}