summaryrefslogtreecommitdiff
path: root/src/pkg/archive/tar/reader.go
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/archive/tar/reader.go')
-rw-r--r--src/pkg/archive/tar/reader.go465
1 files changed, 443 insertions, 22 deletions
diff --git a/src/pkg/archive/tar/reader.go b/src/pkg/archive/tar/reader.go
index b2d62f3c5..920a9b08f 100644
--- a/src/pkg/archive/tar/reader.go
+++ b/src/pkg/archive/tar/reader.go
@@ -29,12 +29,57 @@ const maxNanoSecondIntSize = 9
// The Next method advances to the next file in the archive (including the first),
// and then it can be treated as an io.Reader to access the file's data.
type Reader struct {
- r io.Reader
- err error
- nb int64 // number of unread bytes for current file entry
- pad int64 // amount of padding (ignored) after current file entry
+ r io.Reader
+ err error
+ pad int64 // amount of padding (ignored) after current file entry
+ curr numBytesReader // reader for current file entry
}
+// A numBytesReader is an io.Reader with a numBytes method, returning the number
+// of bytes remaining in the underlying encoded data.
+type numBytesReader interface {
+ io.Reader
+ numBytes() int64
+}
+
+// A regFileReader is a numBytesReader for reading file data from a tar archive.
+type regFileReader struct {
+ r io.Reader // underlying reader
+ nb int64 // number of unread bytes for current file entry
+}
+
+// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive.
+type sparseFileReader struct {
+ rfr *regFileReader // reads the sparse-encoded file data
+ sp []sparseEntry // the sparse map for the file
+ pos int64 // keeps track of file position
+ tot int64 // total size of the file
+}
+
+// Keywords for GNU sparse files in a PAX extended header
+const (
+ paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
+ paxGNUSparseOffset = "GNU.sparse.offset"
+ paxGNUSparseNumBytes = "GNU.sparse.numbytes"
+ paxGNUSparseMap = "GNU.sparse.map"
+ paxGNUSparseName = "GNU.sparse.name"
+ paxGNUSparseMajor = "GNU.sparse.major"
+ paxGNUSparseMinor = "GNU.sparse.minor"
+ paxGNUSparseSize = "GNU.sparse.size"
+ paxGNUSparseRealSize = "GNU.sparse.realsize"
+)
+
+// Keywords for old GNU sparse headers
+const (
+ oldGNUSparseMainHeaderOffset = 386
+ oldGNUSparseMainHeaderIsExtendedOffset = 482
+ oldGNUSparseMainHeaderNumEntries = 4
+ oldGNUSparseExtendedHeaderIsExtendedOffset = 504
+ oldGNUSparseExtendedHeaderNumEntries = 21
+ oldGNUSparseOffsetSize = 12
+ oldGNUSparseNumBytesSize = 12
+)
+
// NewReader creates a new Reader reading from r.
func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
@@ -64,6 +109,18 @@ func (tr *Reader) Next() (*Header, error) {
tr.skipUnread()
hdr = tr.readHeader()
mergePAX(hdr, headers)
+
+ // Check for a PAX format sparse file
+ sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers)
+ if err != nil {
+ tr.err = err
+ return nil, err
+ }
+ if sp != nil {
+ // Current file is a PAX format GNU sparse file.
+ // Set the current file reader to a sparse file reader.
+ tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
+ }
return hdr, nil
case TypeGNULongName:
// We have a GNU long name header. Its contents are the real file name.
@@ -87,6 +144,67 @@ func (tr *Reader) Next() (*Header, error) {
return hdr, tr.err
}
+// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
+// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to
+// be treated as a regular file.
+func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) {
+ var sparseFormat string
+
+ // Check for sparse format indicators
+ major, majorOk := headers[paxGNUSparseMajor]
+ minor, minorOk := headers[paxGNUSparseMinor]
+ sparseName, sparseNameOk := headers[paxGNUSparseName]
+ _, sparseMapOk := headers[paxGNUSparseMap]
+ sparseSize, sparseSizeOk := headers[paxGNUSparseSize]
+ sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize]
+
+ // Identify which, if any, sparse format applies from which PAX headers are set
+ if majorOk && minorOk {
+ sparseFormat = major + "." + minor
+ } else if sparseNameOk && sparseMapOk {
+ sparseFormat = "0.1"
+ } else if sparseSizeOk {
+ sparseFormat = "0.0"
+ } else {
+ // Not a PAX format GNU sparse file.
+ return nil, nil
+ }
+
+ // Check for unknown sparse format
+ if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" {
+ return nil, nil
+ }
+
+ // Update hdr from GNU sparse PAX headers
+ if sparseNameOk {
+ hdr.Name = sparseName
+ }
+ if sparseSizeOk {
+ realSize, err := strconv.ParseInt(sparseSize, 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+ hdr.Size = realSize
+ } else if sparseRealSizeOk {
+ realSize, err := strconv.ParseInt(sparseRealSize, 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+ hdr.Size = realSize
+ }
+
+ // Set up the sparse map, according to the particular sparse format in use
+ var sp []sparseEntry
+ var err error
+ switch sparseFormat {
+ case "0.0", "0.1":
+ sp, err = readGNUSparseMap0x1(headers)
+ case "1.0":
+ sp, err = readGNUSparseMap1x0(tr.curr)
+ }
+ return sp, err
+}
+
// mergePAX merges well known headers according to PAX standard.
// In general headers with the same name as those found
// in the header struct overwrite those found in the header
@@ -139,8 +257,14 @@ func mergePAX(hdr *Header, headers map[string]string) error {
return err
}
hdr.Size = int64(size)
+ default:
+ if strings.HasPrefix(k, paxXattr) {
+ if hdr.Xattrs == nil {
+ hdr.Xattrs = make(map[string]string)
+ }
+ hdr.Xattrs[k[len(paxXattr):]] = v
+ }
}
-
}
return nil
}
@@ -188,6 +312,11 @@ func parsePAX(r io.Reader) (map[string]string, error) {
if err != nil {
return nil, err
}
+
+ // For GNU PAX sparse format 0.0 support.
+ // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
+ var sparseMap bytes.Buffer
+
headers := make(map[string]string)
// Each record is constructed as
// "%d %s=%s\n", length, keyword, value
@@ -205,7 +334,7 @@ func parsePAX(r io.Reader) (map[string]string, error) {
return nil, ErrHeader
}
// Extract everything between the decimal and the n -1 on the
- // beginning to to eat the ' ', -1 on the end to skip the newline.
+ // beginning to eat the ' ', -1 on the end to skip the newline.
var record []byte
record, buf = buf[sp+1:n-1], buf[n:]
// The first equals is guaranteed to mark the end of the key.
@@ -215,7 +344,21 @@ func parsePAX(r io.Reader) (map[string]string, error) {
return nil, ErrHeader
}
key, value := record[:eq], record[eq+1:]
- headers[string(key)] = string(value)
+
+ keyStr := string(key)
+ if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
+ // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
+ sparseMap.Write(value)
+ sparseMap.Write([]byte{','})
+ } else {
+ // Normal key. Set the value in the headers map.
+ headers[keyStr] = string(value)
+ }
+ }
+ if sparseMap.Len() != 0 {
+ // Add sparse info to headers, chopping off the extra comma
+ sparseMap.Truncate(sparseMap.Len() - 1)
+ headers[paxGNUSparseMap] = sparseMap.String()
}
return headers, nil
}
@@ -262,8 +405,8 @@ func (tr *Reader) octal(b []byte) int64 {
// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
func (tr *Reader) skipUnread() {
- nr := tr.nb + tr.pad // number of bytes to skip
- tr.nb, tr.pad = 0, 0
+ nr := tr.numBytes() + tr.pad // number of bytes to skip
+ tr.curr, tr.pad = nil, 0
if sr, ok := tr.r.(io.Seeker); ok {
if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
return
@@ -325,14 +468,14 @@ func (tr *Reader) readHeader() *Header {
// so its magic bytes, like the rest of the block, are NULs.
magic := string(s.next(8)) // contains version field as well.
var format string
- switch magic {
- case "ustar\x0000": // POSIX tar (1003.1-1988)
+ switch {
+ case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
if string(header[508:512]) == "tar\x00" {
format = "star"
} else {
format = "posix"
}
- case "ustar \x00": // old GNU tar
+ case magic == "ustar \x00": // old GNU tar
format = "gnu"
}
@@ -367,30 +510,308 @@ func (tr *Reader) readHeader() *Header {
// Maximum value of hdr.Size is 64 GB (12 octal digits),
// so there's no risk of int64 overflowing.
- tr.nb = int64(hdr.Size)
- tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two
+ nb := int64(hdr.Size)
+ tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
+
+ // Set the current file reader.
+ tr.curr = &regFileReader{r: tr.r, nb: nb}
+
+ // Check for old GNU sparse format entry.
+ if hdr.Typeflag == TypeGNUSparse {
+ // Get the real size of the file.
+ hdr.Size = tr.octal(header[483:495])
+
+ // Read the sparse map.
+ sp := tr.readOldGNUSparseMap(header)
+ if tr.err != nil {
+ return nil
+ }
+ // Current file is a GNU sparse file. Update the current file reader.
+ tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
+ }
return hdr
}
+// A sparseEntry holds a single entry in a sparse file's sparse map.
+// A sparse entry indicates the offset and size in a sparse file of a
+// block of data.
+type sparseEntry struct {
+ offset int64
+ numBytes int64
+}
+
+// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
+// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
+// then one or more extension headers are used to store the rest of the sparse map.
+func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
+ isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
+ spCap := oldGNUSparseMainHeaderNumEntries
+ if isExtended {
+ spCap += oldGNUSparseExtendedHeaderNumEntries
+ }
+ sp := make([]sparseEntry, 0, spCap)
+ s := slicer(header[oldGNUSparseMainHeaderOffset:])
+
+ // Read the four entries from the main tar header
+ for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
+ offset := tr.octal(s.next(oldGNUSparseOffsetSize))
+ numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
+ if tr.err != nil {
+ tr.err = ErrHeader
+ return nil
+ }
+ if offset == 0 && numBytes == 0 {
+ break
+ }
+ sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+ }
+
+ for isExtended {
+ // There are more entries. Read an extension header and parse its entries.
+ sparseHeader := make([]byte, blockSize)
+ if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
+ return nil
+ }
+ isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
+ s = slicer(sparseHeader)
+ for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
+ offset := tr.octal(s.next(oldGNUSparseOffsetSize))
+ numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
+ if tr.err != nil {
+ tr.err = ErrHeader
+ return nil
+ }
+ if offset == 0 && numBytes == 0 {
+ break
+ }
+ sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+ }
+ }
+ return sp
+}
+
+// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0.
+// The sparse map is stored just before the file data and padded out to the nearest block boundary.
+func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
+ buf := make([]byte, 2*blockSize)
+ sparseHeader := buf[:blockSize]
+
+ // readDecimal is a helper function to read a decimal integer from the sparse map
+ // while making sure to read from the file in blocks of size blockSize
+ readDecimal := func() (int64, error) {
+ // Look for newline
+ nl := bytes.IndexByte(sparseHeader, '\n')
+ if nl == -1 {
+ if len(sparseHeader) >= blockSize {
+ // This is an error
+ return 0, ErrHeader
+ }
+ oldLen := len(sparseHeader)
+ newLen := oldLen + blockSize
+ if cap(sparseHeader) < newLen {
+ // There's more header, but we need to make room for the next block
+ copy(buf, sparseHeader)
+ sparseHeader = buf[:newLen]
+ } else {
+ // There's more header, and we can just reslice
+ sparseHeader = sparseHeader[:newLen]
+ }
+
+ // Now that sparseHeader is large enough, read next block
+ if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil {
+ return 0, err
+ }
+
+ // Look for a newline in the new data
+ nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n')
+ if nl == -1 {
+ // This is an error
+ return 0, ErrHeader
+ }
+ nl += oldLen // We want the position from the beginning
+ }
+ // Now that we've found a newline, read a number
+ n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0)
+ if err != nil {
+ return 0, ErrHeader
+ }
+
+ // Update sparseHeader to consume this number
+ sparseHeader = sparseHeader[nl+1:]
+ return n, nil
+ }
+
+ // Read the first block
+ if _, err := io.ReadFull(r, sparseHeader); err != nil {
+ return nil, err
+ }
+
+ // The first line contains the number of entries
+ numEntries, err := readDecimal()
+ if err != nil {
+ return nil, err
+ }
+
+ // Read all the entries
+ sp := make([]sparseEntry, 0, numEntries)
+ for i := int64(0); i < numEntries; i++ {
+ // Read the offset
+ offset, err := readDecimal()
+ if err != nil {
+ return nil, err
+ }
+ // Read numBytes
+ numBytes, err := readDecimal()
+ if err != nil {
+ return nil, err
+ }
+
+ sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+ }
+
+ return sp, nil
+}
+
+// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1.
+// The sparse map is stored in the PAX headers.
+func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) {
+ // Get number of entries
+ numEntriesStr, ok := headers[paxGNUSparseNumBlocks]
+ if !ok {
+ return nil, ErrHeader
+ }
+ numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+
+ sparseMap := strings.Split(headers[paxGNUSparseMap], ",")
+
+ // There should be two numbers in sparseMap for each entry
+ if int64(len(sparseMap)) != 2*numEntries {
+ return nil, ErrHeader
+ }
+
+ // Loop through the entries in the sparse map
+ sp := make([]sparseEntry, 0, numEntries)
+ for i := int64(0); i < numEntries; i++ {
+ offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+ numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0)
+ if err != nil {
+ return nil, ErrHeader
+ }
+ sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
+ }
+
+ return sp, nil
+}
+
+// numBytes returns the number of bytes left to read in the current file's entry
+// in the tar archive, or 0 if there is no current file.
+func (tr *Reader) numBytes() int64 {
+ if tr.curr == nil {
+ // No current file, so no bytes
+ return 0
+ }
+ return tr.curr.numBytes()
+}
+
// Read reads from the current entry in the tar archive.
// It returns 0, io.EOF when it reaches the end of that entry,
// until Next is called to advance to the next entry.
func (tr *Reader) Read(b []byte) (n int, err error) {
- if tr.nb == 0 {
- // file consumed
+ if tr.curr == nil {
return 0, io.EOF
}
+ n, err = tr.curr.Read(b)
+ if err != nil && err != io.EOF {
+ tr.err = err
+ }
+ return
+}
- if int64(len(b)) > tr.nb {
- b = b[0:tr.nb]
+func (rfr *regFileReader) Read(b []byte) (n int, err error) {
+ if rfr.nb == 0 {
+ // file consumed
+ return 0, io.EOF
}
- n, err = tr.r.Read(b)
- tr.nb -= int64(n)
+ if int64(len(b)) > rfr.nb {
+ b = b[0:rfr.nb]
+ }
+ n, err = rfr.r.Read(b)
+ rfr.nb -= int64(n)
- if err == io.EOF && tr.nb > 0 {
+ if err == io.EOF && rfr.nb > 0 {
err = io.ErrUnexpectedEOF
}
- tr.err = err
return
}
+
+// numBytes returns the number of bytes left to read in the file's data in the tar archive.
+func (rfr *regFileReader) numBytes() int64 {
+ return rfr.nb
+}
+
+// readHole reads a sparse file hole ending at offset toOffset
+func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
+ n64 := toOffset - sfr.pos
+ if n64 > int64(len(b)) {
+ n64 = int64(len(b))
+ }
+ n := int(n64)
+ for i := 0; i < n; i++ {
+ b[i] = 0
+ }
+ sfr.pos += n64
+ return n
+}
+
+// Read reads the sparse file data in expanded form.
+func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
+ if len(sfr.sp) == 0 {
+ // No more data fragments to read from.
+ if sfr.pos < sfr.tot {
+ // We're in the last hole
+ n = sfr.readHole(b, sfr.tot)
+ return
+ }
+ // Otherwise, we're at the end of the file
+ return 0, io.EOF
+ }
+ if sfr.pos < sfr.sp[0].offset {
+ // We're in a hole
+ n = sfr.readHole(b, sfr.sp[0].offset)
+ return
+ }
+
+ // We're not in a hole, so we'll read from the next data fragment
+ posInFragment := sfr.pos - sfr.sp[0].offset
+ bytesLeft := sfr.sp[0].numBytes - posInFragment
+ if int64(len(b)) > bytesLeft {
+ b = b[0:bytesLeft]
+ }
+
+ n, err = sfr.rfr.Read(b)
+ sfr.pos += int64(n)
+
+ if int64(n) == bytesLeft {
+ // We're done with this fragment
+ sfr.sp = sfr.sp[1:]
+ }
+
+ if err == io.EOF && sfr.pos < sfr.tot {
+ // We reached the end of the last fragment's data, but there's a final hole
+ err = nil
+ }
+ return
+}
+
+// numBytes returns the number of bytes left to read in the sparse file's
+// sparse-encoded data in the tar archive.
+func (sfr *sparseFileReader) numBytes() int64 {
+ return sfr.rfr.nb
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-23 09:57:22 +0000