format.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package tar
     6  
     7  import "strings"
     8  
     9  // Format represents the tar archive format.
    10  //
    11  // The original tar format was introduced in Unix V7.
    12  // Since then, there have been multiple competing formats attempting to
    13  // standardize or extend the V7 format to overcome its limitations.
    14  // The most common formats are the USTAR, PAX, and GNU formats,
    15  // each with their own advantages and limitations.
    16  //
    17  // The following table captures the capabilities of each format:
    18  //
    19  //	                  |  USTAR |       PAX |       GNU
    20  //	------------------+--------+-----------+----------
    21  //	Name              |   256B | unlimited | unlimited
    22  //	Linkname          |   100B | unlimited | unlimited
    23  //	Size              | uint33 | unlimited |    uint89
    24  //	Mode              | uint21 |    uint21 |    uint57
    25  //	Uid/Gid           | uint21 | unlimited |    uint57
    26  //	Uname/Gname       |    32B | unlimited |       32B
    27  //	ModTime           | uint33 | unlimited |     int89
    28  //	AccessTime        |    n/a | unlimited |     int89
    29  //	ChangeTime        |    n/a | unlimited |     int89
    30  //	Devmajor/Devminor | uint21 |    uint21 |    uint57
    31  //	------------------+--------+-----------+----------
    32  //	string encoding   |  ASCII |     UTF-8 |    binary
    33  //	sub-second times  |     no |       yes |        no
    34  //	sparse files      |     no |       yes |       yes
    35  //
    36  // The table's upper portion shows the [Header] fields, where each format reports
    37  // the maximum number of bytes allowed for each string field and
    38  // the integer type used to store each numeric field
    39  // (where timestamps are stored as the number of seconds since the Unix epoch).
    40  //
    41  // The table's lower portion shows specialized features of each format,
    42  // such as supported string encodings, support for sub-second timestamps,
    43  // or support for sparse files.
    44  //
    45  // The Writer currently provides no support for sparse files.
    46  type Format int
    47  
    48  // Constants to identify various tar formats.
    49  const (
    50  	// Deliberately hide the meaning of constants from public API.
    51  	_ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc...
    52  
    53  	// FormatUnknown indicates that the format is unknown.
    54  	FormatUnknown
    55  
    56  	// The format of the original Unix V7 tar tool prior to standardization.
    57  	formatV7
    58  
    59  	// FormatUSTAR represents the USTAR header format defined in POSIX.1-1988.
    60  	//
    61  	// While this format is compatible with most tar readers,
    62  	// the format has several limitations making it unsuitable for some usages.
    63  	// Most notably, it cannot support sparse files, files larger than 8GiB,
    64  	// filenames larger than 256 characters, and non-ASCII filenames.
    65  	//
    66  	// Reference:
    67  	//	http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06
    68  	FormatUSTAR
    69  
    70  	// FormatPAX represents the PAX header format defined in POSIX.1-2001.
    71  	//
    72  	// PAX extends USTAR by writing a special file with Typeflag TypeXHeader
    73  	// preceding the original header. This file contains a set of key-value
    74  	// records, which are used to overcome USTAR's shortcomings, in addition to
    75  	// providing the ability to have sub-second resolution for timestamps.
    76  	//
    77  	// Some newer formats add their own extensions to PAX by defining their
    78  	// own keys and assigning certain semantic meaning to the associated values.
    79  	// For example, sparse file support in PAX is implemented using keys
    80  	// defined by the GNU manual (e.g., "GNU.sparse.map").
    81  	//
    82  	// Reference:
    83  	//	http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html
    84  	FormatPAX
    85  
    86  	// FormatGNU represents the GNU header format.
    87  	//
    88  	// The GNU header format is older than the USTAR and PAX standards and
    89  	// is not compatible with them. The GNU format supports
    90  	// arbitrary file sizes, filenames of arbitrary encoding and length,
    91  	// sparse files, and other features.
    92  	//
    93  	// It is recommended that PAX be chosen over GNU unless the target
    94  	// application can only parse GNU formatted archives.
    95  	//
    96  	// Reference:
    97  	//	https://www.gnu.org/software/tar/manual/html_node/Standard.html
    98  	FormatGNU
    99  
   100  	// Schily's tar format, which is incompatible with USTAR.
   101  	// This does not cover STAR extensions to the PAX format; these fall under
   102  	// the PAX format.
   103  	formatSTAR
   104  
   105  	formatMax
   106  )
   107  
   108  func (f Format) has(f2 Format) bool   { return f&f2 != 0 }
   109  func (f *Format) mayBe(f2 Format)     { *f |= f2 }
   110  func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 }
   111  func (f *Format) mustNotBe(f2 Format) { *f &^= f2 }
   112  
   113  var formatNames = map[Format]string{
   114  	formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR",
   115  }
   116  
   117  func (f Format) String() string {
   118  	var ss []string
   119  	for f2 := Format(1); f2 < formatMax; f2 <<= 1 {
   120  		if f.has(f2) {
   121  			ss = append(ss, formatNames[f2])
   122  		}
   123  	}
   124  	switch len(ss) {
   125  	case 0:
   126  		return "<unknown>"
   127  	case 1:
   128  		return ss[0]
   129  	default:
   130  		return "(" + strings.Join(ss, " | ") + ")"
   131  	}
   132  }
   133  
   134  // Magics used to identify various formats.
   135  const (
   136  	magicGNU, versionGNU     = "ustar ", " \x00"
   137  	magicUSTAR, versionUSTAR = "ustar\x00", "00"
   138  	trailerSTAR              = "tar\x00"
   139  )
   140  
   141  // Size constants from various tar specifications.
   142  const (
   143  	blockSize  = 512 // Size of each block in a tar stream
   144  	nameSize   = 100 // Max length of the name field in USTAR format
   145  	prefixSize = 155 // Max length of the prefix field in USTAR format
   146  
   147  	// Max length of a special file (PAX header, GNU long name or link).
   148  	// This matches the limit used by libarchive.
   149  	maxSpecialFileSize = 1 << 20
   150  
   151  	// Maximum number of sparse file entries.
   152  	// We should never actually hit this limit
   153  	// (every sparse encoding will first be limited by maxSpecialFileSize),
   154  	// but this adds an additional layer of defense.
   155  	maxSparseFileEntries = 1 << 20
   156  )
   157  
   158  // blockPadding computes the number of bytes needed to pad offset up to the
   159  // nearest block edge where 0 <= n < blockSize.
   160  func blockPadding(offset int64) (n int64) {
   161  	return -offset & (blockSize - 1)
   162  }
   163  
   164  var zeroBlock block
   165  
   166  type block [blockSize]byte
   167  
   168  // Convert block to any number of formats.
   169  func (b *block) toV7() *headerV7       { return (*headerV7)(b) }
   170  func (b *block) toGNU() *headerGNU     { return (*headerGNU)(b) }
   171  func (b *block) toSTAR() *headerSTAR   { return (*headerSTAR)(b) }
   172  func (b *block) toUSTAR() *headerUSTAR { return (*headerUSTAR)(b) }
   173  func (b *block) toSparse() sparseArray { return sparseArray(b[:]) }
   174  
   175  // getFormat checks that the block is a valid tar header based on the checksum.
   176  // It then attempts to guess the specific format based on magic values.
   177  // If the checksum fails, then FormatUnknown is returned.
   178  func (b *block) getFormat() Format {
   179  	// Verify checksum.
   180  	var p parser
   181  	value := p.parseOctal(b.toV7().chksum())
   182  	chksum1, chksum2 := b.computeChecksum()
   183  	if p.err != nil || (value != chksum1 && value != chksum2) {
   184  		return FormatUnknown
   185  	}
   186  
   187  	// Guess the magic values.
   188  	magic := string(b.toUSTAR().magic())
   189  	version := string(b.toUSTAR().version())
   190  	trailer := string(b.toSTAR().trailer())
   191  	switch {
   192  	case magic == magicUSTAR && trailer == trailerSTAR:
   193  		return formatSTAR
   194  	case magic == magicUSTAR:
   195  		return FormatUSTAR | FormatPAX
   196  	case magic == magicGNU && version == versionGNU:
   197  		return FormatGNU
   198  	default:
   199  		return formatV7
   200  	}
   201  }
   202  
   203  // setFormat writes the magic values necessary for specified format
   204  // and then updates the checksum accordingly.
   205  func (b *block) setFormat(format Format) {
   206  	// Set the magic values.
   207  	switch {
   208  	case format.has(formatV7):
   209  		// Do nothing.
   210  	case format.has(FormatGNU):
   211  		copy(b.toGNU().magic(), magicGNU)
   212  		copy(b.toGNU().version(), versionGNU)
   213  	case format.has(formatSTAR):
   214  		copy(b.toSTAR().magic(), magicUSTAR)
   215  		copy(b.toSTAR().version(), versionUSTAR)
   216  		copy(b.toSTAR().trailer(), trailerSTAR)
   217  	case format.has(FormatUSTAR | FormatPAX):
   218  		copy(b.toUSTAR().magic(), magicUSTAR)
   219  		copy(b.toUSTAR().version(), versionUSTAR)
   220  	default:
   221  		panic("invalid format")
   222  	}
   223  
   224  	// Update checksum.
   225  	// This field is special in that it is terminated by a NULL then space.
   226  	var f formatter
   227  	field := b.toV7().chksum()
   228  	chksum, _ := b.computeChecksum() // Possible values are 256..128776
   229  	f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143
   230  	field[7] = ' '
   231  }
   232  
   233  // computeChecksum computes the checksum for the header block.
   234  // POSIX specifies a sum of the unsigned byte values, but the Sun tar used
   235  // signed byte values.
   236  // We compute and return both.
   237  func (b *block) computeChecksum() (unsigned, signed int64) {
   238  	for i, c := range b {
   239  		if 148 <= i && i < 156 {
   240  			c = ' ' // Treat the checksum field itself as all spaces.
   241  		}
   242  		unsigned += int64(c)
   243  		signed += int64(int8(c))
   244  	}
   245  	return unsigned, signed
   246  }
   247  
   248  // reset clears the block with all zeros.
   249  func (b *block) reset() {
   250  	*b = block{}
   251  }
   252  
   253  type headerV7 [blockSize]byte
   254  
   255  func (h *headerV7) name() []byte     { return h[000:][:100] }
   256  func (h *headerV7) mode() []byte     { return h[100:][:8] }
   257  func (h *headerV7) uid() []byte      { return h[108:][:8] }
   258  func (h *headerV7) gid() []byte      { return h[116:][:8] }
   259  func (h *headerV7) size() []byte     { return h[124:][:12] }
   260  func (h *headerV7) modTime() []byte  { return h[136:][:12] }
   261  func (h *headerV7) chksum() []byte   { return h[148:][:8] }
   262  func (h *headerV7) typeFlag() []byte { return h[156:][:1] }
   263  func (h *headerV7) linkName() []byte { return h[157:][:100] }
   264  
   265  type headerGNU [blockSize]byte
   266  
   267  func (h *headerGNU) v7() *headerV7       { return (*headerV7)(h) }
   268  func (h *headerGNU) magic() []byte       { return h[257:][:6] }
   269  func (h *headerGNU) version() []byte     { return h[263:][:2] }
   270  func (h *headerGNU) userName() []byte    { return h[265:][:32] }
   271  func (h *headerGNU) groupName() []byte   { return h[297:][:32] }
   272  func (h *headerGNU) devMajor() []byte    { return h[329:][:8] }
   273  func (h *headerGNU) devMinor() []byte    { return h[337:][:8] }
   274  func (h *headerGNU) accessTime() []byte  { return h[345:][:12] }
   275  func (h *headerGNU) changeTime() []byte  { return h[357:][:12] }
   276  func (h *headerGNU) sparse() sparseArray { return sparseArray(h[386:][:24*4+1]) }
   277  func (h *headerGNU) realSize() []byte    { return h[483:][:12] }
   278  
   279  type headerSTAR [blockSize]byte
   280  
   281  func (h *headerSTAR) v7() *headerV7      { return (*headerV7)(h) }
   282  func (h *headerSTAR) magic() []byte      { return h[257:][:6] }
   283  func (h *headerSTAR) version() []byte    { return h[263:][:2] }
   284  func (h *headerSTAR) userName() []byte   { return h[265:][:32] }
   285  func (h *headerSTAR) groupName() []byte  { return h[297:][:32] }
   286  func (h *headerSTAR) devMajor() []byte   { return h[329:][:8] }
   287  func (h *headerSTAR) devMinor() []byte   { return h[337:][:8] }
   288  func (h *headerSTAR) prefix() []byte     { return h[345:][:131] }
   289  func (h *headerSTAR) accessTime() []byte { return h[476:][:12] }
   290  func (h *headerSTAR) changeTime() []byte { return h[488:][:12] }
   291  func (h *headerSTAR) trailer() []byte    { return h[508:][:4] }
   292  
   293  type headerUSTAR [blockSize]byte
   294  
   295  func (h *headerUSTAR) v7() *headerV7     { return (*headerV7)(h) }
   296  func (h *headerUSTAR) magic() []byte     { return h[257:][:6] }
   297  func (h *headerUSTAR) version() []byte   { return h[263:][:2] }
   298  func (h *headerUSTAR) userName() []byte  { return h[265:][:32] }
   299  func (h *headerUSTAR) groupName() []byte { return h[297:][:32] }
   300  func (h *headerUSTAR) devMajor() []byte  { return h[329:][:8] }
   301  func (h *headerUSTAR) devMinor() []byte  { return h[337:][:8] }
   302  func (h *headerUSTAR) prefix() []byte    { return h[345:][:155] }
   303  
   304  type sparseArray []byte
   305  
   306  func (s sparseArray) entry(i int) sparseElem { return sparseElem(s[i*24:]) }
   307  func (s sparseArray) isExtended() []byte     { return s[24*s.maxEntries():][:1] }
   308  func (s sparseArray) maxEntries() int        { return len(s) / 24 }
   309  
   310  type sparseElem []byte
   311  
   312  func (s sparseElem) offset() []byte { return s[00:][:12] }
   313  func (s sparseElem) length() []byte { return s[12:][:12] }
   314
View as plain text