github.com/noisysockets/netstack@v0.6.0/pkg/state/statefile/statefile.go (about)

     1  // Copyright 2018 The gVisor Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package statefile defines the state file data stream.
    16  //
    17  // This package currently does not include any details regarding the state
    18  // encoding itself, only details regarding state metadata and data layout.
    19  //
    20  // The file format is defined as follows.
    21  //
    22  // /------------------------------------------------------\
    23  // |                   header (8-bytes)                   |
    24  // +------------------------------------------------------+
    25  // |              metadata length (8-bytes)               |
    26  // +------------------------------------------------------+
    27  // |                       metadata                       |
    28  // +------------------------------------------------------+
    29  // |                         data                         |
    30  // \------------------------------------------------------/
    31  //
    32  // First, it includes a 8-byte magic header which is the following
    33  // sequence of bytes [0x67, 0x56, 0x69, 0x73, 0x6f, 0x72, 0x53, 0x46]
    34  //
    35  // This header is followed by an 8-byte length N (big endian), and an
    36  // ASCII-encoded JSON map that is exactly N bytes long.
    37  //
    38  // This map includes only strings for keys and strings for values. Keys in the
    39  // map that begin with "_" are for internal use only. They may be read, but may
    40  // not be provided by the user. In the future, this metadata may contain some
    41  // information relating to the state encoding itself.
    42  //
    43  // After the map, the remainder of the file is the state data.
    44  package statefile
    45  
    46  import (
    47  	"bytes"
    48  	"compress/flate"
    49  	"crypto/hmac"
    50  	"crypto/sha256"
    51  	"encoding/binary"
    52  	"encoding/json"
    53  	"fmt"
    54  	"hash"
    55  	"io"
    56  	"strings"
    57  	"time"
    58  
    59  	"github.com/noisysockets/netstack/pkg/compressio"
    60  )
    61  
    62  // keySize is the AES-256 key length.
    63  const keySize = 32
    64  
    65  // compressionChunkSize is the chunk size for compression.
    66  const compressionChunkSize = 1024 * 1024
    67  
    68  // maxMetadataSize is the size limit of metadata section.
    69  const maxMetadataSize = 16 * 1024 * 1024
    70  
    71  // magicHeader is the byte sequence beginning each file.
    72  var magicHeader = []byte("\x67\x56\x69\x73\x6f\x72\x53\x46")
    73  
    74  // ErrBadMagic is returned if the header does not match.
    75  var ErrBadMagic = fmt.Errorf("bad magic header")
    76  
    77  // ErrMetadataMissing is returned if the state file is missing mandatory metadata.
    78  var ErrMetadataMissing = fmt.Errorf("missing metadata")
    79  
    80  // ErrInvalidMetadataLength is returned if the metadata length is too large.
    81  var ErrInvalidMetadataLength = fmt.Errorf("metadata length invalid, maximum size is %d", maxMetadataSize)
    82  
    83  // ErrMetadataInvalid is returned if passed metadata is invalid.
    84  var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _")
    85  
    86  // ErrInvalidFlags is returned if passed flags set is invalid.
    87  var ErrInvalidFlags = fmt.Errorf("flags set is invalid")
    88  
    89  const (
    90  	compressionKey = "compression"
    91  )
    92  
    93  // CompressionLevel is the image compression level.
    94  type CompressionLevel string
    95  
    96  const (
    97  	// CompressionLevelFlateBestSpeed represents flate algorithm in best-speed mode.
    98  	CompressionLevelFlateBestSpeed = CompressionLevel("flate-best-speed")
    99  	// CompressionLevelNone represents the absence of any compression on an image.
   100  	CompressionLevelNone = CompressionLevel("none")
   101  	// CompressionLevelDefault represents the default compression level.
   102  	CompressionLevelDefault = CompressionLevelFlateBestSpeed
   103  )
   104  
   105  // Options is statefile options.
   106  type Options struct {
   107  	// Compression is an image compression type/level.
   108  	Compression CompressionLevel
   109  
   110  	// Resume indicates if the sandbox process should continue running
   111  	// after checkpointing.
   112  	Resume bool
   113  }
   114  
   115  // WriteToMetadata save options to the metadata storage.  Method returns the
   116  // reference to the original metadata map to allow to be used in the chain calls.
   117  func (o Options) WriteToMetadata(metadata map[string]string) map[string]string {
   118  	metadata[compressionKey] = string(o.Compression)
   119  	return metadata
   120  }
   121  
   122  // CompressionLevelFromString parses a string into the CompressionLevel.
   123  func CompressionLevelFromString(val string) (CompressionLevel, error) {
   124  	switch val {
   125  	case string(CompressionLevelFlateBestSpeed):
   126  		return CompressionLevelFlateBestSpeed, nil
   127  	case string(CompressionLevelNone):
   128  		return CompressionLevelNone, nil
   129  	default:
   130  		return CompressionLevelNone, ErrInvalidFlags
   131  	}
   132  }
   133  
   134  // CompressionLevelFromMetadata returns image compression type stored in the metadata.
   135  // If the metadata doesn't contain compression information the default behavior
   136  // is the "flate-best-speed" state because the default behavior used to be to always
   137  // compress. If the parameter is missing it will be set to default.
   138  func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) {
   139  	var err error
   140  
   141  	compression := CompressionLevelFlateBestSpeed
   142  
   143  	if val, ok := metadata[compressionKey]; ok {
   144  		if compression, err = CompressionLevelFromString(val); err != nil {
   145  			return CompressionLevelNone, err
   146  		}
   147  	} else {
   148  		metadata[compressionKey] = string(compression)
   149  	}
   150  
   151  	return compression, nil
   152  }
   153  
   154  func writeMetadataLen(w io.Writer, val uint64) error {
   155  	var buf [8]byte
   156  	binary.BigEndian.PutUint64(buf[:], val)
   157  	_, err := w.Write(buf[:])
   158  	return err
   159  }
   160  
   161  // NewWriter returns a state data writer for a statefile.
   162  //
   163  // Note that the returned WriteCloser must be closed.
   164  func NewWriter(w io.Writer, key []byte, metadata map[string]string) (io.WriteCloser, error) {
   165  	if metadata == nil {
   166  		metadata = make(map[string]string)
   167  	}
   168  	for k := range metadata {
   169  		if strings.HasPrefix(k, "_") {
   170  			return nil, ErrMetadataInvalid
   171  		}
   172  	}
   173  
   174  	// Create our HMAC function.
   175  	h := hmac.New(sha256.New, key)
   176  	mw := io.MultiWriter(w, h)
   177  
   178  	// First, write the header.
   179  	if _, err := mw.Write(magicHeader); err != nil {
   180  		return nil, err
   181  	}
   182  
   183  	// Generate a timestamp, for convenience only.
   184  	metadata["_timestamp"] = time.Now().UTC().String()
   185  	defer delete(metadata, "_timestamp")
   186  
   187  	// Save compression state
   188  	compression, err := CompressionLevelFromMetadata(metadata)
   189  	if err != nil {
   190  		return nil, err
   191  	}
   192  
   193  	// Write the metadata.
   194  	b, err := json.Marshal(metadata)
   195  	if err != nil {
   196  		return nil, err
   197  	}
   198  
   199  	if len(b) > maxMetadataSize {
   200  		return nil, ErrInvalidMetadataLength
   201  	}
   202  
   203  	// Metadata length.
   204  	if err := writeMetadataLen(mw, uint64(len(b))); err != nil {
   205  		return nil, err
   206  	}
   207  	// Metadata bytes; io.MultiWriter will return a short write error if
   208  	// any of the writers returns < n.
   209  	if _, err := mw.Write(b); err != nil {
   210  		return nil, err
   211  	}
   212  	// Write the current hash.
   213  	cur := h.Sum(nil)
   214  	for done := 0; done < len(cur); {
   215  		n, err := mw.Write(cur[done:])
   216  		done += n
   217  		if err != nil {
   218  			return nil, err
   219  		}
   220  	}
   221  
   222  	// Wrap in compression. When using "best compression" mode, there is usually
   223  	// only a little gain in file size reduction, which translate to even smaller
   224  	// gain in restore latency reduction, while inccuring much more CPU usage at
   225  	// save time.
   226  	if compression == CompressionLevelFlateBestSpeed {
   227  		return compressio.NewWriter(w, key, compressionChunkSize, flate.BestSpeed)
   228  	}
   229  
   230  	return compressio.NewSimpleWriter(w, key)
   231  }
   232  
   233  // MetadataUnsafe reads out the metadata from a state file without verifying any
   234  // HMAC. This function shouldn't be called for untrusted input files.
   235  func MetadataUnsafe(r io.Reader) (map[string]string, error) {
   236  	return metadata(r, nil)
   237  }
   238  
   239  func readMetadataLen(r io.Reader) (uint64, error) {
   240  	var buf [8]byte
   241  	if _, err := io.ReadFull(r, buf[:]); err != nil {
   242  		return 0, err
   243  	}
   244  	return binary.BigEndian.Uint64(buf[:]), nil
   245  }
   246  
   247  // metadata validates the magic header and reads out the metadata from a state
   248  // data stream.
   249  func metadata(r io.Reader, h hash.Hash) (map[string]string, error) {
   250  	if h != nil {
   251  		r = io.TeeReader(r, h)
   252  	}
   253  
   254  	// Read and validate magic header.
   255  	b := make([]byte, len(magicHeader))
   256  	if _, err := r.Read(b); err != nil {
   257  		return nil, err
   258  	}
   259  	if !bytes.Equal(b, magicHeader) {
   260  		return nil, ErrBadMagic
   261  	}
   262  
   263  	// Read and validate metadata.
   264  	b, err := func() (b []byte, err error) {
   265  		defer func() {
   266  			if r := recover(); r != nil {
   267  				b = nil
   268  				err = fmt.Errorf("%v", r)
   269  			}
   270  		}()
   271  
   272  		metadataLen, err := readMetadataLen(r)
   273  		if err != nil {
   274  			return nil, err
   275  		}
   276  		if metadataLen > maxMetadataSize {
   277  			return nil, ErrInvalidMetadataLength
   278  		}
   279  		b = make([]byte, int(metadataLen))
   280  		if _, err := io.ReadFull(r, b); err != nil {
   281  			return nil, err
   282  		}
   283  		return b, nil
   284  	}()
   285  	if err != nil {
   286  		return nil, err
   287  	}
   288  
   289  	if h != nil {
   290  		// Check the hash prior to decoding.
   291  		cur := h.Sum(nil)
   292  		buf := make([]byte, len(cur))
   293  		if _, err := io.ReadFull(r, buf); err != nil {
   294  			return nil, err
   295  		}
   296  		if !hmac.Equal(cur, buf) {
   297  			return nil, compressio.ErrHashMismatch
   298  		}
   299  	}
   300  
   301  	// Decode the metadata.
   302  	metadata := make(map[string]string)
   303  	if err := json.Unmarshal(b, &metadata); err != nil {
   304  		return nil, err
   305  	}
   306  
   307  	return metadata, nil
   308  }
   309  
   310  // NewReader returns a reader for a statefile.
   311  func NewReader(r io.Reader, key []byte) (io.Reader, map[string]string, error) {
   312  	// Read the metadata with the hash.
   313  	h := hmac.New(sha256.New, key)
   314  	metadata, err := metadata(r, h)
   315  	if err != nil {
   316  		return nil, nil, err
   317  	}
   318  
   319  	// Determine image compression state. If the metadata doesn't contain
   320  	// compression information the default behavior is the "compressed" state
   321  	// because the default behavior used to be to always compress.
   322  	compression, err := CompressionLevelFromMetadata(metadata)
   323  	if err != nil {
   324  		return nil, nil, err
   325  	}
   326  
   327  	// Pick correct reader
   328  	var cr io.Reader
   329  
   330  	if compression == CompressionLevelFlateBestSpeed {
   331  		cr, err = compressio.NewReader(r, key)
   332  	} else if compression == CompressionLevelNone {
   333  		cr, err = compressio.NewSimpleReader(r, key)
   334  	} else {
   335  		// Should never occur, as it has the default path.
   336  		return nil, nil, fmt.Errorf("metadata contains invalid compression flag value: %v", compression)
   337  	}
   338  
   339  	if err != nil {
   340  		return nil, nil, err
   341  	}
   342  
   343  	return cr, metadata, nil
   344  }