github.com/noisysockets/netstack@v0.6.0/pkg/state/statefile/statefile.go (about) 1 // Copyright 2018 The gVisor Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package statefile defines the state file data stream. 16 // 17 // This package currently does not include any details regarding the state 18 // encoding itself, only details regarding state metadata and data layout. 19 // 20 // The file format is defined as follows. 21 // 22 // /------------------------------------------------------\ 23 // | header (8-bytes) | 24 // +------------------------------------------------------+ 25 // | metadata length (8-bytes) | 26 // +------------------------------------------------------+ 27 // | metadata | 28 // +------------------------------------------------------+ 29 // | data | 30 // \------------------------------------------------------/ 31 // 32 // First, it includes a 8-byte magic header which is the following 33 // sequence of bytes [0x67, 0x56, 0x69, 0x73, 0x6f, 0x72, 0x53, 0x46] 34 // 35 // This header is followed by an 8-byte length N (big endian), and an 36 // ASCII-encoded JSON map that is exactly N bytes long. 37 // 38 // This map includes only strings for keys and strings for values. Keys in the 39 // map that begin with "_" are for internal use only. They may be read, but may 40 // not be provided by the user. In the future, this metadata may contain some 41 // information relating to the state encoding itself. 42 // 43 // After the map, the remainder of the file is the state data. 44 package statefile 45 46 import ( 47 "bytes" 48 "compress/flate" 49 "crypto/hmac" 50 "crypto/sha256" 51 "encoding/binary" 52 "encoding/json" 53 "fmt" 54 "hash" 55 "io" 56 "strings" 57 "time" 58 59 "github.com/noisysockets/netstack/pkg/compressio" 60 ) 61 62 // keySize is the AES-256 key length. 63 const keySize = 32 64 65 // compressionChunkSize is the chunk size for compression. 66 const compressionChunkSize = 1024 * 1024 67 68 // maxMetadataSize is the size limit of metadata section. 69 const maxMetadataSize = 16 * 1024 * 1024 70 71 // magicHeader is the byte sequence beginning each file. 72 var magicHeader = []byte("\x67\x56\x69\x73\x6f\x72\x53\x46") 73 74 // ErrBadMagic is returned if the header does not match. 75 var ErrBadMagic = fmt.Errorf("bad magic header") 76 77 // ErrMetadataMissing is returned if the state file is missing mandatory metadata. 78 var ErrMetadataMissing = fmt.Errorf("missing metadata") 79 80 // ErrInvalidMetadataLength is returned if the metadata length is too large. 81 var ErrInvalidMetadataLength = fmt.Errorf("metadata length invalid, maximum size is %d", maxMetadataSize) 82 83 // ErrMetadataInvalid is returned if passed metadata is invalid. 84 var ErrMetadataInvalid = fmt.Errorf("metadata invalid, can't start with _") 85 86 // ErrInvalidFlags is returned if passed flags set is invalid. 87 var ErrInvalidFlags = fmt.Errorf("flags set is invalid") 88 89 const ( 90 compressionKey = "compression" 91 ) 92 93 // CompressionLevel is the image compression level. 94 type CompressionLevel string 95 96 const ( 97 // CompressionLevelFlateBestSpeed represents flate algorithm in best-speed mode. 98 CompressionLevelFlateBestSpeed = CompressionLevel("flate-best-speed") 99 // CompressionLevelNone represents the absence of any compression on an image. 100 CompressionLevelNone = CompressionLevel("none") 101 // CompressionLevelDefault represents the default compression level. 102 CompressionLevelDefault = CompressionLevelFlateBestSpeed 103 ) 104 105 // Options is statefile options. 106 type Options struct { 107 // Compression is an image compression type/level. 108 Compression CompressionLevel 109 110 // Resume indicates if the sandbox process should continue running 111 // after checkpointing. 112 Resume bool 113 } 114 115 // WriteToMetadata save options to the metadata storage. Method returns the 116 // reference to the original metadata map to allow to be used in the chain calls. 117 func (o Options) WriteToMetadata(metadata map[string]string) map[string]string { 118 metadata[compressionKey] = string(o.Compression) 119 return metadata 120 } 121 122 // CompressionLevelFromString parses a string into the CompressionLevel. 123 func CompressionLevelFromString(val string) (CompressionLevel, error) { 124 switch val { 125 case string(CompressionLevelFlateBestSpeed): 126 return CompressionLevelFlateBestSpeed, nil 127 case string(CompressionLevelNone): 128 return CompressionLevelNone, nil 129 default: 130 return CompressionLevelNone, ErrInvalidFlags 131 } 132 } 133 134 // CompressionLevelFromMetadata returns image compression type stored in the metadata. 135 // If the metadata doesn't contain compression information the default behavior 136 // is the "flate-best-speed" state because the default behavior used to be to always 137 // compress. If the parameter is missing it will be set to default. 138 func CompressionLevelFromMetadata(metadata map[string]string) (CompressionLevel, error) { 139 var err error 140 141 compression := CompressionLevelFlateBestSpeed 142 143 if val, ok := metadata[compressionKey]; ok { 144 if compression, err = CompressionLevelFromString(val); err != nil { 145 return CompressionLevelNone, err 146 } 147 } else { 148 metadata[compressionKey] = string(compression) 149 } 150 151 return compression, nil 152 } 153 154 func writeMetadataLen(w io.Writer, val uint64) error { 155 var buf [8]byte 156 binary.BigEndian.PutUint64(buf[:], val) 157 _, err := w.Write(buf[:]) 158 return err 159 } 160 161 // NewWriter returns a state data writer for a statefile. 162 // 163 // Note that the returned WriteCloser must be closed. 164 func NewWriter(w io.Writer, key []byte, metadata map[string]string) (io.WriteCloser, error) { 165 if metadata == nil { 166 metadata = make(map[string]string) 167 } 168 for k := range metadata { 169 if strings.HasPrefix(k, "_") { 170 return nil, ErrMetadataInvalid 171 } 172 } 173 174 // Create our HMAC function. 175 h := hmac.New(sha256.New, key) 176 mw := io.MultiWriter(w, h) 177 178 // First, write the header. 179 if _, err := mw.Write(magicHeader); err != nil { 180 return nil, err 181 } 182 183 // Generate a timestamp, for convenience only. 184 metadata["_timestamp"] = time.Now().UTC().String() 185 defer delete(metadata, "_timestamp") 186 187 // Save compression state 188 compression, err := CompressionLevelFromMetadata(metadata) 189 if err != nil { 190 return nil, err 191 } 192 193 // Write the metadata. 194 b, err := json.Marshal(metadata) 195 if err != nil { 196 return nil, err 197 } 198 199 if len(b) > maxMetadataSize { 200 return nil, ErrInvalidMetadataLength 201 } 202 203 // Metadata length. 204 if err := writeMetadataLen(mw, uint64(len(b))); err != nil { 205 return nil, err 206 } 207 // Metadata bytes; io.MultiWriter will return a short write error if 208 // any of the writers returns < n. 209 if _, err := mw.Write(b); err != nil { 210 return nil, err 211 } 212 // Write the current hash. 213 cur := h.Sum(nil) 214 for done := 0; done < len(cur); { 215 n, err := mw.Write(cur[done:]) 216 done += n 217 if err != nil { 218 return nil, err 219 } 220 } 221 222 // Wrap in compression. When using "best compression" mode, there is usually 223 // only a little gain in file size reduction, which translate to even smaller 224 // gain in restore latency reduction, while inccuring much more CPU usage at 225 // save time. 226 if compression == CompressionLevelFlateBestSpeed { 227 return compressio.NewWriter(w, key, compressionChunkSize, flate.BestSpeed) 228 } 229 230 return compressio.NewSimpleWriter(w, key) 231 } 232 233 // MetadataUnsafe reads out the metadata from a state file without verifying any 234 // HMAC. This function shouldn't be called for untrusted input files. 235 func MetadataUnsafe(r io.Reader) (map[string]string, error) { 236 return metadata(r, nil) 237 } 238 239 func readMetadataLen(r io.Reader) (uint64, error) { 240 var buf [8]byte 241 if _, err := io.ReadFull(r, buf[:]); err != nil { 242 return 0, err 243 } 244 return binary.BigEndian.Uint64(buf[:]), nil 245 } 246 247 // metadata validates the magic header and reads out the metadata from a state 248 // data stream. 249 func metadata(r io.Reader, h hash.Hash) (map[string]string, error) { 250 if h != nil { 251 r = io.TeeReader(r, h) 252 } 253 254 // Read and validate magic header. 255 b := make([]byte, len(magicHeader)) 256 if _, err := r.Read(b); err != nil { 257 return nil, err 258 } 259 if !bytes.Equal(b, magicHeader) { 260 return nil, ErrBadMagic 261 } 262 263 // Read and validate metadata. 264 b, err := func() (b []byte, err error) { 265 defer func() { 266 if r := recover(); r != nil { 267 b = nil 268 err = fmt.Errorf("%v", r) 269 } 270 }() 271 272 metadataLen, err := readMetadataLen(r) 273 if err != nil { 274 return nil, err 275 } 276 if metadataLen > maxMetadataSize { 277 return nil, ErrInvalidMetadataLength 278 } 279 b = make([]byte, int(metadataLen)) 280 if _, err := io.ReadFull(r, b); err != nil { 281 return nil, err 282 } 283 return b, nil 284 }() 285 if err != nil { 286 return nil, err 287 } 288 289 if h != nil { 290 // Check the hash prior to decoding. 291 cur := h.Sum(nil) 292 buf := make([]byte, len(cur)) 293 if _, err := io.ReadFull(r, buf); err != nil { 294 return nil, err 295 } 296 if !hmac.Equal(cur, buf) { 297 return nil, compressio.ErrHashMismatch 298 } 299 } 300 301 // Decode the metadata. 302 metadata := make(map[string]string) 303 if err := json.Unmarshal(b, &metadata); err != nil { 304 return nil, err 305 } 306 307 return metadata, nil 308 } 309 310 // NewReader returns a reader for a statefile. 311 func NewReader(r io.Reader, key []byte) (io.Reader, map[string]string, error) { 312 // Read the metadata with the hash. 313 h := hmac.New(sha256.New, key) 314 metadata, err := metadata(r, h) 315 if err != nil { 316 return nil, nil, err 317 } 318 319 // Determine image compression state. If the metadata doesn't contain 320 // compression information the default behavior is the "compressed" state 321 // because the default behavior used to be to always compress. 322 compression, err := CompressionLevelFromMetadata(metadata) 323 if err != nil { 324 return nil, nil, err 325 } 326 327 // Pick correct reader 328 var cr io.Reader 329 330 if compression == CompressionLevelFlateBestSpeed { 331 cr, err = compressio.NewReader(r, key) 332 } else if compression == CompressionLevelNone { 333 cr, err = compressio.NewSimpleReader(r, key) 334 } else { 335 // Should never occur, as it has the default path. 336 return nil, nil, fmt.Errorf("metadata contains invalid compression flag value: %v", compression) 337 } 338 339 if err != nil { 340 return nil, nil, err 341 } 342 343 return cr, metadata, nil 344 }