github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/image/tarexport/save.go (about) 1 package tarexport // import "github.com/demonoid81/moby/image/tarexport" 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "path" 10 "path/filepath" 11 "runtime" 12 "time" 13 14 "github.com/docker/distribution" 15 "github.com/docker/distribution/reference" 16 "github.com/demonoid81/moby/image" 17 v1 "github.com/demonoid81/moby/image/v1" 18 "github.com/demonoid81/moby/layer" 19 "github.com/demonoid81/moby/pkg/archive" 20 "github.com/demonoid81/moby/pkg/system" 21 digest "github.com/opencontainers/go-digest" 22 "github.com/pkg/errors" 23 ) 24 25 type imageDescriptor struct { 26 refs []reference.NamedTagged 27 layers []string 28 image *image.Image 29 layerRef layer.Layer 30 } 31 32 type saveSession struct { 33 *tarexporter 34 outDir string 35 images map[image.ID]*imageDescriptor 36 savedLayers map[string]struct{} 37 diffIDPaths map[layer.DiffID]string // cache every diffID blob to avoid duplicates 38 } 39 40 func (l *tarexporter) Save(names []string, outStream io.Writer) error { 41 images, err := l.parseNames(names) 42 if err != nil { 43 return err 44 } 45 46 // Release all the image top layer references 47 defer l.releaseLayerReferences(images) 48 return (&saveSession{tarexporter: l, images: images}).save(outStream) 49 } 50 51 // parseNames will parse the image names to a map which contains image.ID to *imageDescriptor. 52 // Each imageDescriptor holds an image top layer reference named 'layerRef'. It is taken here, should be released later. 53 func (l *tarexporter) parseNames(names []string) (desc map[image.ID]*imageDescriptor, rErr error) { 54 imgDescr := make(map[image.ID]*imageDescriptor) 55 defer func() { 56 if rErr != nil { 57 l.releaseLayerReferences(imgDescr) 58 } 59 }() 60 61 addAssoc := func(id image.ID, ref reference.Named) error { 62 if _, ok := imgDescr[id]; !ok { 63 descr := &imageDescriptor{} 64 if err := l.takeLayerReference(id, descr); err != nil { 65 return err 66 } 67 imgDescr[id] = descr 68 } 69 70 if ref != nil { 71 if _, ok := ref.(reference.Canonical); ok { 72 return nil 73 } 74 tagged, ok := reference.TagNameOnly(ref).(reference.NamedTagged) 75 if !ok { 76 return nil 77 } 78 79 for _, t := range imgDescr[id].refs { 80 if tagged.String() == t.String() { 81 return nil 82 } 83 } 84 imgDescr[id].refs = append(imgDescr[id].refs, tagged) 85 } 86 return nil 87 } 88 89 for _, name := range names { 90 ref, err := reference.ParseAnyReference(name) 91 if err != nil { 92 return nil, err 93 } 94 namedRef, ok := ref.(reference.Named) 95 if !ok { 96 // Check if digest ID reference 97 if digested, ok := ref.(reference.Digested); ok { 98 id := image.IDFromDigest(digested.Digest()) 99 if err := addAssoc(id, nil); err != nil { 100 return nil, err 101 } 102 continue 103 } 104 return nil, errors.Errorf("invalid reference: %v", name) 105 } 106 107 if reference.FamiliarName(namedRef) == string(digest.Canonical) { 108 imgID, err := l.is.Search(name) 109 if err != nil { 110 return nil, err 111 } 112 if err := addAssoc(imgID, nil); err != nil { 113 return nil, err 114 } 115 continue 116 } 117 if reference.IsNameOnly(namedRef) { 118 assocs := l.rs.ReferencesByName(namedRef) 119 for _, assoc := range assocs { 120 if err := addAssoc(image.IDFromDigest(assoc.ID), assoc.Ref); err != nil { 121 return nil, err 122 } 123 } 124 if len(assocs) == 0 { 125 imgID, err := l.is.Search(name) 126 if err != nil { 127 return nil, err 128 } 129 if err := addAssoc(imgID, nil); err != nil { 130 return nil, err 131 } 132 } 133 continue 134 } 135 id, err := l.rs.Get(namedRef) 136 if err != nil { 137 return nil, err 138 } 139 if err := addAssoc(image.IDFromDigest(id), namedRef); err != nil { 140 return nil, err 141 } 142 143 } 144 return imgDescr, nil 145 } 146 147 // takeLayerReference will take/Get the image top layer reference 148 func (l *tarexporter) takeLayerReference(id image.ID, imgDescr *imageDescriptor) error { 149 img, err := l.is.Get(id) 150 if err != nil { 151 return err 152 } 153 imgDescr.image = img 154 topLayerID := img.RootFS.ChainID() 155 if topLayerID == "" { 156 return nil 157 } 158 os := img.OS 159 if os == "" { 160 os = runtime.GOOS 161 } 162 if !system.IsOSSupported(os) { 163 return fmt.Errorf("os %q is not supported", os) 164 } 165 layer, err := l.lss[os].Get(topLayerID) 166 if err != nil { 167 return err 168 } 169 imgDescr.layerRef = layer 170 return nil 171 } 172 173 // releaseLayerReferences will release all the image top layer references 174 func (l *tarexporter) releaseLayerReferences(imgDescr map[image.ID]*imageDescriptor) error { 175 for _, descr := range imgDescr { 176 if descr.layerRef != nil { 177 os := descr.image.OS 178 if os == "" { 179 os = runtime.GOOS 180 } 181 l.lss[os].Release(descr.layerRef) 182 } 183 } 184 return nil 185 } 186 187 func (s *saveSession) save(outStream io.Writer) error { 188 s.savedLayers = make(map[string]struct{}) 189 s.diffIDPaths = make(map[layer.DiffID]string) 190 191 // get image json 192 tempDir, err := ioutil.TempDir("", "docker-export-") 193 if err != nil { 194 return err 195 } 196 defer os.RemoveAll(tempDir) 197 198 s.outDir = tempDir 199 reposLegacy := make(map[string]map[string]string) 200 201 var manifest []manifestItem 202 var parentLinks []parentLink 203 204 for id, imageDescr := range s.images { 205 foreignSrcs, err := s.saveImage(id) 206 if err != nil { 207 return err 208 } 209 210 var repoTags []string 211 var layers []string 212 213 for _, ref := range imageDescr.refs { 214 familiarName := reference.FamiliarName(ref) 215 if _, ok := reposLegacy[familiarName]; !ok { 216 reposLegacy[familiarName] = make(map[string]string) 217 } 218 reposLegacy[familiarName][ref.Tag()] = imageDescr.layers[len(imageDescr.layers)-1] 219 repoTags = append(repoTags, reference.FamiliarString(ref)) 220 } 221 222 for _, l := range imageDescr.layers { 223 // IMPORTANT: We use path, not filepath here to ensure the layers 224 // in the manifest use Unix-style forward-slashes. Otherwise, a 225 // Linux image saved from LCOW won't be able to be imported on 226 // LCOL. 227 layers = append(layers, path.Join(l, legacyLayerFileName)) 228 } 229 230 manifest = append(manifest, manifestItem{ 231 Config: id.Digest().Hex() + ".json", 232 RepoTags: repoTags, 233 Layers: layers, 234 LayerSources: foreignSrcs, 235 }) 236 237 parentID, _ := s.is.GetParent(id) 238 parentLinks = append(parentLinks, parentLink{id, parentID}) 239 s.tarexporter.loggerImgEvent.LogImageEvent(id.String(), id.String(), "save") 240 } 241 242 for i, p := range validatedParentLinks(parentLinks) { 243 if p.parentID != "" { 244 manifest[i].Parent = p.parentID 245 } 246 } 247 248 if len(reposLegacy) > 0 { 249 reposFile := filepath.Join(tempDir, legacyRepositoriesFileName) 250 rf, err := os.OpenFile(reposFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) 251 if err != nil { 252 return err 253 } 254 255 if err := json.NewEncoder(rf).Encode(reposLegacy); err != nil { 256 rf.Close() 257 return err 258 } 259 260 rf.Close() 261 262 if err := system.Chtimes(reposFile, time.Unix(0, 0), time.Unix(0, 0)); err != nil { 263 return err 264 } 265 } 266 267 manifestFileName := filepath.Join(tempDir, manifestFileName) 268 f, err := os.OpenFile(manifestFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) 269 if err != nil { 270 return err 271 } 272 273 if err := json.NewEncoder(f).Encode(manifest); err != nil { 274 f.Close() 275 return err 276 } 277 278 f.Close() 279 280 if err := system.Chtimes(manifestFileName, time.Unix(0, 0), time.Unix(0, 0)); err != nil { 281 return err 282 } 283 284 fs, err := archive.Tar(tempDir, archive.Uncompressed) 285 if err != nil { 286 return err 287 } 288 defer fs.Close() 289 290 _, err = io.Copy(outStream, fs) 291 return err 292 } 293 294 func (s *saveSession) saveImage(id image.ID) (map[layer.DiffID]distribution.Descriptor, error) { 295 img := s.images[id].image 296 if len(img.RootFS.DiffIDs) == 0 { 297 return nil, fmt.Errorf("empty export - not implemented") 298 } 299 300 var parent digest.Digest 301 var layers []string 302 var foreignSrcs map[layer.DiffID]distribution.Descriptor 303 for i := range img.RootFS.DiffIDs { 304 v1Img := image.V1Image{ 305 // This is for backward compatibility used for 306 // pre v1.9 docker. 307 Created: time.Unix(0, 0), 308 } 309 if i == len(img.RootFS.DiffIDs)-1 { 310 v1Img = img.V1Image 311 } 312 rootFS := *img.RootFS 313 rootFS.DiffIDs = rootFS.DiffIDs[:i+1] 314 v1ID, err := v1.CreateID(v1Img, rootFS.ChainID(), parent) 315 if err != nil { 316 return nil, err 317 } 318 319 v1Img.ID = v1ID.Hex() 320 if parent != "" { 321 v1Img.Parent = parent.Hex() 322 } 323 324 v1Img.OS = img.OS 325 src, err := s.saveLayer(rootFS.ChainID(), v1Img, img.Created) 326 if err != nil { 327 return nil, err 328 } 329 layers = append(layers, v1Img.ID) 330 parent = v1ID 331 if src.Digest != "" { 332 if foreignSrcs == nil { 333 foreignSrcs = make(map[layer.DiffID]distribution.Descriptor) 334 } 335 foreignSrcs[img.RootFS.DiffIDs[i]] = src 336 } 337 } 338 339 configFile := filepath.Join(s.outDir, id.Digest().Hex()+".json") 340 if err := ioutil.WriteFile(configFile, img.RawJSON(), 0644); err != nil { 341 return nil, err 342 } 343 if err := system.Chtimes(configFile, img.Created, img.Created); err != nil { 344 return nil, err 345 } 346 347 s.images[id].layers = layers 348 return foreignSrcs, nil 349 } 350 351 func (s *saveSession) saveLayer(id layer.ChainID, legacyImg image.V1Image, createdTime time.Time) (distribution.Descriptor, error) { 352 if _, exists := s.savedLayers[legacyImg.ID]; exists { 353 return distribution.Descriptor{}, nil 354 } 355 356 outDir := filepath.Join(s.outDir, legacyImg.ID) 357 if err := os.Mkdir(outDir, 0755); err != nil { 358 return distribution.Descriptor{}, err 359 } 360 361 // todo: why is this version file here? 362 if err := ioutil.WriteFile(filepath.Join(outDir, legacyVersionFileName), []byte("1.0"), 0644); err != nil { 363 return distribution.Descriptor{}, err 364 } 365 366 imageConfig, err := json.Marshal(legacyImg) 367 if err != nil { 368 return distribution.Descriptor{}, err 369 } 370 371 if err := ioutil.WriteFile(filepath.Join(outDir, legacyConfigFileName), imageConfig, 0644); err != nil { 372 return distribution.Descriptor{}, err 373 } 374 375 // serialize filesystem 376 layerPath := filepath.Join(outDir, legacyLayerFileName) 377 operatingSystem := legacyImg.OS 378 if operatingSystem == "" { 379 operatingSystem = runtime.GOOS 380 } 381 l, err := s.lss[operatingSystem].Get(id) 382 if err != nil { 383 return distribution.Descriptor{}, err 384 } 385 defer layer.ReleaseAndLog(s.lss[operatingSystem], l) 386 387 if oldPath, exists := s.diffIDPaths[l.DiffID()]; exists { 388 relPath, err := filepath.Rel(outDir, oldPath) 389 if err != nil { 390 return distribution.Descriptor{}, err 391 } 392 if err := os.Symlink(relPath, layerPath); err != nil { 393 return distribution.Descriptor{}, errors.Wrap(err, "error creating symlink while saving layer") 394 } 395 } else { 396 // Use system.CreateSequential rather than os.Create. This ensures sequential 397 // file access on Windows to avoid eating into MM standby list. 398 // On Linux, this equates to a regular os.Create. 399 tarFile, err := system.CreateSequential(layerPath) 400 if err != nil { 401 return distribution.Descriptor{}, err 402 } 403 defer tarFile.Close() 404 405 arch, err := l.TarStream() 406 if err != nil { 407 return distribution.Descriptor{}, err 408 } 409 defer arch.Close() 410 411 if _, err := io.Copy(tarFile, arch); err != nil { 412 return distribution.Descriptor{}, err 413 } 414 415 for _, fname := range []string{"", legacyVersionFileName, legacyConfigFileName, legacyLayerFileName} { 416 // todo: maybe save layer created timestamp? 417 if err := system.Chtimes(filepath.Join(outDir, fname), createdTime, createdTime); err != nil { 418 return distribution.Descriptor{}, err 419 } 420 } 421 422 s.diffIDPaths[l.DiffID()] = layerPath 423 } 424 s.savedLayers[legacyImg.ID] = struct{}{} 425 426 var src distribution.Descriptor 427 if fs, ok := l.(distribution.Describable); ok { 428 src = fs.Descriptor() 429 } 430 return src, nil 431 }