github.com/sams1990/dockerrepo@v17.12.1-ce-rc2+incompatible/migrate/v1/migratev1.go (about) 1 package v1 2 3 import ( 4 "errors" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "runtime" 10 "strconv" 11 "sync" 12 "time" 13 14 "encoding/json" 15 16 "github.com/docker/distribution/reference" 17 "github.com/docker/docker/distribution/metadata" 18 "github.com/docker/docker/image" 19 imagev1 "github.com/docker/docker/image/v1" 20 "github.com/docker/docker/layer" 21 "github.com/docker/docker/pkg/ioutils" 22 refstore "github.com/docker/docker/reference" 23 "github.com/opencontainers/go-digest" 24 "github.com/sirupsen/logrus" 25 ) 26 27 type graphIDRegistrar interface { 28 RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error) 29 Release(layer.Layer) ([]layer.Metadata, error) 30 } 31 32 type graphIDMounter interface { 33 CreateRWLayerByGraphID(string, string, layer.ChainID) error 34 } 35 36 type checksumCalculator interface { 37 ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error) 38 } 39 40 const ( 41 graphDirName = "graph" 42 tarDataFileName = "tar-data.json.gz" 43 migrationFileName = ".migration-v1-images.json" 44 migrationTagsFileName = ".migration-v1-tags" 45 migrationDiffIDFileName = ".migration-diffid" 46 migrationSizeFileName = ".migration-size" 47 migrationTarDataFileName = ".migration-tardata" 48 containersDirName = "containers" 49 configFileNameLegacy = "config.json" 50 configFileName = "config.v2.json" 51 repositoriesFilePrefixLegacy = "repositories-" 52 ) 53 54 var ( 55 errUnsupported = errors.New("migration is not supported") 56 ) 57 58 // Migrate takes an old graph directory and transforms the metadata into the 59 // new format. 60 func Migrate(root, driverName string, ls layer.Store, is image.Store, rs refstore.Store, ms metadata.Store) error { 61 graphDir := filepath.Join(root, graphDirName) 62 if _, err := os.Lstat(graphDir); os.IsNotExist(err) { 63 return nil 64 } 65 66 mappings, err := restoreMappings(root) 67 if err != nil { 68 return err 69 } 70 71 if cc, ok := ls.(checksumCalculator); ok { 72 CalculateLayerChecksums(root, cc, mappings) 73 } 74 75 if registrar, ok := ls.(graphIDRegistrar); !ok { 76 return errUnsupported 77 } else if err := migrateImages(root, registrar, is, ms, mappings); err != nil { 78 return err 79 } 80 81 err = saveMappings(root, mappings) 82 if err != nil { 83 return err 84 } 85 86 if mounter, ok := ls.(graphIDMounter); !ok { 87 return errUnsupported 88 } else if err := migrateContainers(root, mounter, is, mappings); err != nil { 89 return err 90 } 91 92 if err := migrateRefs(root, driverName, rs, mappings); err != nil { 93 return err 94 } 95 96 return nil 97 } 98 99 // CalculateLayerChecksums walks an old graph directory and calculates checksums 100 // for each layer. These checksums are later used for migration. 101 func CalculateLayerChecksums(root string, ls checksumCalculator, mappings map[string]image.ID) { 102 graphDir := filepath.Join(root, graphDirName) 103 // spawn some extra workers also for maximum performance because the process is bounded by both cpu and io 104 workers := runtime.NumCPU() * 3 105 workQueue := make(chan string, workers) 106 107 wg := sync.WaitGroup{} 108 109 for i := 0; i < workers; i++ { 110 wg.Add(1) 111 go func() { 112 for id := range workQueue { 113 start := time.Now() 114 if err := calculateLayerChecksum(graphDir, id, ls); err != nil { 115 logrus.Errorf("could not calculate checksum for %q, %q", id, err) 116 } 117 elapsed := time.Since(start) 118 logrus.Debugf("layer %s took %.2f seconds", id, elapsed.Seconds()) 119 } 120 wg.Done() 121 }() 122 } 123 124 dir, err := ioutil.ReadDir(graphDir) 125 if err != nil { 126 logrus.Errorf("could not read directory %q", graphDir) 127 return 128 } 129 for _, v := range dir { 130 v1ID := v.Name() 131 if err := imagev1.ValidateID(v1ID); err != nil { 132 continue 133 } 134 if _, ok := mappings[v1ID]; ok { // support old migrations without helper files 135 continue 136 } 137 workQueue <- v1ID 138 } 139 close(workQueue) 140 wg.Wait() 141 } 142 143 func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error { 144 diffIDFile := filepath.Join(graphDir, id, migrationDiffIDFileName) 145 if _, err := os.Lstat(diffIDFile); err == nil { 146 return nil 147 } else if !os.IsNotExist(err) { 148 return err 149 } 150 151 parent, err := getParent(filepath.Join(graphDir, id)) 152 if err != nil { 153 return err 154 } 155 156 diffID, size, err := ls.ChecksumForGraphID(id, parent, filepath.Join(graphDir, id, tarDataFileName), filepath.Join(graphDir, id, migrationTarDataFileName)) 157 if err != nil { 158 return err 159 } 160 161 if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationSizeFileName), []byte(strconv.Itoa(int(size))), 0600); err != nil { 162 return err 163 } 164 165 if err := ioutils.AtomicWriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil { 166 return err 167 } 168 169 logrus.Infof("calculated checksum for layer %s: %s", id, diffID) 170 return nil 171 } 172 173 func restoreMappings(root string) (map[string]image.ID, error) { 174 mappings := make(map[string]image.ID) 175 176 mfile := filepath.Join(root, migrationFileName) 177 f, err := os.Open(mfile) 178 if err != nil && !os.IsNotExist(err) { 179 return nil, err 180 } else if err == nil { 181 err := json.NewDecoder(f).Decode(&mappings) 182 if err != nil { 183 f.Close() 184 return nil, err 185 } 186 f.Close() 187 } 188 189 return mappings, nil 190 } 191 192 func saveMappings(root string, mappings map[string]image.ID) error { 193 mfile := filepath.Join(root, migrationFileName) 194 f, err := os.OpenFile(mfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) 195 if err != nil { 196 return err 197 } 198 defer f.Close() 199 return json.NewEncoder(f).Encode(mappings) 200 } 201 202 func migrateImages(root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) error { 203 graphDir := filepath.Join(root, graphDirName) 204 205 dir, err := ioutil.ReadDir(graphDir) 206 if err != nil { 207 return err 208 } 209 for _, v := range dir { 210 v1ID := v.Name() 211 if err := imagev1.ValidateID(v1ID); err != nil { 212 continue 213 } 214 if _, exists := mappings[v1ID]; exists { 215 continue 216 } 217 if err := migrateImage(v1ID, root, ls, is, ms, mappings); err != nil { 218 continue 219 } 220 } 221 222 return nil 223 } 224 225 func migrateContainers(root string, ls graphIDMounter, is image.Store, imageMappings map[string]image.ID) error { 226 containersDir := filepath.Join(root, containersDirName) 227 dir, err := ioutil.ReadDir(containersDir) 228 if err != nil { 229 return err 230 } 231 for _, v := range dir { 232 id := v.Name() 233 234 if _, err := os.Stat(filepath.Join(containersDir, id, configFileName)); err == nil { 235 continue 236 } 237 238 containerJSON, err := ioutil.ReadFile(filepath.Join(containersDir, id, configFileNameLegacy)) 239 if err != nil { 240 logrus.Errorf("migrate container error: %v", err) 241 continue 242 } 243 244 var c map[string]*json.RawMessage 245 if err := json.Unmarshal(containerJSON, &c); err != nil { 246 logrus.Errorf("migrate container error: %v", err) 247 continue 248 } 249 250 imageStrJSON, ok := c["Image"] 251 if !ok { 252 return fmt.Errorf("invalid container configuration for %v", id) 253 } 254 255 var image string 256 if err := json.Unmarshal([]byte(*imageStrJSON), &image); err != nil { 257 logrus.Errorf("migrate container error: %v", err) 258 continue 259 } 260 261 imageID, ok := imageMappings[image] 262 if !ok { 263 logrus.Errorf("image not migrated %v", imageID) // non-fatal error 264 continue 265 } 266 267 c["Image"] = rawJSON(imageID) 268 269 containerJSON, err = json.Marshal(c) 270 if err != nil { 271 return err 272 } 273 274 if err := ioutil.WriteFile(filepath.Join(containersDir, id, configFileName), containerJSON, 0600); err != nil { 275 return err 276 } 277 278 img, err := is.Get(imageID) 279 if err != nil { 280 return err 281 } 282 283 if err := ls.CreateRWLayerByGraphID(id, id, img.RootFS.ChainID()); err != nil { 284 logrus.Errorf("migrate container error: %v", err) 285 continue 286 } 287 288 logrus.Infof("migrated container %s to point to %s", id, imageID) 289 290 } 291 return nil 292 } 293 294 type refAdder interface { 295 AddTag(ref reference.Named, id digest.Digest, force bool) error 296 AddDigest(ref reference.Canonical, id digest.Digest, force bool) error 297 } 298 299 func migrateRefs(root, driverName string, rs refAdder, mappings map[string]image.ID) error { 300 migrationFile := filepath.Join(root, migrationTagsFileName) 301 if _, err := os.Lstat(migrationFile); !os.IsNotExist(err) { 302 return err 303 } 304 305 type repositories struct { 306 Repositories map[string]map[string]string 307 } 308 309 var repos repositories 310 311 f, err := os.Open(filepath.Join(root, repositoriesFilePrefixLegacy+driverName)) 312 if err != nil { 313 if os.IsNotExist(err) { 314 return nil 315 } 316 return err 317 } 318 defer f.Close() 319 if err := json.NewDecoder(f).Decode(&repos); err != nil { 320 return err 321 } 322 323 for name, repo := range repos.Repositories { 324 for tag, id := range repo { 325 if strongID, exists := mappings[id]; exists { 326 ref, err := reference.ParseNormalizedNamed(name) 327 if err != nil { 328 logrus.Errorf("migrate tags: invalid name %q, %q", name, err) 329 continue 330 } 331 if !reference.IsNameOnly(ref) { 332 logrus.Errorf("migrate tags: invalid name %q, unexpected tag or digest", name) 333 continue 334 } 335 if dgst, err := digest.Parse(tag); err == nil { 336 canonical, err := reference.WithDigest(reference.TrimNamed(ref), dgst) 337 if err != nil { 338 logrus.Errorf("migrate tags: invalid digest %q, %q", dgst, err) 339 continue 340 } 341 if err := rs.AddDigest(canonical, strongID.Digest(), false); err != nil { 342 logrus.Errorf("can't migrate digest %q for %q, err: %q", reference.FamiliarString(ref), strongID, err) 343 } 344 } else { 345 tagRef, err := reference.WithTag(ref, tag) 346 if err != nil { 347 logrus.Errorf("migrate tags: invalid tag %q, %q", tag, err) 348 continue 349 } 350 if err := rs.AddTag(tagRef, strongID.Digest(), false); err != nil { 351 logrus.Errorf("can't migrate tag %q for %q, err: %q", reference.FamiliarString(ref), strongID, err) 352 } 353 } 354 logrus.Infof("migrated tag %s:%s to point to %s", name, tag, strongID) 355 } 356 } 357 } 358 359 mf, err := os.Create(migrationFile) 360 if err != nil { 361 return err 362 } 363 mf.Close() 364 365 return nil 366 } 367 368 func getParent(confDir string) (string, error) { 369 jsonFile := filepath.Join(confDir, "json") 370 imageJSON, err := ioutil.ReadFile(jsonFile) 371 if err != nil { 372 return "", err 373 } 374 var parent struct { 375 Parent string 376 ParentID digest.Digest `json:"parent_id"` 377 } 378 if err := json.Unmarshal(imageJSON, &parent); err != nil { 379 return "", err 380 } 381 if parent.Parent == "" && parent.ParentID != "" { // v1.9 382 parent.Parent = parent.ParentID.Hex() 383 } 384 // compatibilityID for parent 385 parentCompatibilityID, err := ioutil.ReadFile(filepath.Join(confDir, "parent")) 386 if err == nil && len(parentCompatibilityID) > 0 { 387 parent.Parent = string(parentCompatibilityID) 388 } 389 return parent.Parent, nil 390 } 391 392 func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metadata.Store, mappings map[string]image.ID) (err error) { 393 defer func() { 394 if err != nil { 395 logrus.Errorf("migration failed for %v, err: %v", id, err) 396 } 397 }() 398 399 parent, err := getParent(filepath.Join(root, graphDirName, id)) 400 if err != nil { 401 return err 402 } 403 404 var parentID image.ID 405 if parent != "" { 406 var exists bool 407 if parentID, exists = mappings[parent]; !exists { 408 if err := migrateImage(parent, root, ls, is, ms, mappings); err != nil { 409 // todo: fail or allow broken chains? 410 return err 411 } 412 parentID = mappings[parent] 413 } 414 } 415 416 rootFS := image.NewRootFS() 417 var history []image.History 418 419 if parentID != "" { 420 parentImg, err := is.Get(parentID) 421 if err != nil { 422 return err 423 } 424 425 rootFS = parentImg.RootFS 426 history = parentImg.History 427 } 428 429 diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName)) 430 if err != nil { 431 return err 432 } 433 diffID, err := digest.Parse(string(diffIDData)) 434 if err != nil { 435 return err 436 } 437 438 sizeStr, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationSizeFileName)) 439 if err != nil { 440 return err 441 } 442 size, err := strconv.ParseInt(string(sizeStr), 10, 64) 443 if err != nil { 444 return err 445 } 446 447 layer, err := ls.RegisterByGraphID(id, rootFS.ChainID(), layer.DiffID(diffID), filepath.Join(root, graphDirName, id, migrationTarDataFileName), size) 448 if err != nil { 449 return err 450 } 451 logrus.Infof("migrated layer %s to %s", id, layer.DiffID()) 452 453 jsonFile := filepath.Join(root, graphDirName, id, "json") 454 imageJSON, err := ioutil.ReadFile(jsonFile) 455 if err != nil { 456 return err 457 } 458 459 h, err := imagev1.HistoryFromConfig(imageJSON, false) 460 if err != nil { 461 return err 462 } 463 history = append(history, h) 464 465 rootFS.Append(layer.DiffID()) 466 467 config, err := imagev1.MakeConfigFromV1Config(imageJSON, rootFS, history) 468 if err != nil { 469 return err 470 } 471 strongID, err := is.Create(config) 472 if err != nil { 473 return err 474 } 475 logrus.Infof("migrated image %s to %s", id, strongID) 476 477 if parentID != "" { 478 if err := is.SetParent(strongID, parentID); err != nil { 479 return err 480 } 481 } 482 483 checksum, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, "checksum")) 484 if err == nil { // best effort 485 dgst, err := digest.Parse(string(checksum)) 486 if err == nil { 487 V2MetadataService := metadata.NewV2MetadataService(ms) 488 V2MetadataService.Add(layer.DiffID(), metadata.V2Metadata{Digest: dgst}) 489 } 490 } 491 _, err = ls.Release(layer) 492 if err != nil { 493 return err 494 } 495 496 mappings[id] = strongID 497 return 498 } 499 500 func rawJSON(value interface{}) *json.RawMessage { 501 jsonval, err := json.Marshal(value) 502 if err != nil { 503 return nil 504 } 505 return (*json.RawMessage)(&jsonval) 506 }