github.com/mheon/docker@v0.11.2-0.20150922122814-44f47903a831/graph/push_v1.go (about)

     1  package graph
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"os"
     7  	"sync"
     8  
     9  	"github.com/Sirupsen/logrus"
    10  	"github.com/docker/distribution/registry/client/transport"
    11  	"github.com/docker/docker/pkg/ioutils"
    12  	"github.com/docker/docker/pkg/progressreader"
    13  	"github.com/docker/docker/pkg/streamformatter"
    14  	"github.com/docker/docker/pkg/stringid"
    15  	"github.com/docker/docker/registry"
    16  	"github.com/docker/docker/utils"
    17  )
    18  
    19  type v1Pusher struct {
    20  	*TagStore
    21  	endpoint  registry.APIEndpoint
    22  	localRepo Repository
    23  	repoInfo  *registry.RepositoryInfo
    24  	config    *ImagePushConfig
    25  	sf        *streamformatter.StreamFormatter
    26  	session   *registry.Session
    27  
    28  	out io.Writer
    29  }
    30  
    31  func (p *v1Pusher) Push() (fallback bool, err error) {
    32  	tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name)
    33  	if err != nil {
    34  		return false, err
    35  	}
    36  	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
    37  	tr := transport.NewTransport(
    38  		// TODO(tiborvass): was NoTimeout
    39  		registry.NewTransport(tlsConfig),
    40  		registry.DockerHeaders(p.config.MetaHeaders)...,
    41  	)
    42  	client := registry.HTTPClient(tr)
    43  	v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders)
    44  	if err != nil {
    45  		logrus.Debugf("Could not get v1 endpoint: %v", err)
    46  		return true, err
    47  	}
    48  	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
    49  	if err != nil {
    50  		// TODO(dmcgowan): Check if should fallback
    51  		return true, err
    52  	}
    53  	if err := p.pushRepository(p.config.Tag); err != nil {
    54  		// TODO(dmcgowan): Check if should fallback
    55  		return false, err
    56  	}
    57  	return false, nil
    58  }
    59  
    60  // Retrieve the all the images to be uploaded in the correct order
    61  func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) {
    62  	var (
    63  		imageList   []string
    64  		imagesSeen  = make(map[string]bool)
    65  		tagsByImage = make(map[string][]string)
    66  	)
    67  
    68  	for tag, id := range p.localRepo {
    69  		if requestedTag != "" && requestedTag != tag {
    70  			// Include only the requested tag.
    71  			continue
    72  		}
    73  
    74  		if utils.DigestReference(tag) {
    75  			// Ignore digest references.
    76  			continue
    77  		}
    78  
    79  		var imageListForThisTag []string
    80  
    81  		tagsByImage[id] = append(tagsByImage[id], tag)
    82  
    83  		for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) {
    84  			if err != nil {
    85  				return nil, nil, err
    86  			}
    87  
    88  			if imagesSeen[img.ID] {
    89  				// This image is already on the list, we can ignore it and all its parents
    90  				break
    91  			}
    92  
    93  			imagesSeen[img.ID] = true
    94  			imageListForThisTag = append(imageListForThisTag, img.ID)
    95  		}
    96  
    97  		// reverse the image list for this tag (so the "most"-parent image is first)
    98  		for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
    99  			imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
   100  		}
   101  
   102  		// append to main image list
   103  		imageList = append(imageList, imageListForThisTag...)
   104  	}
   105  	if len(imageList) == 0 {
   106  		return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
   107  	}
   108  	logrus.Debugf("Image list: %v", imageList)
   109  	logrus.Debugf("Tags by image: %v", tagsByImage)
   110  
   111  	return imageList, tagsByImage, nil
   112  }
   113  
   114  // createImageIndex returns an index of an image's layer IDs and tags.
   115  func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData {
   116  	var imageIndex []*registry.ImgData
   117  	for _, id := range images {
   118  		if tags, hasTags := tags[id]; hasTags {
   119  			// If an image has tags you must add an entry in the image index
   120  			// for each tag
   121  			for _, tag := range tags {
   122  				imageIndex = append(imageIndex, &registry.ImgData{
   123  					ID:  id,
   124  					Tag: tag,
   125  				})
   126  			}
   127  			continue
   128  		}
   129  		// If the image does not have a tag it still needs to be sent to the
   130  		// registry with an empty tag so that it is accociated with the repository
   131  		imageIndex = append(imageIndex, &registry.ImgData{
   132  			ID:  id,
   133  			Tag: "",
   134  		})
   135  	}
   136  	return imageIndex
   137  }
   138  
   139  type imagePushData struct {
   140  	id       string
   141  	endpoint string
   142  }
   143  
   144  // lookupImageOnEndpoint checks the specified endpoint to see if an image exists
   145  // and if it is absent then it sends the image id to the channel to be pushed.
   146  func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
   147  	defer wg.Done()
   148  	for image := range images {
   149  		if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil {
   150  			logrus.Errorf("Error in LookupRemoteImage: %s", err)
   151  			imagesToPush <- image.id
   152  			continue
   153  		}
   154  		p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id)))
   155  	}
   156  }
   157  
   158  func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error {
   159  	workerCount := len(imageIDs)
   160  	// start a maximum of 5 workers to check if images exist on the specified endpoint.
   161  	if workerCount > 5 {
   162  		workerCount = 5
   163  	}
   164  	var (
   165  		wg           = &sync.WaitGroup{}
   166  		imageData    = make(chan imagePushData, workerCount*2)
   167  		imagesToPush = make(chan string, workerCount*2)
   168  		pushes       = make(chan map[string]struct{}, 1)
   169  	)
   170  	for i := 0; i < workerCount; i++ {
   171  		wg.Add(1)
   172  		go p.lookupImageOnEndpoint(wg, imageData, imagesToPush)
   173  	}
   174  	// start a go routine that consumes the images to push
   175  	go func() {
   176  		shouldPush := make(map[string]struct{})
   177  		for id := range imagesToPush {
   178  			shouldPush[id] = struct{}{}
   179  		}
   180  		pushes <- shouldPush
   181  	}()
   182  	for _, id := range imageIDs {
   183  		imageData <- imagePushData{
   184  			id:       id,
   185  			endpoint: endpoint,
   186  		}
   187  	}
   188  	// close the channel to notify the workers that there will be no more images to check.
   189  	close(imageData)
   190  	wg.Wait()
   191  	close(imagesToPush)
   192  	// wait for all the images that require pushes to be collected into a consumable map.
   193  	shouldPush := <-pushes
   194  	// finish by pushing any images and tags to the endpoint.  The order that the images are pushed
   195  	// is very important that is why we are still iterating over the ordered list of imageIDs.
   196  	for _, id := range imageIDs {
   197  		if _, push := shouldPush[id]; push {
   198  			if _, err := p.pushImage(id, endpoint); err != nil {
   199  				// FIXME: Continue on error?
   200  				return err
   201  			}
   202  		}
   203  		for _, tag := range tags[id] {
   204  			p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
   205  			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil {
   206  				return err
   207  			}
   208  		}
   209  	}
   210  	return nil
   211  }
   212  
   213  // pushRepository pushes layers that do not already exist on the registry.
   214  func (p *v1Pusher) pushRepository(tag string) error {
   215  	logrus.Debugf("Local repo: %s", p.localRepo)
   216  	p.out = ioutils.NewWriteFlusher(p.config.OutStream)
   217  	imgList, tags, err := p.getImageList(tag)
   218  	if err != nil {
   219  		return err
   220  	}
   221  	p.out.Write(p.sf.FormatStatus("", "Sending image list"))
   222  
   223  	imageIndex := p.createImageIndex(imgList, tags)
   224  	logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
   225  	for _, data := range imageIndex {
   226  		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
   227  	}
   228  
   229  	if _, found := p.poolAdd("push", p.repoInfo.LocalName); found {
   230  		return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName)
   231  	}
   232  	defer p.poolRemove("push", p.repoInfo.LocalName)
   233  
   234  	// Register all the images in a repository with the registry
   235  	// If an image is not in this list it will not be associated with the repository
   236  	repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil)
   237  	if err != nil {
   238  		return err
   239  	}
   240  	nTag := 1
   241  	if tag == "" {
   242  		nTag = len(p.localRepo)
   243  	}
   244  	p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag))
   245  	// push the repository to each of the endpoints only if it does not exist.
   246  	for _, endpoint := range repoData.Endpoints {
   247  		if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil {
   248  			return err
   249  		}
   250  	}
   251  	_, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints)
   252  	return err
   253  }
   254  
   255  func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) {
   256  	jsonRaw, err := p.graph.RawJSON(imgID)
   257  	if err != nil {
   258  		return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
   259  	}
   260  	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
   261  
   262  	imgData := &registry.ImgData{
   263  		ID: imgID,
   264  	}
   265  
   266  	// Send the json
   267  	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
   268  		if err == registry.ErrAlreadyExists {
   269  			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil))
   270  			return "", nil
   271  		}
   272  		return "", err
   273  	}
   274  
   275  	layerData, err := p.graph.TempLayerArchive(imgID, p.sf, p.out)
   276  	if err != nil {
   277  		return "", fmt.Errorf("Failed to generate layer archive: %s", err)
   278  	}
   279  	defer os.RemoveAll(layerData.Name())
   280  
   281  	// Send the layer
   282  	logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size)
   283  
   284  	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
   285  		progressreader.New(progressreader.Config{
   286  			In:        layerData,
   287  			Out:       p.out,
   288  			Formatter: p.sf,
   289  			Size:      layerData.Size,
   290  			NewLines:  false,
   291  			ID:        stringid.TruncateID(imgData.ID),
   292  			Action:    "Pushing",
   293  		}), ep, jsonRaw)
   294  	if err != nil {
   295  		return "", err
   296  	}
   297  	imgData.Checksum = checksum
   298  	imgData.ChecksumPayload = checksumPayload
   299  	// Send the checksum
   300  	if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
   301  		return "", err
   302  	}
   303  
   304  	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil))
   305  	return imgData.Checksum, nil
   306  }