github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imageserver/rpcd/replicator.go (about)

     1  package rpcd
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"time"
     8  
     9  	imageclient "github.com/Cloud-Foundations/Dominator/imageserver/client"
    10  	"github.com/Cloud-Foundations/Dominator/lib/format"
    11  	"github.com/Cloud-Foundations/Dominator/lib/image"
    12  	"github.com/Cloud-Foundations/Dominator/lib/log"
    13  	"github.com/Cloud-Foundations/Dominator/lib/log/prefixlogger"
    14  	objectclient "github.com/Cloud-Foundations/Dominator/lib/objectserver/client"
    15  	"github.com/Cloud-Foundations/Dominator/lib/srpc"
    16  	"github.com/Cloud-Foundations/Dominator/proto/imageserver"
    17  )
    18  
    19  func (t *srpcType) replicator(finishedReplication chan<- struct{}) {
    20  	initialTimeout := time.Second * 15
    21  	timeout := initialTimeout
    22  	var nextSleepStopTime time.Time
    23  	for {
    24  		nextSleepStopTime = time.Now().Add(timeout)
    25  		if client, err := srpc.DialHTTP("tcp", t.replicationMaster,
    26  			timeout); err != nil {
    27  			t.logger.Printf("Error dialing: %s %s\n", t.replicationMaster, err)
    28  		} else {
    29  			if conn, err := client.Call(
    30  				"ImageServer.GetImageUpdates"); err != nil {
    31  				t.logger.Println(err)
    32  			} else {
    33  				if err := t.getUpdates(conn, &finishedReplication); err != nil {
    34  					if err == io.EOF {
    35  						t.logger.Println(
    36  							"Connection to image replicator closed")
    37  						if nextSleepStopTime.Sub(time.Now()) < 1 {
    38  							timeout = initialTimeout
    39  						}
    40  					} else {
    41  						t.logger.Println(err)
    42  					}
    43  				}
    44  				conn.Close()
    45  			}
    46  			client.Close()
    47  		}
    48  		time.Sleep(nextSleepStopTime.Sub(time.Now()))
    49  		if timeout < time.Minute {
    50  			timeout *= 2
    51  		}
    52  	}
    53  }
    54  
    55  func (t *srpcType) getUpdates(conn *srpc.Conn,
    56  	finishedReplication *chan<- struct{}) error {
    57  	t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster)
    58  	replicationStartTime := time.Now()
    59  	initialImages := make(map[string]struct{})
    60  	if t.archiveMode {
    61  		initialImages = nil
    62  	}
    63  	for {
    64  		var imageUpdate imageserver.ImageUpdate
    65  		if err := conn.Decode(&imageUpdate); err != nil {
    66  			if err == io.EOF {
    67  				return err
    68  			}
    69  			return errors.New("decode err: " + err.Error())
    70  		}
    71  		switch imageUpdate.Operation {
    72  		case imageserver.OperationAddImage:
    73  			if imageUpdate.Name == "" {
    74  				if initialImages != nil {
    75  					t.deleteMissingImages(initialImages)
    76  					initialImages = nil
    77  				}
    78  				if *finishedReplication != nil {
    79  					close(*finishedReplication)
    80  					*finishedReplication = nil
    81  				}
    82  				t.logger.Printf("Replicated all current images in %s\n",
    83  					format.Duration(time.Since(replicationStartTime)))
    84  				continue
    85  			}
    86  			if initialImages != nil {
    87  				initialImages[imageUpdate.Name] = struct{}{}
    88  			}
    89  			if err := t.addImage(imageUpdate.Name); err != nil {
    90  				return errors.New("error adding image: " + imageUpdate.Name +
    91  					": " + err.Error())
    92  			}
    93  		case imageserver.OperationDeleteImage:
    94  			if t.archiveMode {
    95  				continue
    96  			}
    97  			t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name)
    98  			err := t.imageDataBase.DeleteImage(imageUpdate.Name,
    99  				&srpc.AuthInformation{HaveMethodAccess: true})
   100  			if err != nil {
   101  				return err
   102  			}
   103  		case imageserver.OperationMakeDirectory:
   104  			directory := imageUpdate.Directory
   105  			if directory == nil {
   106  				return errors.New("nil imageUpdate.Directory")
   107  			}
   108  			if err := t.imageDataBase.UpdateDirectory(*directory); err != nil {
   109  				return err
   110  			}
   111  		}
   112  	}
   113  }
   114  
   115  func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) {
   116  	missingImages := make([]string, 0)
   117  	for _, imageName := range t.imageDataBase.ListImages() {
   118  		if _, ok := imagesToKeep[imageName]; !ok {
   119  			missingImages = append(missingImages, imageName)
   120  		}
   121  	}
   122  	for _, imageName := range missingImages {
   123  		t.logger.Printf("Replicator(%s): delete missing image\n", imageName)
   124  		err := t.imageDataBase.DeleteImage(imageName,
   125  			&srpc.AuthInformation{HaveMethodAccess: true})
   126  		if err != nil {
   127  			t.logger.Println(err)
   128  		}
   129  	}
   130  }
   131  
   132  func (t *srpcType) extendImageExpiration(name string,
   133  	img *image.Image) (bool, error) {
   134  	timeout := time.Second * 60
   135  	client, err := t.imageserverResource.GetHTTP(nil, timeout)
   136  	if err != nil {
   137  		return false, err
   138  	}
   139  	defer client.Put()
   140  	expiresAt, err := imageclient.GetImageExpiration(client, name)
   141  	if err != nil {
   142  		if err == io.EOF {
   143  			client.Close()
   144  		}
   145  		return false, err
   146  	}
   147  	return t.imageDataBase.ChangeImageExpiration(name, expiresAt,
   148  		&srpc.AuthInformation{HaveMethodAccess: true})
   149  }
   150  
   151  func (t *srpcType) addImage(name string) error {
   152  	timeout := time.Second * 60
   153  	if t.checkImageBeingInjected(name) {
   154  		return nil
   155  	}
   156  	logger := prefixlogger.New(fmt.Sprintf("Replicator(%s): ", name), t.logger)
   157  	if img := t.imageDataBase.GetImage(name); img != nil {
   158  		if img.ExpiresAt.IsZero() {
   159  			return nil
   160  		}
   161  		if changed, err := t.extendImageExpiration(name, img); err != nil {
   162  			logger.Println(err)
   163  		} else if changed {
   164  			logger.Println("extended expiration time")
   165  		}
   166  		return nil
   167  	}
   168  	logger.Println("add image")
   169  	client, err := t.imageserverResource.GetHTTP(nil, timeout)
   170  	if err != nil {
   171  		return err
   172  	}
   173  	defer client.Put()
   174  	request := imageserver.GetImageRequest{
   175  		ImageName: name,
   176  		Timeout:   timeout,
   177  	}
   178  	if t.archiveMode && !*archiveExpiringImages {
   179  		request.IgnoreFilesystemIfExpiring = true
   180  	}
   181  	var reply imageserver.GetImageResponse
   182  	err = client.RequestReply("ImageServer.GetImage", request, &reply)
   183  	if err != nil {
   184  		client.Close()
   185  		return err
   186  	}
   187  	img := reply.Image
   188  	if img == nil {
   189  		return errors.New(name + ": not found")
   190  	}
   191  	logger.Println("downloaded image")
   192  	if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages {
   193  		logger.Println("ignoring expiring image in archiver mode")
   194  		return nil
   195  	}
   196  	img.FileSystem.RebuildInodePointers()
   197  	err = t.imageDataBase.DoWithPendingImage(img, func() error {
   198  		if err := t.getMissingObjects(img, client, logger); err != nil {
   199  			client.Close()
   200  			return err
   201  		}
   202  		err := t.imageDataBase.AddImage(img, name,
   203  			&srpc.AuthInformation{HaveMethodAccess: true})
   204  		if err != nil {
   205  			return err
   206  		}
   207  		return nil
   208  	})
   209  	if err != nil {
   210  		return err
   211  	}
   212  	logger.Println("added image")
   213  	return nil
   214  }
   215  
   216  func (t *srpcType) checkImageBeingInjected(name string) bool {
   217  	t.imagesBeingInjectedLock.Lock()
   218  	defer t.imagesBeingInjectedLock.Unlock()
   219  	_, ok := t.imagesBeingInjected[name]
   220  	return ok
   221  }
   222  
   223  func (t *srpcType) getMissingObjects(img *image.Image, client *srpc.Client,
   224  	logger log.DebugLogger) error {
   225  	objClient := objectclient.AttachObjectClient(client)
   226  	defer objClient.Close()
   227  	return img.GetMissingObjects(t.objSrv, objClient, logger)
   228  }