github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imageserver/rpcd/replicator.go (about) 1 package rpcd 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "time" 8 9 imageclient "github.com/Cloud-Foundations/Dominator/imageserver/client" 10 "github.com/Cloud-Foundations/Dominator/lib/format" 11 "github.com/Cloud-Foundations/Dominator/lib/image" 12 "github.com/Cloud-Foundations/Dominator/lib/log" 13 "github.com/Cloud-Foundations/Dominator/lib/log/prefixlogger" 14 objectclient "github.com/Cloud-Foundations/Dominator/lib/objectserver/client" 15 "github.com/Cloud-Foundations/Dominator/lib/srpc" 16 "github.com/Cloud-Foundations/Dominator/proto/imageserver" 17 ) 18 19 func (t *srpcType) replicator(finishedReplication chan<- struct{}) { 20 initialTimeout := time.Second * 15 21 timeout := initialTimeout 22 var nextSleepStopTime time.Time 23 for { 24 nextSleepStopTime = time.Now().Add(timeout) 25 if client, err := srpc.DialHTTP("tcp", t.replicationMaster, 26 timeout); err != nil { 27 t.logger.Printf("Error dialing: %s %s\n", t.replicationMaster, err) 28 } else { 29 if conn, err := client.Call( 30 "ImageServer.GetImageUpdates"); err != nil { 31 t.logger.Println(err) 32 } else { 33 if err := t.getUpdates(conn, &finishedReplication); err != nil { 34 if err == io.EOF { 35 t.logger.Println( 36 "Connection to image replicator closed") 37 if nextSleepStopTime.Sub(time.Now()) < 1 { 38 timeout = initialTimeout 39 } 40 } else { 41 t.logger.Println(err) 42 } 43 } 44 conn.Close() 45 } 46 client.Close() 47 } 48 time.Sleep(nextSleepStopTime.Sub(time.Now())) 49 if timeout < time.Minute { 50 timeout *= 2 51 } 52 } 53 } 54 55 func (t *srpcType) getUpdates(conn *srpc.Conn, 56 finishedReplication *chan<- struct{}) error { 57 t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster) 58 replicationStartTime := time.Now() 59 initialImages := make(map[string]struct{}) 60 if t.archiveMode { 61 initialImages = nil 62 } 63 for { 64 var imageUpdate imageserver.ImageUpdate 65 if err := conn.Decode(&imageUpdate); err != nil { 66 if err == io.EOF { 67 return err 68 } 69 return errors.New("decode err: " + err.Error()) 70 } 71 switch imageUpdate.Operation { 72 case imageserver.OperationAddImage: 73 if imageUpdate.Name == "" { 74 if initialImages != nil { 75 t.deleteMissingImages(initialImages) 76 initialImages = nil 77 } 78 if *finishedReplication != nil { 79 close(*finishedReplication) 80 *finishedReplication = nil 81 } 82 t.logger.Printf("Replicated all current images in %s\n", 83 format.Duration(time.Since(replicationStartTime))) 84 continue 85 } 86 if initialImages != nil { 87 initialImages[imageUpdate.Name] = struct{}{} 88 } 89 if err := t.addImage(imageUpdate.Name); err != nil { 90 return errors.New("error adding image: " + imageUpdate.Name + 91 ": " + err.Error()) 92 } 93 case imageserver.OperationDeleteImage: 94 if t.archiveMode { 95 continue 96 } 97 t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name) 98 err := t.imageDataBase.DeleteImage(imageUpdate.Name, 99 &srpc.AuthInformation{HaveMethodAccess: true}) 100 if err != nil { 101 return err 102 } 103 case imageserver.OperationMakeDirectory: 104 directory := imageUpdate.Directory 105 if directory == nil { 106 return errors.New("nil imageUpdate.Directory") 107 } 108 if err := t.imageDataBase.UpdateDirectory(*directory); err != nil { 109 return err 110 } 111 } 112 } 113 } 114 115 func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) { 116 missingImages := make([]string, 0) 117 for _, imageName := range t.imageDataBase.ListImages() { 118 if _, ok := imagesToKeep[imageName]; !ok { 119 missingImages = append(missingImages, imageName) 120 } 121 } 122 for _, imageName := range missingImages { 123 t.logger.Printf("Replicator(%s): delete missing image\n", imageName) 124 err := t.imageDataBase.DeleteImage(imageName, 125 &srpc.AuthInformation{HaveMethodAccess: true}) 126 if err != nil { 127 t.logger.Println(err) 128 } 129 } 130 } 131 132 func (t *srpcType) extendImageExpiration(name string, 133 img *image.Image) (bool, error) { 134 timeout := time.Second * 60 135 client, err := t.imageserverResource.GetHTTP(nil, timeout) 136 if err != nil { 137 return false, err 138 } 139 defer client.Put() 140 expiresAt, err := imageclient.GetImageExpiration(client, name) 141 if err != nil { 142 if err == io.EOF { 143 client.Close() 144 } 145 return false, err 146 } 147 return t.imageDataBase.ChangeImageExpiration(name, expiresAt, 148 &srpc.AuthInformation{HaveMethodAccess: true}) 149 } 150 151 func (t *srpcType) addImage(name string) error { 152 timeout := time.Second * 60 153 if t.checkImageBeingInjected(name) { 154 return nil 155 } 156 logger := prefixlogger.New(fmt.Sprintf("Replicator(%s): ", name), t.logger) 157 if img := t.imageDataBase.GetImage(name); img != nil { 158 if img.ExpiresAt.IsZero() { 159 return nil 160 } 161 if changed, err := t.extendImageExpiration(name, img); err != nil { 162 logger.Println(err) 163 } else if changed { 164 logger.Println("extended expiration time") 165 } 166 return nil 167 } 168 logger.Println("add image") 169 client, err := t.imageserverResource.GetHTTP(nil, timeout) 170 if err != nil { 171 return err 172 } 173 defer client.Put() 174 request := imageserver.GetImageRequest{ 175 ImageName: name, 176 Timeout: timeout, 177 } 178 if t.archiveMode && !*archiveExpiringImages { 179 request.IgnoreFilesystemIfExpiring = true 180 } 181 var reply imageserver.GetImageResponse 182 err = client.RequestReply("ImageServer.GetImage", request, &reply) 183 if err != nil { 184 client.Close() 185 return err 186 } 187 img := reply.Image 188 if img == nil { 189 return errors.New(name + ": not found") 190 } 191 logger.Println("downloaded image") 192 if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages { 193 logger.Println("ignoring expiring image in archiver mode") 194 return nil 195 } 196 img.FileSystem.RebuildInodePointers() 197 err = t.imageDataBase.DoWithPendingImage(img, func() error { 198 if err := t.getMissingObjects(img, client, logger); err != nil { 199 client.Close() 200 return err 201 } 202 err := t.imageDataBase.AddImage(img, name, 203 &srpc.AuthInformation{HaveMethodAccess: true}) 204 if err != nil { 205 return err 206 } 207 return nil 208 }) 209 if err != nil { 210 return err 211 } 212 logger.Println("added image") 213 return nil 214 } 215 216 func (t *srpcType) checkImageBeingInjected(name string) bool { 217 t.imagesBeingInjectedLock.Lock() 218 defer t.imagesBeingInjectedLock.Unlock() 219 _, ok := t.imagesBeingInjected[name] 220 return ok 221 } 222 223 func (t *srpcType) getMissingObjects(img *image.Image, client *srpc.Client, 224 logger log.DebugLogger) error { 225 objClient := objectclient.AttachObjectClient(client) 226 defer objClient.Close() 227 return img.GetMissingObjects(t.objSrv, objClient, logger) 228 }