github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imagebuilder/builder/build.go (about) 1 package builder 2 3 import ( 4 "bytes" 5 "errors" 6 "fmt" 7 "io" 8 "strings" 9 "time" 10 11 buildclient "github.com/Cloud-Foundations/Dominator/imagebuilder/client" 12 imgclient "github.com/Cloud-Foundations/Dominator/imageserver/client" 13 "github.com/Cloud-Foundations/Dominator/lib/format" 14 "github.com/Cloud-Foundations/Dominator/lib/image" 15 "github.com/Cloud-Foundations/Dominator/lib/srpc" 16 proto "github.com/Cloud-Foundations/Dominator/proto/imaginator" 17 ) 18 19 const errNoSourceImage = "no source image: " 20 const errTooOldSourceImage = "too old source image: " 21 22 type dualBuildLogger struct { 23 buffer *bytes.Buffer 24 writer io.Writer 25 } 26 27 func checkPermission(builder imageBuilder, request proto.BuildImageRequest, 28 authInfo *srpc.AuthInformation) error { 29 if authInfo == nil || authInfo.HaveMethodAccess { 30 return nil 31 } 32 if request.ExpiresIn > time.Hour*24 { 33 return errors.New("maximum expiration time is 1 day") 34 } 35 if builder, ok := builder.(*imageStreamType); ok { 36 for _, group := range builder.BuilderGroups { 37 if _, ok := authInfo.GroupList[group]; ok { 38 return nil 39 } 40 } 41 } 42 return errors.New("no permission to build: " + request.StreamName) 43 } 44 45 func needSourceImage(err error) (bool, string) { 46 errString := err.Error() 47 if index := strings.Index(errString, errNoSourceImage); index >= 0 { 48 return true, errString[index+len(errNoSourceImage):] 49 } 50 if index := strings.Index(errString, errTooOldSourceImage); index >= 0 { 51 return true, errString[index+len(errTooOldSourceImage):] 52 } 53 return false, "" 54 } 55 56 func (bl *dualBuildLogger) Bytes() []byte { 57 return bl.buffer.Bytes() 58 } 59 60 func (bl *dualBuildLogger) Write(p []byte) (int, error) { 61 return bl.writer.Write(p) 62 } 63 64 func (b *Builder) rebuildImage(client *srpc.Client, streamName string, 65 expiresIn time.Duration) { 66 _, _, err := b.build(client, proto.BuildImageRequest{ 67 StreamName: streamName, 68 ExpiresIn: expiresIn, 69 }, 70 nil, nil) 71 if err == nil { 72 return 73 } 74 imageName, _ := imgclient.FindLatestImage(client, streamName, false) 75 if imageName != "" { 76 e := imgclient.ChangeImageExpiration(client, imageName, 77 time.Now().Add(expiresIn)) 78 if e == nil { 79 b.logger.Printf("Error building image: %s: %s, extended: %s\n", 80 streamName, err, imageName) 81 return 82 } 83 b.logger.Printf( 84 "Error building image: %s: %s, failed to extend: %s: %s\n", 85 streamName, err, imageName, e) 86 return 87 } 88 b.logger.Printf("Error building image: %s: %s\n", streamName, err) 89 } 90 91 func (b *Builder) rebuildImages(minInterval time.Duration) { 92 if minInterval < 1 { 93 return 94 } 95 var sleepUntil time.Time 96 for ; ; time.Sleep(time.Until(sleepUntil)) { 97 sleepUntil = time.Now().Add(minInterval) 98 client, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0) 99 if err != nil { 100 b.logger.Printf("%s: %s\n", b.imageServerAddress, err) 101 continue 102 } 103 for _, streamName := range b.listStreamsToAutoRebuild() { 104 b.rebuildImage(client, streamName, minInterval*2) 105 } 106 client.Close() 107 } 108 } 109 110 func (b *Builder) buildImage(request proto.BuildImageRequest, 111 authInfo *srpc.AuthInformation, 112 logWriter io.Writer) (*image.Image, string, error) { 113 if request.ExpiresIn < time.Minute*15 { 114 return nil, "", errors.New("minimum expiration time is 15 minutes") 115 } 116 client, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0) 117 if err != nil { 118 return nil, "", err 119 } 120 defer client.Close() 121 img, name, err := b.build(client, request, authInfo, logWriter) 122 if request.ReturnImage { 123 return img, "", err 124 } 125 return nil, name, err 126 } 127 128 func (b *Builder) build(client *srpc.Client, request proto.BuildImageRequest, 129 authInfo *srpc.AuthInformation, 130 logWriter io.Writer) (*image.Image, string, error) { 131 startTime := time.Now() 132 builder := b.getImageBuilderWithReload(request.StreamName) 133 if builder == nil { 134 return nil, "", errors.New("unknown stream: " + request.StreamName) 135 } 136 if err := checkPermission(builder, request, authInfo); err != nil { 137 return nil, "", err 138 } 139 buildLogBuffer := &bytes.Buffer{} 140 b.buildResultsLock.Lock() 141 b.currentBuildLogs[request.StreamName] = buildLogBuffer 142 b.buildResultsLock.Unlock() 143 var buildLog buildLogger 144 if logWriter == nil { 145 buildLog = buildLogBuffer 146 } else { 147 buildLog = &dualBuildLogger{ 148 buffer: buildLogBuffer, 149 writer: io.MultiWriter(buildLogBuffer, logWriter), 150 } 151 } 152 img, name, err := b.buildWithLogger(builder, client, request, authInfo, 153 startTime, buildLog) 154 finishTime := time.Now() 155 b.buildResultsLock.Lock() 156 defer b.buildResultsLock.Unlock() 157 delete(b.currentBuildLogs, request.StreamName) 158 b.lastBuildResults[request.StreamName] = buildResultType{ 159 name, startTime, finishTime, buildLog.Bytes(), err} 160 if err == nil { 161 b.logger.Printf("Built image for stream: %s in %s\n", 162 request.StreamName, format.Duration(finishTime.Sub(startTime))) 163 } 164 return img, name, err 165 } 166 167 func (b *Builder) buildSomewhere(builder imageBuilder, client *srpc.Client, 168 request proto.BuildImageRequest, authInfo *srpc.AuthInformation, 169 buildLog buildLogger) (*image.Image, error) { 170 if b.slaveDriver == nil { 171 if authInfo == nil { 172 b.logger.Printf("Auto building image for stream: %s\n", 173 request.StreamName) 174 } else { 175 b.logger.Printf("%s requested building image for stream: %s\n", 176 authInfo.Username, request.StreamName) 177 } 178 img, err := builder.build(b, client, request, buildLog) 179 if err != nil { 180 fmt.Fprintf(buildLog, "Error building image: %s\n", err) 181 } 182 return img, err 183 } else { 184 return b.buildOnSlave(client, request, authInfo, buildLog) 185 } 186 } 187 188 func (b *Builder) buildWithLogger(builder imageBuilder, client *srpc.Client, 189 request proto.BuildImageRequest, authInfo *srpc.AuthInformation, 190 startTime time.Time, buildLog buildLogger) (*image.Image, string, error) { 191 img, err := b.buildSomewhere(builder, client, request, authInfo, buildLog) 192 if err != nil { 193 if needSource, sourceImage := needSourceImage(err); needSource { 194 if request.DisableRecursiveBuild { 195 return nil, "", err 196 } 197 // Try to build source image. 198 expiresIn := time.Hour 199 if request.ExpiresIn > 0 { 200 expiresIn = request.ExpiresIn 201 } 202 sourceReq := proto.BuildImageRequest{ 203 StreamName: sourceImage, 204 ExpiresIn: expiresIn, 205 MaxSourceAge: request.MaxSourceAge, 206 Variables: request.Variables, 207 } 208 if _, _, e := b.build(client, sourceReq, nil, buildLog); e != nil { 209 return nil, "", e 210 } 211 img, err = b.buildSomewhere(builder, client, request, authInfo, 212 buildLog) 213 } 214 } 215 if err != nil { 216 return nil, "", err 217 } 218 if request.ReturnImage { 219 return img, "", nil 220 } 221 uploadStartTime := time.Now() 222 if name, err := addImage(client, request, img); err != nil { 223 fmt.Fprintln(buildLog, err) 224 return nil, "", err 225 } else { 226 finishTime := time.Now() 227 fmt.Fprintf(buildLog, 228 "Uploaded %s in %s, total build duration: %s\n", 229 name, format.Duration(finishTime.Sub(uploadStartTime)), 230 format.Duration(finishTime.Sub(startTime))) 231 return img, name, nil 232 } 233 } 234 235 func (b *Builder) buildOnSlave(client *srpc.Client, 236 request proto.BuildImageRequest, authInfo *srpc.AuthInformation, 237 buildLog buildLogger) (*image.Image, error) { 238 request.DisableRecursiveBuild = true 239 request.ReturnImage = true 240 request.StreamBuildLog = true 241 if len(request.Variables) < 1 { 242 request.Variables = b.variables 243 } else if len(b.variables) > 0 { 244 variables := make(map[string]string, 245 len(b.variables)+len(request.Variables)) 246 for key, value := range b.variables { 247 variables[key] = value 248 } 249 for key, value := range request.Variables { 250 variables[key] = value 251 } 252 request.Variables = variables 253 } 254 slave, err := b.slaveDriver.GetSlave() 255 if err != nil { 256 return nil, fmt.Errorf("error getting slave: %s", err) 257 } 258 keepSlave := false 259 defer func() { 260 if keepSlave { 261 slave.Release() 262 } else { 263 slave.Destroy() 264 } 265 }() 266 if authInfo == nil { 267 b.logger.Printf("Auto building image on %s for stream: %s\n", 268 slave, request.StreamName) 269 fmt.Fprintf(buildLog, "Auto building image on %s for stream: %s\n", 270 slave, request.StreamName) 271 } else { 272 b.logger.Printf("%s requested building image on %s for stream: %s\n", 273 authInfo.Username, slave, request.StreamName) 274 fmt.Fprintf(buildLog, 275 "%s requested building image on %s for stream: %s\n", 276 authInfo.Username, slave, request.StreamName) 277 } 278 var reply proto.BuildImageResponse 279 err = buildclient.BuildImage(slave.GetClient(), request, &reply, buildLog) 280 if err != nil { 281 if needSource, _ := needSourceImage(err); needSource { 282 keepSlave = true 283 } 284 return nil, err 285 } 286 return reply.Image, nil 287 } 288 289 func (b *Builder) getImageBuilder(streamName string) imageBuilder { 290 if stream := b.getBootstrapStream(streamName); stream != nil { 291 return stream 292 } 293 if stream := b.getNormalStream(streamName); stream != nil { 294 return stream 295 } 296 // Ensure a nil interface is returned, not a stream with value == nil. 297 return nil 298 } 299 300 func (b *Builder) getImageBuilderWithReload(streamName string) imageBuilder { 301 if stream := b.getImageBuilder(streamName); stream != nil { 302 return stream 303 } 304 if err := b.reloadNormalStreamsConfiguration(); err != nil { 305 b.logger.Printf("Error reloading configuration: %s\n", err) 306 return nil 307 } 308 return b.getImageBuilder(streamName) 309 } 310 311 func (b *Builder) getCurrentBuildLog(streamName string) ([]byte, error) { 312 b.buildResultsLock.RLock() 313 defer b.buildResultsLock.RUnlock() 314 if result, ok := b.currentBuildLogs[streamName]; !ok { 315 return nil, errors.New("unknown image: " + streamName) 316 } else { 317 log := make([]byte, result.Len()) 318 copy(log, result.Bytes()) 319 return log, nil 320 } 321 } 322 323 func (b *Builder) getLatestBuildLog(streamName string) ([]byte, error) { 324 b.buildResultsLock.RLock() 325 defer b.buildResultsLock.RUnlock() 326 if result, ok := b.lastBuildResults[streamName]; !ok { 327 return nil, errors.New("unknown image: " + streamName) 328 } else { 329 log := make([]byte, len(result.buildLog)) 330 copy(log, result.buildLog) 331 return log, nil 332 } 333 }