github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imagebuilder/builder/build.go (about)

     1  package builder
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"fmt"
     7  	"io"
     8  	"strings"
     9  	"time"
    10  
    11  	buildclient "github.com/Cloud-Foundations/Dominator/imagebuilder/client"
    12  	imgclient "github.com/Cloud-Foundations/Dominator/imageserver/client"
    13  	"github.com/Cloud-Foundations/Dominator/lib/format"
    14  	"github.com/Cloud-Foundations/Dominator/lib/image"
    15  	"github.com/Cloud-Foundations/Dominator/lib/srpc"
    16  	proto "github.com/Cloud-Foundations/Dominator/proto/imaginator"
    17  )
    18  
    19  const errNoSourceImage = "no source image: "
    20  const errTooOldSourceImage = "too old source image: "
    21  
    22  type dualBuildLogger struct {
    23  	buffer *bytes.Buffer
    24  	writer io.Writer
    25  }
    26  
    27  func checkPermission(builder imageBuilder, request proto.BuildImageRequest,
    28  	authInfo *srpc.AuthInformation) error {
    29  	if authInfo == nil || authInfo.HaveMethodAccess {
    30  		return nil
    31  	}
    32  	if request.ExpiresIn > time.Hour*24 {
    33  		return errors.New("maximum expiration time is 1 day")
    34  	}
    35  	if builder, ok := builder.(*imageStreamType); ok {
    36  		for _, group := range builder.BuilderGroups {
    37  			if _, ok := authInfo.GroupList[group]; ok {
    38  				return nil
    39  			}
    40  		}
    41  	}
    42  	return errors.New("no permission to build: " + request.StreamName)
    43  }
    44  
    45  func needSourceImage(err error) (bool, string) {
    46  	errString := err.Error()
    47  	if index := strings.Index(errString, errNoSourceImage); index >= 0 {
    48  		return true, errString[index+len(errNoSourceImage):]
    49  	}
    50  	if index := strings.Index(errString, errTooOldSourceImage); index >= 0 {
    51  		return true, errString[index+len(errTooOldSourceImage):]
    52  	}
    53  	return false, ""
    54  }
    55  
    56  func (bl *dualBuildLogger) Bytes() []byte {
    57  	return bl.buffer.Bytes()
    58  }
    59  
    60  func (bl *dualBuildLogger) Write(p []byte) (int, error) {
    61  	return bl.writer.Write(p)
    62  }
    63  
    64  func (b *Builder) rebuildImage(client *srpc.Client, streamName string,
    65  	expiresIn time.Duration) {
    66  	_, _, err := b.build(client, proto.BuildImageRequest{
    67  		StreamName: streamName,
    68  		ExpiresIn:  expiresIn,
    69  	},
    70  		nil, nil)
    71  	if err == nil {
    72  		return
    73  	}
    74  	imageName, _ := imgclient.FindLatestImage(client, streamName, false)
    75  	if imageName != "" {
    76  		e := imgclient.ChangeImageExpiration(client, imageName,
    77  			time.Now().Add(expiresIn))
    78  		if e == nil {
    79  			b.logger.Printf("Error building image: %s: %s, extended: %s\n",
    80  				streamName, err, imageName)
    81  			return
    82  		}
    83  		b.logger.Printf(
    84  			"Error building image: %s: %s, failed to extend: %s: %s\n",
    85  			streamName, err, imageName, e)
    86  		return
    87  	}
    88  	b.logger.Printf("Error building image: %s: %s\n", streamName, err)
    89  }
    90  
    91  func (b *Builder) rebuildImages(minInterval time.Duration) {
    92  	if minInterval < 1 {
    93  		return
    94  	}
    95  	var sleepUntil time.Time
    96  	for ; ; time.Sleep(time.Until(sleepUntil)) {
    97  		sleepUntil = time.Now().Add(minInterval)
    98  		client, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0)
    99  		if err != nil {
   100  			b.logger.Printf("%s: %s\n", b.imageServerAddress, err)
   101  			continue
   102  		}
   103  		for _, streamName := range b.listStreamsToAutoRebuild() {
   104  			b.rebuildImage(client, streamName, minInterval*2)
   105  		}
   106  		client.Close()
   107  	}
   108  }
   109  
   110  func (b *Builder) buildImage(request proto.BuildImageRequest,
   111  	authInfo *srpc.AuthInformation,
   112  	logWriter io.Writer) (*image.Image, string, error) {
   113  	if request.ExpiresIn < time.Minute*15 {
   114  		return nil, "", errors.New("minimum expiration time is 15 minutes")
   115  	}
   116  	client, err := srpc.DialHTTP("tcp", b.imageServerAddress, 0)
   117  	if err != nil {
   118  		return nil, "", err
   119  	}
   120  	defer client.Close()
   121  	img, name, err := b.build(client, request, authInfo, logWriter)
   122  	if request.ReturnImage {
   123  		return img, "", err
   124  	}
   125  	return nil, name, err
   126  }
   127  
   128  func (b *Builder) build(client *srpc.Client, request proto.BuildImageRequest,
   129  	authInfo *srpc.AuthInformation,
   130  	logWriter io.Writer) (*image.Image, string, error) {
   131  	startTime := time.Now()
   132  	builder := b.getImageBuilderWithReload(request.StreamName)
   133  	if builder == nil {
   134  		return nil, "", errors.New("unknown stream: " + request.StreamName)
   135  	}
   136  	if err := checkPermission(builder, request, authInfo); err != nil {
   137  		return nil, "", err
   138  	}
   139  	buildLogBuffer := &bytes.Buffer{}
   140  	b.buildResultsLock.Lock()
   141  	b.currentBuildLogs[request.StreamName] = buildLogBuffer
   142  	b.buildResultsLock.Unlock()
   143  	var buildLog buildLogger
   144  	if logWriter == nil {
   145  		buildLog = buildLogBuffer
   146  	} else {
   147  		buildLog = &dualBuildLogger{
   148  			buffer: buildLogBuffer,
   149  			writer: io.MultiWriter(buildLogBuffer, logWriter),
   150  		}
   151  	}
   152  	img, name, err := b.buildWithLogger(builder, client, request, authInfo,
   153  		startTime, buildLog)
   154  	finishTime := time.Now()
   155  	b.buildResultsLock.Lock()
   156  	defer b.buildResultsLock.Unlock()
   157  	delete(b.currentBuildLogs, request.StreamName)
   158  	b.lastBuildResults[request.StreamName] = buildResultType{
   159  		name, startTime, finishTime, buildLog.Bytes(), err}
   160  	if err == nil {
   161  		b.logger.Printf("Built image for stream: %s in %s\n",
   162  			request.StreamName, format.Duration(finishTime.Sub(startTime)))
   163  	}
   164  	return img, name, err
   165  }
   166  
   167  func (b *Builder) buildSomewhere(builder imageBuilder, client *srpc.Client,
   168  	request proto.BuildImageRequest, authInfo *srpc.AuthInformation,
   169  	buildLog buildLogger) (*image.Image, error) {
   170  	if b.slaveDriver == nil {
   171  		if authInfo == nil {
   172  			b.logger.Printf("Auto building image for stream: %s\n",
   173  				request.StreamName)
   174  		} else {
   175  			b.logger.Printf("%s requested building image for stream: %s\n",
   176  				authInfo.Username, request.StreamName)
   177  		}
   178  		img, err := builder.build(b, client, request, buildLog)
   179  		if err != nil {
   180  			fmt.Fprintf(buildLog, "Error building image: %s\n", err)
   181  		}
   182  		return img, err
   183  	} else {
   184  		return b.buildOnSlave(client, request, authInfo, buildLog)
   185  	}
   186  }
   187  
   188  func (b *Builder) buildWithLogger(builder imageBuilder, client *srpc.Client,
   189  	request proto.BuildImageRequest, authInfo *srpc.AuthInformation,
   190  	startTime time.Time, buildLog buildLogger) (*image.Image, string, error) {
   191  	img, err := b.buildSomewhere(builder, client, request, authInfo, buildLog)
   192  	if err != nil {
   193  		if needSource, sourceImage := needSourceImage(err); needSource {
   194  			if request.DisableRecursiveBuild {
   195  				return nil, "", err
   196  			}
   197  			// Try to build source image.
   198  			expiresIn := time.Hour
   199  			if request.ExpiresIn > 0 {
   200  				expiresIn = request.ExpiresIn
   201  			}
   202  			sourceReq := proto.BuildImageRequest{
   203  				StreamName:   sourceImage,
   204  				ExpiresIn:    expiresIn,
   205  				MaxSourceAge: request.MaxSourceAge,
   206  				Variables:    request.Variables,
   207  			}
   208  			if _, _, e := b.build(client, sourceReq, nil, buildLog); e != nil {
   209  				return nil, "", e
   210  			}
   211  			img, err = b.buildSomewhere(builder, client, request, authInfo,
   212  				buildLog)
   213  		}
   214  	}
   215  	if err != nil {
   216  		return nil, "", err
   217  	}
   218  	if request.ReturnImage {
   219  		return img, "", nil
   220  	}
   221  	uploadStartTime := time.Now()
   222  	if name, err := addImage(client, request, img); err != nil {
   223  		fmt.Fprintln(buildLog, err)
   224  		return nil, "", err
   225  	} else {
   226  		finishTime := time.Now()
   227  		fmt.Fprintf(buildLog,
   228  			"Uploaded %s in %s, total build duration: %s\n",
   229  			name, format.Duration(finishTime.Sub(uploadStartTime)),
   230  			format.Duration(finishTime.Sub(startTime)))
   231  		return img, name, nil
   232  	}
   233  }
   234  
   235  func (b *Builder) buildOnSlave(client *srpc.Client,
   236  	request proto.BuildImageRequest, authInfo *srpc.AuthInformation,
   237  	buildLog buildLogger) (*image.Image, error) {
   238  	request.DisableRecursiveBuild = true
   239  	request.ReturnImage = true
   240  	request.StreamBuildLog = true
   241  	if len(request.Variables) < 1 {
   242  		request.Variables = b.variables
   243  	} else if len(b.variables) > 0 {
   244  		variables := make(map[string]string,
   245  			len(b.variables)+len(request.Variables))
   246  		for key, value := range b.variables {
   247  			variables[key] = value
   248  		}
   249  		for key, value := range request.Variables {
   250  			variables[key] = value
   251  		}
   252  		request.Variables = variables
   253  	}
   254  	slave, err := b.slaveDriver.GetSlave()
   255  	if err != nil {
   256  		return nil, fmt.Errorf("error getting slave: %s", err)
   257  	}
   258  	keepSlave := false
   259  	defer func() {
   260  		if keepSlave {
   261  			slave.Release()
   262  		} else {
   263  			slave.Destroy()
   264  		}
   265  	}()
   266  	if authInfo == nil {
   267  		b.logger.Printf("Auto building image on %s for stream: %s\n",
   268  			slave, request.StreamName)
   269  		fmt.Fprintf(buildLog, "Auto building image on %s for stream: %s\n",
   270  			slave, request.StreamName)
   271  	} else {
   272  		b.logger.Printf("%s requested building image on %s for stream: %s\n",
   273  			authInfo.Username, slave, request.StreamName)
   274  		fmt.Fprintf(buildLog,
   275  			"%s requested building image on %s for stream: %s\n",
   276  			authInfo.Username, slave, request.StreamName)
   277  	}
   278  	var reply proto.BuildImageResponse
   279  	err = buildclient.BuildImage(slave.GetClient(), request, &reply, buildLog)
   280  	if err != nil {
   281  		if needSource, _ := needSourceImage(err); needSource {
   282  			keepSlave = true
   283  		}
   284  		return nil, err
   285  	}
   286  	return reply.Image, nil
   287  }
   288  
   289  func (b *Builder) getImageBuilder(streamName string) imageBuilder {
   290  	if stream := b.getBootstrapStream(streamName); stream != nil {
   291  		return stream
   292  	}
   293  	if stream := b.getNormalStream(streamName); stream != nil {
   294  		return stream
   295  	}
   296  	// Ensure a nil interface is returned, not a stream with value == nil.
   297  	return nil
   298  }
   299  
   300  func (b *Builder) getImageBuilderWithReload(streamName string) imageBuilder {
   301  	if stream := b.getImageBuilder(streamName); stream != nil {
   302  		return stream
   303  	}
   304  	if err := b.reloadNormalStreamsConfiguration(); err != nil {
   305  		b.logger.Printf("Error reloading configuration: %s\n", err)
   306  		return nil
   307  	}
   308  	return b.getImageBuilder(streamName)
   309  }
   310  
   311  func (b *Builder) getCurrentBuildLog(streamName string) ([]byte, error) {
   312  	b.buildResultsLock.RLock()
   313  	defer b.buildResultsLock.RUnlock()
   314  	if result, ok := b.currentBuildLogs[streamName]; !ok {
   315  		return nil, errors.New("unknown image: " + streamName)
   316  	} else {
   317  		log := make([]byte, result.Len())
   318  		copy(log, result.Bytes())
   319  		return log, nil
   320  	}
   321  }
   322  
   323  func (b *Builder) getLatestBuildLog(streamName string) ([]byte, error) {
   324  	b.buildResultsLock.RLock()
   325  	defer b.buildResultsLock.RUnlock()
   326  	if result, ok := b.lastBuildResults[streamName]; !ok {
   327  		return nil, errors.New("unknown image: " + streamName)
   328  	} else {
   329  		log := make([]byte, len(result.buildLog))
   330  		copy(log, result.buildLog)
   331  		return log, nil
   332  	}
   333  }