github.com/projecteru2/core@v0.0.0-20240321043226-06bcc1c23f58/cluster/calcium/realloc.go (about)

     1  package calcium
     2  
     3  import (
     4  	"context"
     5  
     6  	"github.com/sanity-io/litter"
     7  
     8  	"github.com/projecteru2/core/log"
     9  	resourcetypes "github.com/projecteru2/core/resource/types"
    10  	"github.com/projecteru2/core/types"
    11  	"github.com/projecteru2/core/utils"
    12  )
    13  
    14  // ReallocResource updates workload resource dynamically
    15  func (c *Calcium) ReallocResource(ctx context.Context, opts *types.ReallocOptions) (err error) {
    16  	logger := log.WithFunc("calcium.ReallocResource").WithField("opts", opts)
    17  	logger.Infof(ctx, "realloc workload %+v with options %+v", opts.ID, opts.Resources)
    18  	workload, err := c.GetWorkload(ctx, opts.ID)
    19  	if err != nil {
    20  		return
    21  	}
    22  	// copy origin workload
    23  	originWorkload := *workload
    24  	return c.withNodePodLocked(ctx, workload.Nodename, func(ctx context.Context, node *types.Node) error {
    25  		return c.withWorkloadLocked(ctx, opts.ID, false, func(ctx context.Context, workload *types.Workload) error {
    26  			err := c.doReallocOnNode(ctx, node, workload, originWorkload, opts)
    27  			logger.Error(ctx, err)
    28  			return err
    29  		})
    30  	})
    31  }
    32  
    33  func (c *Calcium) doReallocOnNode(ctx context.Context, node *types.Node, workload *types.Workload, originWorkload types.Workload, opts *types.ReallocOptions) error {
    34  	var resources resourcetypes.Resources
    35  	var deltaResources resourcetypes.Resources
    36  	var engineParams resourcetypes.Resources
    37  	var err error
    38  
    39  	logger := log.WithFunc("calcium.doReallocOnNode").WithField("opts", opts)
    40  	err = utils.Txn(
    41  		ctx,
    42  		// if: update workload resource
    43  		func(ctx context.Context) error {
    44  			// note here will change the node resource meta (stored in resource plugin)
    45  			// todo: add wal here
    46  			engineParams, deltaResources, resources, err = c.rmgr.Realloc(ctx, workload.Nodename, workload.Resources, opts.Resources)
    47  			if err != nil {
    48  				return err
    49  			}
    50  			logger.Debugf(ctx, "realloc workload %+v, resource args %+v, engine args %+v", workload.ID, litter.Sdump(resources), litter.Sdump(engineParams))
    51  			workload.EngineParams = engineParams
    52  			workload.Resources = resources
    53  			return c.store.UpdateWorkload(ctx, workload)
    54  		},
    55  		// then: update virtualization
    56  		func(ctx context.Context) error {
    57  			return node.Engine.VirtualizationUpdateResource(ctx, opts.ID, engineParams)
    58  		},
    59  		// rollback: revert the resource changes and rollback workload meta
    60  		func(ctx context.Context, failureByCond bool) error {
    61  			if failureByCond {
    62  				return nil
    63  			}
    64  			if err := c.rmgr.RollbackRealloc(ctx, workload.Nodename, deltaResources); err != nil {
    65  				logger.Errorf(ctx, err, "failed to rollback workload %+v, resource args %+v, engine args %+v", workload.ID, litter.Sdump(resources), litter.Sdump(engineParams))
    66  				// don't return here, so the node resource can still be fixed
    67  			}
    68  			return c.store.UpdateWorkload(ctx, &originWorkload)
    69  		},
    70  		c.config.GlobalTimeout,
    71  	)
    72  	if err != nil {
    73  		return err
    74  	}
    75  	_ = c.pool.Invoke(func() { c.RemapResourceAndLog(ctx, logger, node) })
    76  	return nil
    77  }