github.com/Cloud-Foundations/Dominator@v0.3.4/dom/herd/subdInstaller.go (about)

     1  package herd
     2  
     3  import (
     4  	"bytes"
     5  	"os/exec"
     6  	"runtime"
     7  	"time"
     8  )
     9  
    10  var (
    11  	carriageReturnLiteral = []byte{'\r'}
    12  	newlineLiteral        = []byte{'\n'}
    13  	newlineReplacement    = []byte{'\\', 'n'}
    14  )
    15  
    16  type completionType struct {
    17  	failed   bool
    18  	hostname string
    19  }
    20  
    21  type installerQueueType struct {
    22  	entries map[string]*queueEntry // Key: subHostname (nil: processing).
    23  	first   *queueEntry
    24  	last    *queueEntry
    25  }
    26  
    27  type queueEntry struct {
    28  	startTime time.Time
    29  	hostname  string
    30  	prev      *queueEntry
    31  	next      *queueEntry
    32  }
    33  
    34  func (herd *Herd) subdInstallerLoop() {
    35  	if *subdInstaller == "" {
    36  		return
    37  	}
    38  	availableSlots := runtime.NumCPU()
    39  	completion := make(chan completionType, 1)
    40  	queueAdd := make(chan string, 1)
    41  	herd.subdInstallerQueueAdd = queueAdd
    42  	queueDelete := make(chan string, 1)
    43  	herd.subdInstallerQueueDelete = queueDelete
    44  	queueErase := make(chan string, 1)
    45  	herd.subdInstallerQueueErase = queueErase
    46  	queue := installerQueueType{entries: make(map[string]*queueEntry)}
    47  	for {
    48  		sleepInterval := time.Hour
    49  		if queue.first != nil && availableSlots > 0 {
    50  			sleepInterval = time.Until(queue.first.startTime)
    51  		}
    52  		timer := time.NewTimer(sleepInterval)
    53  		select {
    54  		case <-timer.C:
    55  		case hostname := <-queueAdd:
    56  			if _, ok := queue.entries[hostname]; !ok {
    57  				entry := &queueEntry{
    58  					startTime: time.Now().Add(*subdInstallDelay),
    59  					hostname:  hostname,
    60  					prev:      queue.last,
    61  				}
    62  				queue.add(entry)
    63  			}
    64  		case hostname := <-queueDelete:
    65  			if entry := queue.entries[hostname]; entry != nil {
    66  				queue.delete(entry)
    67  				delete(queue.entries, hostname)
    68  			}
    69  		case hostname := <-queueErase:
    70  			if entry := queue.entries[hostname]; entry != nil {
    71  				queue.delete(entry)
    72  			}
    73  			delete(queue.entries, hostname)
    74  		case result := <-completion:
    75  			availableSlots++
    76  			if _, ok := queue.entries[result.hostname]; ok { // Not erased.
    77  				delete(queue.entries, result.hostname)
    78  				if result.failed && *subdInstallRetryDelay > *subdInstallDelay {
    79  					// Come back later rather than sooner, must re-inject now.
    80  					entry := &queueEntry{
    81  						startTime: time.Now().Add(*subdInstallRetryDelay),
    82  						hostname:  result.hostname,
    83  						prev:      queue.last,
    84  					}
    85  					queue.add(entry)
    86  				}
    87  			}
    88  		}
    89  		timer.Stop()
    90  		entry := queue.first
    91  		if entry != nil &&
    92  			availableSlots > 0 &&
    93  			time.Since(entry.startTime) >= 0 {
    94  			availableSlots--
    95  			go herd.subInstall(entry.hostname, completion)
    96  			queue.delete(entry)
    97  			queue.entries[entry.hostname] = nil // Mark as processing.
    98  		}
    99  	}
   100  }
   101  
   102  func (herd *Herd) addSubToInstallerQueue(subHostname string) {
   103  	if herd.subdInstallerQueueAdd != nil {
   104  		herd.subdInstallerQueueAdd <- subHostname
   105  	}
   106  }
   107  
   108  func (herd *Herd) eraseSubFromInstallerQueue(subHostname string) {
   109  	if herd.subdInstallerQueueErase != nil {
   110  		herd.subdInstallerQueueErase <- subHostname
   111  	}
   112  }
   113  
   114  func (herd *Herd) removeSubFromInstallerQueue(subHostname string) {
   115  	if herd.subdInstallerQueueDelete != nil {
   116  		herd.subdInstallerQueueDelete <- subHostname
   117  	}
   118  }
   119  
   120  func (herd *Herd) subInstall(subHostname string,
   121  	completion chan<- completionType) {
   122  	failed := false
   123  	defer func() { completion <- completionType{failed, subHostname} }()
   124  	herd.logger.Printf("Installing subd on: %s\n", subHostname)
   125  	cmd := exec.Command(*subdInstaller, subHostname)
   126  	output, err := cmd.CombinedOutput()
   127  	if err != nil {
   128  		failed = true
   129  		if len(output) > 0 && output[len(output)-1] == '\n' {
   130  			output = output[:len(output)-1]
   131  		}
   132  		output = bytes.ReplaceAll(output, carriageReturnLiteral, nil)
   133  		output = bytes.ReplaceAll(output, newlineLiteral, newlineReplacement)
   134  		herd.logger.Printf("Error installing subd on: %s: %s: %s\n",
   135  			subHostname, err, string(output))
   136  	}
   137  }
   138  
   139  func (queue *installerQueueType) add(entry *queueEntry) {
   140  	entry.prev = queue.last
   141  	if queue.first == nil {
   142  		queue.first = entry
   143  	} else {
   144  		queue.last.next = entry
   145  	}
   146  	queue.last = entry
   147  	queue.entries[entry.hostname] = entry
   148  }
   149  
   150  func (queue *installerQueueType) delete(entry *queueEntry) {
   151  	if entry.prev == nil {
   152  		queue.first = entry.next
   153  	} else {
   154  		entry.prev.next = entry.next
   155  	}
   156  	if entry.next == nil {
   157  		queue.last = entry.prev
   158  	} else {
   159  		entry.next.prev = entry.prev
   160  	}
   161  }