github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/lib/filegen/client/machine.go (about)

     1  package client
     2  
     3  import (
     4  	"reflect"
     5  
     6  	"github.com/Cloud-Foundations/Dominator/lib/hash"
     7  	proto "github.com/Cloud-Foundations/Dominator/proto/filegenerator"
     8  )
     9  
    10  var zeroHash hash.Hash
    11  
    12  func buildMachine(machine Machine) *machineType {
    13  	computedFiles := make(map[string]string, len(machine.ComputedFiles))
    14  	sourceToPaths := make(map[string][]string)
    15  	for _, computedFile := range machine.ComputedFiles {
    16  		computedFiles[computedFile.Pathname] = computedFile.Source
    17  		pathnames := sourceToPaths[computedFile.Source]
    18  		pathnames = append(pathnames, computedFile.Pathname)
    19  		sourceToPaths[computedFile.Source] = pathnames
    20  	}
    21  	return &machineType{
    22  		machine:       machine.Machine,
    23  		computedFiles: computedFiles,
    24  		sourceToPaths: sourceToPaths}
    25  }
    26  
    27  func (m *Manager) addMachine(machine *machineType) {
    28  	hostname := machine.machine.Hostname
    29  	_, ok := m.machineMap[hostname]
    30  	if ok {
    31  		panic(hostname + ": already added")
    32  	}
    33  	m.machineMap[hostname] = machine
    34  	m.sendYieldRequests(machine)
    35  }
    36  
    37  func (m *Manager) removeMachine(hostname string) {
    38  	if machine, ok := m.machineMap[hostname]; !ok {
    39  		panic(hostname + ": not present")
    40  	} else {
    41  		delete(m.machineMap, hostname)
    42  		close(machine.updateChannel)
    43  	}
    44  }
    45  
    46  func (m *Manager) updateMachine(machine *machineType) {
    47  	hostname := machine.machine.Hostname
    48  	if mapMachine, ok := m.machineMap[hostname]; !ok {
    49  		panic(hostname + ": not present")
    50  	} else {
    51  		sendRequests := false
    52  		if !machine.machine.Compare(mapMachine.machine) {
    53  			mapMachine.machine = machine.machine
    54  			sendRequests = true
    55  		}
    56  		if !reflect.DeepEqual(machine.computedFiles, mapMachine.computedFiles) {
    57  			sendRequests = true
    58  			mapMachine.computedFiles = machine.computedFiles
    59  		}
    60  		if !reflect.DeepEqual(machine.sourceToPaths, mapMachine.sourceToPaths) {
    61  			sendRequests = true
    62  			mapMachine.sourceToPaths = machine.sourceToPaths
    63  		}
    64  		if sendRequests {
    65  			m.sendYieldRequests(mapMachine)
    66  		}
    67  	}
    68  }
    69  
    70  func (m *Manager) sendYieldRequests(machine *machineType) {
    71  	for sourceName, pathnames := range machine.sourceToPaths {
    72  		request := &proto.ClientRequest{
    73  			YieldRequest: &proto.YieldRequest{machine.machine, pathnames}}
    74  		m.getSource(sourceName).sendChannel <- request
    75  	}
    76  }
    77  
    78  func (m *Manager) handleYieldResponse(machine *machineType,
    79  	files []proto.FileInfo) {
    80  	objectsToWaitFor := make(map[hash.Hash]struct{})
    81  	waiterChannel := make(chan hash.Hash)
    82  	// Get list of objects to wait for.
    83  	for _, file := range files {
    84  		sourceName, ok := machine.computedFiles[file.Pathname]
    85  		if !ok {
    86  			m.logger.Printf("no source name for: %s on: %s\n",
    87  				file.Pathname, machine.machine.Hostname)
    88  			continue
    89  		}
    90  		source, ok := m.sourceMap[sourceName]
    91  		if !ok {
    92  			panic("no source for: " + sourceName)
    93  		}
    94  		if file.Hash == zeroHash {
    95  			m.logger.Printf("Received zero hash for machine: %s file: %s\n",
    96  				machine.machine.Hostname, file.Pathname)
    97  			continue // No object.
    98  		}
    99  		hashes := []hash.Hash{file.Hash}
   100  		if lengths, err := m.objectServer.CheckObjects(hashes); err != nil {
   101  			panic(err)
   102  		} else if lengths[0] < 1 {
   103  			// Not already in objectserver, so add to the list to wait for.
   104  			request := &proto.ClientRequest{
   105  				GetObjectRequest: &proto.GetObjectRequest{file.Hash}}
   106  			source.sendChannel <- request
   107  			objectsToWaitFor[file.Hash] = struct{}{}
   108  			if _, ok := m.objectWaiters[file.Hash]; !ok {
   109  				m.objectWaiters[file.Hash] = make([]chan<- hash.Hash, 0, 1)
   110  			}
   111  			m.objectWaiters[file.Hash] = append(m.objectWaiters[file.Hash],
   112  				waiterChannel)
   113  		}
   114  	}
   115  	if len(objectsToWaitFor) > 0 {
   116  		go waitForObjectsAndSendUpdate(waiterChannel, objectsToWaitFor,
   117  			machine.updateChannel, files, &m.numObjectWaiters)
   118  	} else {
   119  		machine.updateChannel <- files
   120  	}
   121  }
   122  
   123  func waitForObjectsAndSendUpdate(objectChannel <-chan hash.Hash,
   124  	objectsToWaitFor map[hash.Hash]struct{},
   125  	updateChannel chan<- []proto.FileInfo, files []proto.FileInfo,
   126  	numObjectWaiters *gauge) {
   127  	numObjectWaiters.increment()
   128  	defer func() {
   129  		numObjectWaiters.decrement()
   130  		recover() // If updateChannel is closed, it means the machine went away.
   131  	}()
   132  	for hashVal := range objectChannel {
   133  		delete(objectsToWaitFor, hashVal)
   134  		if len(objectsToWaitFor) < 1 {
   135  			updateChannel <- files // This will panic if the machine went away.
   136  		}
   137  	}
   138  }
   139  
   140  func (g *gauge) decrement() {
   141  	g.Lock()
   142  	g.value--
   143  	g.Unlock()
   144  }
   145  
   146  func (g *gauge) increment() {
   147  	g.Lock()
   148  	g.value++
   149  	g.Unlock()
   150  }