github.com/mhilton/juju-juju@v0.0.0-20150901100907-a94dd2c73455/worker/peergrouper/desired.go (about) 1 // Copyright 2014 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package peergrouper 5 6 import ( 7 "fmt" 8 "sort" 9 10 "github.com/juju/loggo" 11 "github.com/juju/replicaset" 12 ) 13 14 // jujuMachineKey is the key for the tag where we save the member's juju machine id. 15 const jujuMachineKey = "juju-machine-id" 16 17 var logger = loggo.GetLogger("juju.worker.peergrouper") 18 19 // peerGroupInfo holds information that may contribute to 20 // a peer group. 21 type peerGroupInfo struct { 22 machines map[string]*machine // id -> machine 23 statuses []replicaset.MemberStatus 24 members []replicaset.Member 25 } 26 27 // desiredPeerGroup returns the mongo peer group according to the given 28 // servers and a map with an element for each machine in info.machines 29 // specifying whether that machine has been configured as voting. It will 30 // return a nil member list and error if the current group is already 31 // correct, though the voting map will be still be returned in that case. 32 func desiredPeerGroup(info *peerGroupInfo) ([]replicaset.Member, map[*machine]bool, error) { 33 if len(info.members) == 0 { 34 return nil, nil, fmt.Errorf("current member set is empty") 35 } 36 changed := false 37 members, extra, maxId := info.membersMap() 38 logger.Debugf("calculating desired peer group") 39 logger.Debugf("members: %#v", members) 40 logger.Debugf("extra: %#v", extra) 41 logger.Debugf("maxId: %v", maxId) 42 43 // We may find extra peer group members if the machines 44 // have been removed or their state server status removed. 45 // This should only happen if they had been set to non-voting 46 // before removal, in which case we want to remove it 47 // from the members list. If we find a member that's still configured 48 // to vote, it's an error. 49 // TODO There are some other possibilities 50 // for what to do in that case. 51 // 1) leave them untouched, but deal 52 // with others as usual "i didn't see that bit" 53 // 2) leave them untouched, deal with others, 54 // but make sure the extras aren't eligible to 55 // be primary. 56 // 3) remove them "get rid of bad rubbish" 57 // 4) do nothing "nothing to see here" 58 for _, member := range extra { 59 if member.Votes == nil || *member.Votes > 0 { 60 return nil, nil, fmt.Errorf("voting non-machine member %#v found in peer group", member) 61 } 62 changed = true 63 } 64 65 toRemoveVote, toAddVote, toKeep := possiblePeerGroupChanges(info, members) 66 67 // Set up initial record of machine votes. Any changes after 68 // this will trigger a peer group election. 69 machineVoting := make(map[*machine]bool) 70 for _, m := range info.machines { 71 member := members[m] 72 machineVoting[m] = member != nil && isVotingMember(member) 73 } 74 setVoting := func(m *machine, voting bool) { 75 setMemberVoting(members[m], voting) 76 machineVoting[m] = voting 77 changed = true 78 } 79 adjustVotes(toRemoveVote, toAddVote, setVoting) 80 81 addNewMembers(members, toKeep, maxId, setVoting) 82 if updateAddresses(members, info.machines) { 83 changed = true 84 } 85 if !changed { 86 return nil, machineVoting, nil 87 } 88 var memberSet []replicaset.Member 89 for _, member := range members { 90 memberSet = append(memberSet, *member) 91 } 92 return memberSet, machineVoting, nil 93 } 94 95 func isVotingMember(member *replicaset.Member) bool { 96 return member.Votes == nil || *member.Votes > 0 97 } 98 99 // possiblePeerGroupChanges returns a set of slices 100 // classifying all the existing machines according to 101 // how their vote might move. 102 // toRemoveVote holds machines whose vote should 103 // be removed; toAddVote holds machines which are 104 // ready to vote; toKeep holds machines with no desired 105 // change to their voting status (this includes machines 106 // that are not yet represented in the peer group). 107 func possiblePeerGroupChanges(info *peerGroupInfo, members map[*machine]*replicaset.Member) (toRemoveVote, toAddVote, toKeep []*machine) { 108 statuses := info.statusesMap(members) 109 110 logger.Debugf("assessing possible peer group changes:") 111 for _, m := range info.machines { 112 member := members[m] 113 isVoting := member != nil && isVotingMember(member) 114 switch { 115 case m.wantsVote && isVoting: 116 logger.Debugf("machine %q is already voting", m.id) 117 toKeep = append(toKeep, m) 118 case m.wantsVote && !isVoting: 119 if status, ok := statuses[m]; ok && isReady(status) { 120 logger.Debugf("machine %q is a potential voter", m.id) 121 toAddVote = append(toAddVote, m) 122 } else { 123 logger.Debugf("machine %q is not ready (has status: %v)", m.id, ok) 124 toKeep = append(toKeep, m) 125 } 126 case !m.wantsVote && isVoting: 127 logger.Debugf("machine %q is a potential non-voter", m.id) 128 toRemoveVote = append(toRemoveVote, m) 129 case !m.wantsVote && !isVoting: 130 logger.Debugf("machine %q does not want the vote", m.id) 131 toKeep = append(toKeep, m) 132 } 133 } 134 logger.Debugf("assessed") 135 // sort machines to be added and removed so that we 136 // get deterministic behaviour when testing. Earlier 137 // entries will be dealt with preferentially, so we could 138 // potentially sort by some other metric in each case. 139 sort.Sort(byId(toRemoveVote)) 140 sort.Sort(byId(toAddVote)) 141 sort.Sort(byId(toKeep)) 142 return toRemoveVote, toAddVote, toKeep 143 } 144 145 // updateAddresses updates the members' addresses from the machines' addresses. 146 // It reports whether any changes have been made. 147 func updateAddresses(members map[*machine]*replicaset.Member, machines map[string]*machine) bool { 148 changed := false 149 // Make sure all members' machine addresses are up to date. 150 for _, m := range machines { 151 hp := m.mongoHostPort() 152 if hp == "" { 153 continue 154 } 155 // TODO ensure that replicaset works correctly with IPv6 [host]:port addresses. 156 if hp != members[m].Address { 157 members[m].Address = hp 158 changed = true 159 } 160 } 161 return changed 162 } 163 164 // adjustVotes adjusts the votes of the given machines, taking 165 // care not to let the total number of votes become even at 166 // any time. It calls setVoting to change the voting status 167 // of a machine. 168 func adjustVotes(toRemoveVote, toAddVote []*machine, setVoting func(*machine, bool)) { 169 // Remove voting members if they can be replaced by 170 // candidates that are ready. This does not affect 171 // the total number of votes. 172 nreplace := min(len(toRemoveVote), len(toAddVote)) 173 for i := 0; i < nreplace; i++ { 174 from := toRemoveVote[i] 175 to := toAddVote[i] 176 setVoting(from, false) 177 setVoting(to, true) 178 } 179 toAddVote = toAddVote[nreplace:] 180 toRemoveVote = toRemoveVote[nreplace:] 181 182 // At this point, one or both of toAdd or toRemove is empty, so 183 // we can adjust the voting-member count by an even delta, 184 // maintaining the invariant that the total vote count is odd. 185 if len(toAddVote) > 0 { 186 toAddVote = toAddVote[0 : len(toAddVote)-len(toAddVote)%2] 187 for _, m := range toAddVote { 188 setVoting(m, true) 189 } 190 } else { 191 toRemoveVote = toRemoveVote[0 : len(toRemoveVote)-len(toRemoveVote)%2] 192 for _, m := range toRemoveVote { 193 setVoting(m, false) 194 } 195 } 196 } 197 198 // addNewMembers adds new members from toKeep 199 // to the given set of members, allocating ids from 200 // maxId upwards. It calls setVoting to set the voting 201 // status of each new member. 202 func addNewMembers( 203 members map[*machine]*replicaset.Member, 204 toKeep []*machine, 205 maxId int, 206 setVoting func(*machine, bool), 207 ) { 208 for _, m := range toKeep { 209 hasAddress := m.mongoHostPort() != "" 210 if members[m] == nil && hasAddress { 211 // This machine was not previously in the members list, 212 // so add it (as non-voting). We maintain the 213 // id manually to make it easier for tests. 214 maxId++ 215 member := &replicaset.Member{ 216 Tags: map[string]string{ 217 jujuMachineKey: m.id, 218 }, 219 Id: maxId, 220 } 221 members[m] = member 222 setVoting(m, false) 223 } else if !hasAddress { 224 logger.Debugf("ignoring machine %q with no address", m.id) 225 } 226 } 227 } 228 229 func isReady(status replicaset.MemberStatus) bool { 230 return status.Healthy && (status.State == replicaset.PrimaryState || 231 status.State == replicaset.SecondaryState) 232 } 233 234 func setMemberVoting(member *replicaset.Member, voting bool) { 235 if voting { 236 member.Votes = nil 237 member.Priority = nil 238 } else { 239 votes := 0 240 member.Votes = &votes 241 priority := 0.0 242 member.Priority = &priority 243 } 244 } 245 246 type byId []*machine 247 248 func (l byId) Len() int { return len(l) } 249 func (l byId) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 250 func (l byId) Less(i, j int) bool { return l[i].id < l[j].id } 251 252 // membersMap returns the replica-set members inside info keyed 253 // by machine. Any members that do not have a corresponding 254 // machine are returned in extra. 255 // The maximum replica-set id is returned in maxId. 256 func (info *peerGroupInfo) membersMap() (members map[*machine]*replicaset.Member, extra []replicaset.Member, maxId int) { 257 maxId = -1 258 members = make(map[*machine]*replicaset.Member) 259 for key := range info.members { 260 // key is used instead of value to have a loop scoped member value 261 member := info.members[key] 262 mid, ok := member.Tags[jujuMachineKey] 263 var found *machine 264 if ok { 265 found = info.machines[mid] 266 } 267 if found != nil { 268 members[found] = &member 269 } else { 270 extra = append(extra, member) 271 } 272 if member.Id > maxId { 273 maxId = member.Id 274 } 275 } 276 return members, extra, maxId 277 } 278 279 // statusesMap returns the statuses inside info keyed by machine. 280 // The provided members map holds the members keyed by machine, 281 // as returned by membersMap. 282 func (info *peerGroupInfo) statusesMap(members map[*machine]*replicaset.Member) map[*machine]replicaset.MemberStatus { 283 statuses := make(map[*machine]replicaset.MemberStatus) 284 for _, status := range info.statuses { 285 for m, member := range members { 286 if member.Id == status.Id { 287 statuses[m] = status 288 break 289 } 290 } 291 } 292 return statuses 293 } 294 295 func min(i, j int) int { 296 if i < j { 297 return i 298 } 299 return j 300 }