github.com/hugh712/snapd@v0.0.0-20200910133618-1a99902bd583/asserts/pool.go (about) 1 // -*- Mode: Go; indent-tabs-mode: t -*- 2 3 /* 4 * Copyright (C) 2020 Canonical Ltd 5 * 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 3 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 package asserts 21 22 import ( 23 "errors" 24 "fmt" 25 26 "github.com/snapcore/snapd/asserts/internal" 27 ) 28 29 // A Grouping identifies opaquely a grouping of assertions. 30 // Pool uses it to label the interesection between a set of groups. 31 type Grouping string 32 33 // A pool helps holding and tracking a set of assertions and their 34 // prerequisites as they need to be updated or resolved. The 35 // assertions can be organized in groups. Failure can be tracked 36 // isolated to groups, conversely any error related to a single group 37 // alone will stop any work to resolve it. Independent assertions 38 // should not be grouped. Assertions and prerequisites that are part 39 // of more than one group are tracked properly only once. 40 // 41 // Typical usage involves specifying the initial assertions needing to 42 // be resolved or updated using AddUnresolved and AddToUpdate. At this 43 // point ToResolve can be called to get them organized in groupings 44 // ready for fetching. Fetched assertions can then be provided with 45 // Add or AddBatch. Because these can have prerequisites calling 46 // ToResolve and fetching needs to be repeated until ToResolve's 47 // result is empty. Between any two ToResolve invocations but after 48 // any Add or AddBatch AddUnresolved/AddToUpdate can also be used 49 // again. 50 // 51 // V 52 // | 53 // /-> AddUnresolved, AddToUpdate 54 // | | 55 // | V 56 // |------> ToResolve -> empty? done 57 // | | 58 // | V 59 // \ __________ Add 60 // 61 // 62 // If errors prevent from fulfilling assertions from a ToResolve, 63 // AddError and AddGroupingError can be used to report the errors so 64 // that they can be associated with groups. 65 // 66 // All the resolved assertions in a Pool from groups not in error can 67 // be committed to a destination database with CommitTo. 68 type Pool struct { 69 groundDB RODatabase 70 71 numbering map[string]uint16 72 groupings *internal.Groupings 73 74 unresolved map[string]*unresolvedRec 75 prerequisites map[string]*unresolvedRec 76 77 bs Backstore 78 groups map[uint16]*groupRec 79 80 curPhase poolPhase 81 } 82 83 // NewPool creates a new Pool, groundDB is used to resolve trusted and 84 // predefined assertions and to provide the current revision for 85 // assertions to update and their prerequisites. Up to n groups can be 86 // used to organize the assertions. 87 func NewPool(groundDB RODatabase, n int) *Pool { 88 groupings, err := internal.NewGroupings(n) 89 if err != nil { 90 panic(fmt.Sprintf("NewPool: %v", err)) 91 } 92 return &Pool{ 93 groundDB: groundDB, 94 numbering: make(map[string]uint16), 95 groupings: groupings, 96 unresolved: make(map[string]*unresolvedRec), 97 prerequisites: make(map[string]*unresolvedRec), 98 bs: NewMemoryBackstore(), 99 groups: make(map[uint16]*groupRec), 100 } 101 } 102 103 func (p *Pool) groupNum(group string) (gnum uint16, err error) { 104 if gnum, ok := p.numbering[group]; ok { 105 return gnum, nil 106 } 107 gnum = uint16(len(p.numbering)) 108 if err = p.groupings.WithinRange(gnum); err != nil { 109 return 0, err 110 } 111 p.numbering[group] = gnum 112 return gnum, nil 113 } 114 115 func (p *Pool) ensureGroup(group string) (gnum uint16, err error) { 116 gnum, err = p.groupNum(group) 117 if err != nil { 118 return 0, err 119 } 120 if gRec := p.groups[gnum]; gRec == nil { 121 gRec = new(groupRec) 122 p.groups[gnum] = gRec 123 } 124 return gnum, nil 125 } 126 127 // Singleton returns a grouping containing only the given group. 128 // It is useful mainly for tests and to drive Add are AddBatch when the 129 // server is pushing assertions (instead of the usual pull scenario). 130 func (p *Pool) Singleton(group string) (Grouping, error) { 131 gnum, err := p.ensureGroup(group) 132 if err != nil { 133 return Grouping(""), nil 134 } 135 136 var grouping internal.Grouping 137 p.groupings.AddTo(&grouping, gnum) 138 return Grouping(p.groupings.Serialize(&grouping)), nil 139 } 140 141 // An unresolvedRec tracks a single unresolved assertion until it is 142 // resolved or there is an error doing so. The field 'grouping' will 143 // grow to contain all the groups requiring this assertion while it 144 // is unresolved. 145 type unresolvedRec struct { 146 at *AtRevision 147 grouping internal.Grouping 148 149 serializedLabel Grouping 150 151 err error 152 } 153 154 func (u *unresolvedRec) exportTo(r map[Grouping][]*AtRevision, gr *internal.Groupings) { 155 serLabel := Grouping(gr.Serialize(&u.grouping)) 156 // remember serialized label 157 u.serializedLabel = serLabel 158 r[serLabel] = append(r[serLabel], u.at) 159 } 160 161 func (u *unresolvedRec) merge(at *AtRevision, gnum uint16, gr *internal.Groupings) { 162 gr.AddTo(&u.grouping, gnum) 163 // assume we want to resolve/update wrt the highest revision 164 if at.Revision > u.at.Revision { 165 u.at.Revision = at.Revision 166 } 167 } 168 169 // A groupRec keeps track of all the resolved assertions in a group 170 // or whether the group should be considered in error (err != nil). 171 type groupRec struct { 172 err error 173 resolved []Ref 174 } 175 176 func (gRec *groupRec) hasErr() bool { 177 return gRec.err != nil 178 } 179 180 func (gRec *groupRec) setErr(e error) { 181 if gRec.err == nil { 182 gRec.err = e 183 } 184 } 185 186 func (gRec *groupRec) markResolved(ref *Ref) (marked bool) { 187 if gRec.hasErr() { 188 return false 189 } 190 gRec.resolved = append(gRec.resolved, *ref) 191 return true 192 } 193 194 // markResolved marks the assertion referenced by ref as resolved 195 // in all the groups in grouping, except those already in error. 196 func (p *Pool) markResolved(grouping *internal.Grouping, resolved *Ref) (marked bool) { 197 p.groupings.Iter(grouping, func(gnum uint16) error { 198 if p.groups[gnum].markResolved(resolved) { 199 marked = true 200 } 201 return nil 202 }) 203 return marked 204 } 205 206 // setErr marks all the groups in grouping as in error with error err 207 // except those already in error. 208 func (p *Pool) setErr(grouping *internal.Grouping, err error) { 209 p.groupings.Iter(grouping, func(gnum uint16) error { 210 p.groups[gnum].setErr(err) 211 return nil 212 }) 213 } 214 215 func (p *Pool) isPredefined(ref *Ref) (bool, error) { 216 _, err := ref.Resolve(p.groundDB.FindPredefined) 217 if err == nil { 218 return true, nil 219 } 220 if !IsNotFound(err) { 221 return false, err 222 } 223 return false, nil 224 } 225 226 func (p *Pool) isResolved(ref *Ref) (bool, error) { 227 _, err := p.bs.Get(ref.Type, ref.PrimaryKey, ref.Type.MaxSupportedFormat()) 228 if err == nil { 229 return true, nil 230 } 231 if !IsNotFound(err) { 232 return false, err 233 } 234 return false, nil 235 } 236 237 func (p *Pool) curRevision(ref *Ref) (int, error) { 238 a, err := ref.Resolve(p.groundDB.Find) 239 if err != nil && !IsNotFound(err) { 240 return 0, err 241 } 242 if err == nil { 243 return a.Revision(), nil 244 } 245 return RevisionNotKnown, nil 246 } 247 248 type poolPhase int 249 250 const ( 251 poolPhaseAddUnresolved = iota 252 poolPhaseAdd 253 ) 254 255 func (p *Pool) phase(ph poolPhase) error { 256 if ph == p.curPhase { 257 return nil 258 } 259 if ph == poolPhaseAdd { 260 return fmt.Errorf("internal error: cannot switch to Pool add phase without invoking ToResolve first") 261 } 262 // ph == poolPhaseAddUnresolved 263 p.unresolvedBookkeeping() 264 p.curPhase = poolPhaseAddUnresolved 265 return nil 266 } 267 268 // AddUnresolved adds the assertion referenced by unresolved 269 // AtRevision to the Pool as unresolved and as required by the given group. 270 // Usually unresolved.Revision will have been set to RevisionNotKnown. 271 func (p *Pool) AddUnresolved(unresolved *AtRevision, group string) error { 272 if err := p.phase(poolPhaseAddUnresolved); err != nil { 273 return err 274 } 275 gnum, err := p.ensureGroup(group) 276 if err != nil { 277 return err 278 } 279 u := *unresolved 280 ok, err := p.isPredefined(&u.Ref) 281 if err != nil { 282 return err 283 } 284 if ok { 285 // predefined, nothing to do 286 return nil 287 } 288 return p.addUnresolved(&u, gnum) 289 } 290 291 func (p *Pool) addUnresolved(unresolved *AtRevision, gnum uint16) error { 292 ok, err := p.isResolved(&unresolved.Ref) 293 if err != nil { 294 return err 295 } 296 if ok { 297 // We assume that either the resolving of 298 // prerequisites for the already resolved assertion in 299 // progress has succeeded or will. If that's not the 300 // case we will fail at CommitTo time. We could 301 // instead recurse into its prerequisites again but the 302 // complexity isn't clearly worth it. 303 // See TestParallelPartialResolutionFailure 304 // Mark this as resolved in the group. 305 p.groups[gnum].markResolved(&unresolved.Ref) 306 return nil 307 } 308 uniq := unresolved.Ref.Unique() 309 var u *unresolvedRec 310 if u = p.unresolved[uniq]; u == nil { 311 u = &unresolvedRec{ 312 at: unresolved, 313 } 314 p.unresolved[uniq] = u 315 } 316 u.merge(unresolved, gnum, p.groupings) 317 return nil 318 } 319 320 // ToResolve returns all the currently unresolved assertions in the 321 // Pool, organized in opaque groupings based on which set of groups 322 // requires each of them. 323 // At the next ToResolve any unresolved assertion with not known 324 // revision that was not added via Add or AddBatch will result in all 325 // groups requiring it being in error with ErrUnresolved. 326 // Conversely, the remaining unresolved assertions originally added 327 // via AddToUpdate will be assumed to still be at their current 328 // revisions. 329 func (p *Pool) ToResolve() (map[Grouping][]*AtRevision, error) { 330 if p.curPhase == poolPhaseAdd { 331 p.unresolvedBookkeeping() 332 } else { 333 p.curPhase = poolPhaseAdd 334 } 335 r := make(map[Grouping][]*AtRevision) 336 for _, u := range p.unresolved { 337 if u.at.Revision == RevisionNotKnown { 338 rev, err := p.curRevision(&u.at.Ref) 339 if err != nil { 340 return nil, err 341 } 342 if rev != RevisionNotKnown { 343 u.at.Revision = rev 344 } 345 } 346 u.exportTo(r, p.groupings) 347 } 348 return r, nil 349 } 350 351 func (p *Pool) addPrerequisite(pref *Ref, g *internal.Grouping) error { 352 uniq := pref.Unique() 353 u := p.unresolved[uniq] 354 at := &AtRevision{ 355 Ref: *pref, 356 Revision: RevisionNotKnown, 357 } 358 if u == nil { 359 u = p.prerequisites[uniq] 360 } 361 if u != nil { 362 gr := p.groupings 363 gr.Iter(g, func(gnum uint16) error { 364 u.merge(at, gnum, gr) 365 return nil 366 }) 367 return nil 368 } 369 ok, err := p.isPredefined(pref) 370 if err != nil { 371 return err 372 } 373 if ok { 374 // nothing to do 375 return nil 376 } 377 ok, err = p.isResolved(pref) 378 if err != nil { 379 return err 380 } 381 if ok { 382 // nothing to do, it is anyway implied 383 return nil 384 } 385 p.prerequisites[uniq] = &unresolvedRec{ 386 at: at, 387 grouping: g.Copy(), 388 } 389 return nil 390 } 391 392 func (p *Pool) add(a Assertion, g *internal.Grouping) error { 393 if err := p.bs.Put(a.Type(), a); err != nil { 394 if revErr, ok := err.(*RevisionError); ok { 395 if revErr.Current >= a.Revision() { 396 // we already got something more recent 397 return nil 398 } 399 } 400 401 return err 402 } 403 for _, pref := range a.Prerequisites() { 404 if err := p.addPrerequisite(pref, g); err != nil { 405 return err 406 } 407 } 408 keyRef := &Ref{ 409 Type: AccountKeyType, 410 PrimaryKey: []string{a.SignKeyID()}, 411 } 412 if err := p.addPrerequisite(keyRef, g); err != nil { 413 return err 414 } 415 return nil 416 } 417 418 func (p *Pool) resolveWith(unresolved map[string]*unresolvedRec, uniq string, u *unresolvedRec, a Assertion, extrag *internal.Grouping) error { 419 if a.Revision() > u.at.Revision { 420 if extrag == nil { 421 extrag = &u.grouping 422 } else { 423 p.groupings.Iter(&u.grouping, func(gnum uint16) error { 424 p.groupings.AddTo(extrag, gnum) 425 return nil 426 }) 427 } 428 ref := a.Ref() 429 if p.markResolved(extrag, ref) { 430 // remove from tracking - 431 // remove u from unresolved only if the assertion 432 // is added to the resolved backstore; 433 // otherwise it might resurface as unresolved; 434 // it will be ultimately handled in 435 // unresolvedBookkeeping if it stays around 436 delete(unresolved, uniq) 437 if err := p.add(a, extrag); err != nil { 438 p.setErr(extrag, err) 439 return err 440 } 441 } 442 } 443 return nil 444 } 445 446 // Add adds the given assertion associated with the given grouping to the 447 // Pool as resolved in all the groups requiring it. 448 // Any not already resolved prerequisites of the assertion will 449 // be implicitly added as unresolved and required by all of those groups. 450 // The grouping will usually have been associated with the assertion 451 // in a ToResolve's result. Otherwise the union of all groups 452 // requiring the assertion plus the groups in grouping will be considered. 453 // The latter is mostly relevant in scenarios where the server is pushing 454 // assertions. 455 func (p *Pool) Add(a Assertion, grouping Grouping) error { 456 if err := p.phase(poolPhaseAdd); err != nil { 457 return err 458 } 459 460 if !a.SupportedFormat() { 461 return &UnsupportedFormatError{Ref: a.Ref(), Format: a.Format()} 462 } 463 464 uniq := a.Ref().Unique() 465 var u *unresolvedRec 466 var extrag *internal.Grouping 467 var unresolved map[string]*unresolvedRec 468 if u = p.unresolved[uniq]; u != nil { 469 unresolved = p.unresolved 470 } else if u = p.prerequisites[uniq]; u != nil { 471 unresolved = p.prerequisites 472 } else { 473 ok, err := p.isPredefined(a.Ref()) 474 if err != nil { 475 return err 476 } 477 if ok { 478 // nothing to do 479 return nil 480 } 481 // a is not tracked as unresolved in any way so far, 482 // this is an atypical scenario where something gets 483 // pushed but we still want to add it to the resolved 484 // lists of the relevant groups; in case it is 485 // actually already resolved most of resolveWith below will 486 // be a nop 487 u = &unresolvedRec{ 488 at: a.At(), 489 } 490 u.at.Revision = RevisionNotKnown 491 } 492 493 if u.serializedLabel != grouping { 494 var err error 495 extrag, err = p.groupings.Deserialize(string(grouping)) 496 if err != nil { 497 return err 498 } 499 } 500 501 return p.resolveWith(unresolved, uniq, u, a, extrag) 502 } 503 504 // TODO: AddBatch 505 506 var ( 507 ErrUnresolved = errors.New("unresolved assertion") 508 ErrUnknownPoolGroup = errors.New("unknown pool group") 509 ) 510 511 // unresolvedBookkeeping processes any left over unresolved assertions 512 // since the last ToResolve invocation and intervening calls to Add/AddBatch, 513 // * they were either marked as in error which will be propagated 514 // to all groups requiring them 515 // * simply unresolved, which will be propagated to groups requiring them 516 // as ErrUnresolved 517 // * unchanged (update case) 518 // unresolvedBookkeeping will also promote any recorded prerequisites 519 // into actively unresolved, as long as not all the groups requiring them 520 // are in error. 521 func (p *Pool) unresolvedBookkeeping() { 522 // any left over unresolved are either: 523 // * in error 524 // * unchanged 525 // * or unresolved 526 for uniq, u := range p.unresolved { 527 e := u.err 528 if e == nil { 529 if u.at.Revision == RevisionNotKnown { 530 e = ErrUnresolved 531 } 532 } 533 if e != nil { 534 p.setErr(&u.grouping, e) 535 } 536 delete(p.unresolved, uniq) 537 } 538 539 // prerequisites will become the new unresolved but drop them 540 // if all their groups are in error 541 for uniq, prereq := range p.prerequisites { 542 useful := false 543 p.groupings.Iter(&prereq.grouping, func(gnum uint16) error { 544 if !p.groups[gnum].hasErr() { 545 useful = true 546 } 547 return nil 548 }) 549 if !useful { 550 delete(p.prerequisites, uniq) 551 continue 552 } 553 } 554 555 // prerequisites become the new unresolved, the emptied 556 // unresolved is used for prerequisites in the next round 557 p.unresolved, p.prerequisites = p.prerequisites, p.unresolved 558 } 559 560 // Err returns the error for group if group is in error, nil otherwise. 561 func (p *Pool) Err(group string) error { 562 gnum, err := p.groupNum(group) 563 if err != nil { 564 return err 565 } 566 gRec := p.groups[gnum] 567 if gRec == nil { 568 return ErrUnknownPoolGroup 569 } 570 return gRec.err 571 } 572 573 // AddError associates error e with the unresolved assertion. 574 // The error will be propagated to all the affected groups at 575 // the next ToResolve. 576 func (p *Pool) AddError(e error, ref *Ref) error { 577 if err := p.phase(poolPhaseAdd); err != nil { 578 return err 579 } 580 uniq := ref.Unique() 581 if u := p.unresolved[uniq]; u != nil && u.err == nil { 582 u.err = e 583 } 584 return nil 585 } 586 587 // AddGroupingError puts all the groups of grouping in error, with error e. 588 func (p *Pool) AddGroupingError(e error, grouping Grouping) error { 589 if err := p.phase(poolPhaseAdd); err != nil { 590 return err 591 } 592 593 g, err := p.groupings.Deserialize(string(grouping)) 594 if err != nil { 595 return err 596 } 597 598 p.setErr(g, e) 599 return nil 600 } 601 602 // AddToUpdate adds the assertion referenced by toUpdate and all its 603 // prerequisites to the Pool as unresolved and as required by the 604 // given group. It is assumed that the assertion is currently in the 605 // ground database of the Pool, otherwise this will error. 606 // The current revisions of the assertion and its prerequisites will 607 // be recorded and only higher revisions will then resolve them, 608 // otherwise if ultimately unresolved they will be assumed to still be 609 // at their current ones. 610 func (p *Pool) AddToUpdate(toUpdate *Ref, group string) error { 611 if err := p.phase(poolPhaseAddUnresolved); err != nil { 612 return err 613 } 614 gnum, err := p.ensureGroup(group) 615 if err != nil { 616 return err 617 } 618 retrieve := func(ref *Ref) (Assertion, error) { 619 return ref.Resolve(p.groundDB.Find) 620 } 621 add := func(a Assertion) error { 622 return p.addUnresolved(a.At(), gnum) 623 } 624 f := NewFetcher(p.groundDB, retrieve, add) 625 if err := f.Fetch(toUpdate); err != nil { 626 return err 627 } 628 return nil 629 } 630 631 // CommitTo adds the assertions from groups without errors to the 632 // given assertion database. Commit errors can be retrieved via Err 633 // per group. 634 func (p *Pool) CommitTo(db *Database) error { 635 if p.curPhase == poolPhaseAddUnresolved { 636 return fmt.Errorf("internal error: cannot commit Pool during add unresolved phase") 637 } 638 p.unresolvedBookkeeping() 639 640 retrieve := func(ref *Ref) (Assertion, error) { 641 a, err := p.bs.Get(ref.Type, ref.PrimaryKey, ref.Type.MaxSupportedFormat()) 642 if IsNotFound(err) { 643 // fallback to pre-existing assertions 644 a, err = ref.Resolve(db.Find) 645 } 646 if err != nil { 647 return nil, resolveError("cannot resolve prerequisite assertion: %s", ref, err) 648 } 649 return a, nil 650 } 651 save := func(a Assertion) error { 652 err := db.Add(a) 653 if IsUnaccceptedUpdate(err) { 654 // unsupported format case is handled before. 655 // be idempotent, db has already the same or 656 // newer. 657 return nil 658 } 659 return err 660 } 661 662 NextGroup: 663 for _, gRec := range p.groups { 664 if gRec.hasErr() { 665 // already in error, ignore 666 continue 667 } 668 // TODO: try to reuse fetcher 669 f := NewFetcher(db, retrieve, save) 670 for i := range gRec.resolved { 671 if err := f.Fetch(&gRec.resolved[i]); err != nil { 672 gRec.setErr(err) 673 continue NextGroup 674 } 675 } 676 } 677 678 return nil 679 }