cuelang.org/go@v0.13.0/internal/core/adt/fields.go

// Copyright 2023 CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package adt

// This file holds the logic for the insertion of fields and pattern
// constraints, including tracking closedness.
//
//
// DESIGN GOALS
//
// Key to performance is to fail early during evaluation. This is especially
// true for disjunctions. In CUE evaluation, conjuncts may be evaluated in a
// fairly arbitrary order. We want to retain this flexibility while also failing
// on disallowed fields as soon as we have enough data to tell for certain.
//
// Keeping track of which fields are allowed means keeping provenance data on
// whether certain conjuncts originate from embeddings or definitions, as well
// as how they group together with other conjuncts. These data structures should
// allow for a "mark and unwind" approach to allow for backtracking when
// computing disjunctions.
//
// References to the same CUE value may be added as conjuncts through various
// paths. For instance, a reference to a definition may be added directly, or
// through embedding. How they are added affects which set of fields are
// allowed. This can make the removal of duplicate conjuncts hard. A solution
// should make it straightforward to deduplicate conjuncts if they have the same
// impact on field inclusion.
//
// All conjuncts associated with field constraints, including optional fields
// and pattern constraints, should be collated, deduplicated, and evaluated as
// if they were regular fields. This allows comparisons between values to be
// meaningful and helps to filter disjuncts.
//
// The provenance data generated by this algorithm should ideally be easily
// usable in external APIs.
//
//
// DATA STRUCTURES
//
// Conjuncts
//
// To keep track of conjunct provenance, each conjunct has a few flags that
// indicates whether it originates from
//   - an embedding
//   - a definition
//   - a reference (optional and unimplemented)
//
// Conjuncts with the same origin are represented as a single Conjunct in the
// Vertex, where this conjunct is a list of these conjuncts. In other words, the
// conjuncts of a Vertex are really a forest (group of trees) of conjuncts that,
// recursively, reflect the provenance of the conjuncts contained within it.
//
// The current implementation uses a Vertex for listing conjuncts with the same
// origin. This Vertex is marked as "Dynamic", as it does not have a CUE path
// that leads to them.
//
//
// Constraints
//
// Vertex values separately keep track of pattern constraints. These consist of
// a list of patterns with associated conjuncts, and a CUE expression that
// represents the set of allowed fields. This information is mostly for equality
// checking: by the time this data is produced, conjuncts associated with
// patterns are already inserted into the computed subfields.
//
// Note that this representation assumes that patterns are always accrued
// cumulatively: a field that is allowed will accrue the conjuncts of any
// matched pattern, even if it originates from an embedding that itself does not
// allow this field.
//
//
// ALGORITHM
//
// When processing the conjuncts of a Vertex, subfields are tracked per
// "grouping" (the list of conjuncts of the same origin). Each grouping keeps a
// counter of the number of unprocessed conjuncts and subgroups associated with
// it. Field inclusion (closedness) can be computed as soon as all subconjuncts
// and subgroups are processed.
//
// Conjuncts of subfields are inserted in such a way that they reflect the same
// grouping as the parent Vertex, plus any grouping that may be added by the
// subfield itself.
//
// It would be possible, though, to collapse certain (combinations of) groups
// that contain only a single conjunct. This can limit the size of such conjunct
// trees.
//
// As conjuncts are added within their grouping context, it is possible to
// uniquely identify conjuncts only by Vertex and expression pointer,
// disregarding the Environment.
//
//
// EXAMPLE DATA STRUCTURE
//
//    a: #A
//    #A: {
//        #B
//        x: r1
//    }
//    #B: y: r2
//    r1: z: r3
//    r2: 2
//    r3: foo: 2
//
// gets evaluated into:
//
//    V_a: Arcs{
//        x: V_x [ V_def(#A)[ r1 ] ]
//        y: V_y [ V_def(#A)[ V_embed(#B)[ r2 ] ] ]
//    }
//
// When evaluating V_x, its Arcs, in turn become:
//
//    V_x: Arcs{
//        z: V_z [ V_def(#A)[ V_ref(r1)[ r3 ]) ]]
//    }
//
// The V_def(#A) is necessary here to ensure that closedness information can be
// computed, if necessary. The V_ref's, however, are optional, and can be
// omitted if provenance is less important:
//
//    V_x: Arcs{
//        z: V_z [ V_def(#A)[ r3 ]]
//    }
//
// Another possible optimization is to eliminate Vertices if there is only one
// conjunct: the embedding and definition flags in the conjunct can be
// sufficient in that case. The provenance data could potentially be derived
// from the Environment in that case. If an embedding conjunct is itself the
// only conjunct in a list, the embedding bit can be eliminated. So V_y in the
// above example could be reduced to
//
//    V_y [ V_def(#A)[ r2 ] ]
//

// TODO(perf):
// - the data structures could probably be collapsed with Conjunct. and the
//   Vertex inserted into the Conjuncts could be a special ConjunctGroup.

func (n *nodeContext) getArc(f Feature, mode ArcType) (arc *Vertex, isNew bool) {
	// TODO(disjunct,perf): CopyOnRead
	v := n.node
	for _, a := range v.Arcs {
		if a.Label == f {
			if f.IsLet() {
				a.MultiLet = true
				// TODO: add return here?
			}
			a.updateArcType(mode)
			return a, false
		}
	}

	arc = &Vertex{
		Parent:    v,
		Label:     f,
		ArcType:   mode,
		nonRooted: v.IsDynamic || v.nonRooted,
		anonymous: v.anonymous || v.Label.IsLet(),
	}
	if n.scheduler.frozen&fieldSetKnown != 0 {
		b := n.ctx.NewErrf("adding field %v not allowed as field set was already referenced", f)
		n.ctx.AddBottom(b)
		// This may panic for list arithmetic. Safer to leave out for now.
		arc.ArcType = ArcNotPresent
	}
	v.Arcs = append(v.Arcs, arc)
	return arc, true
}

// allowedInClosed reports whether a field with label f is allowed in a closed
// struct, even when it is not explicitly defined.
//
// TODO: see https://github.com/cue-lang/cue/issues/543
// for whether to include f.IsDef.
func allowedInClosed(f Feature) bool {
	return f.IsHidden() || f.IsDef() || f.IsLet()
}

// insertConjunct inserts conjunct c into cc.
func (v *Vertex) insertConjunct(ctx *OpContext, c Conjunct, id CloseInfo, mode ArcType, check, checkClosed bool) (pos int, added bool) {
	n := v.getBareState(ctx)
	if n == nil {
		return 0, false
	}

	n.markNonCyclic(id)

	v.updateArcType(mode)

	var c2 Conjunct
	pos = -1
	if check {
		pos, c2 = findConjunct(v.Conjuncts, c)

	}
	if pos == -1 {
		pos = len(v.Conjuncts)
		v.addConjunctUnchecked(c)
		added = true
	} else if srcRef := c2.CloseInfo.defID; srcRef != 0 {
		// Most duplicates are deduped in insertVertexConjuncts by deduping the
		// reference that brings in conjuncts in the first place. However, with
		// API calls, and in some cases possibly with structure sharing, it may
		// be possible that different Vertices refer to the same conjuncts. In
		// this case, we need to ensure that the current defID also considers
		// the ID associated with the original insertion in its set.
		n.addReplacement(replaceID{from: id.defID, to: srcRef, add: true})
	}

	if v.isInProgress() {
		n.scheduleConjunct(c, id)
	}

	for _, rec := range n.notify {
		// TODO(evalv3): currently we get pending arcs here for some tests.
		// That seems fine. But consider this again when most of evalv3 work
		// is done. See test "pending.cue" in comprehensions/notify2.txtar
		// It seems that only let arcs can be pending, though.

		// TODO: we should probably only notify a conjunct once the root of the
		// conjunct group is completed. This will make it easier to "stitch" the
		// conjunct trees together, as its correctness will be guaranteed.
		rec.v.state.scheduleConjunct(c, id)
	}

	return
}

func (n *nodeContext) insertArc(f Feature, mode ArcType, c Conjunct, id CloseInfo, check bool) *Vertex {
	n.assertInitialized()

	if n == nil {
		panic("nil nodeContext")
	}
	if n.node == nil {
		panic("nil node")
	}

	v, insertedArc := n.getArc(f, mode)

	defer n.ctx.PopArc(n.ctx.PushArc(v))

	// TODO: reporting the cycle error here results in better error paths.
	// However, it causes the reference counting mechanism to be faulty.
	// Reevaluate once the new evaluator is done.
	// if v.ArcType == ArcNotPresent {
	// 	// It was already determined before that this arc may not be present.
	// 	// This case can only manifest itself if we have a cycle.
	// 	n.node.reportFieldCycleError(n.ctx, pos(c.x), f)
	// 	return v
	// }

	_, added := v.insertConjunct(n.ctx, c, id, mode, check, true)
	if !added || !insertedArc {
		return v
	}

	// Match and insert patterns.
	if pcs := n.node.PatternConstraints; pcs != nil {
		for _, pc := range pcs.Pairs {
			if matchPattern(n.ctx, pc.Pattern, f) {
				for _, c := range pc.Constraint.Conjuncts {
					n.addConstraint(v, mode, c, check)
				}
			}
		}
	}

	return v
}

// addConstraint adds a constraint to arc of n.
//
// In order to resolve LabelReferences, it is not always possible to walk up
// the parent Vertex chain to determan the label, because a label reference
// may point past a point of referral. For instance,
//
//	test: [ID=_]: name: ID
//	test: A: {}
//	B: test.A & {}  // B.name should be "A", not "B".
//
// The arc must be the node arc to which the conjunct is added.
func (n *nodeContext) addConstraint(arc *Vertex, mode ArcType, c Conjunct, check bool) {
	n.assertInitialized()

	// TODO(perf): avoid cloning the Environment, if:
	// - the pattern constraint has no LabelReference
	//   (require compile-time support)
	// - there are no references in the conjunct pointing to this node.
	// - consider adding this value to the Conjunct struct
	f := arc.Label
	bulkEnv := *c.Env
	bulkEnv.DynamicLabel = f
	c.Env = &bulkEnv

	// TODO: can go, but do in separate CL.
	arc, _ = n.getArc(f, mode)

	arc.insertConjunct(n.ctx, c, c.CloseInfo, mode, check, false)
}

func (n *nodeContext) insertPattern(pattern Value, c Conjunct) {
	n.assertInitialized()

	// Collect patterns in root vertex. This allows comparing disjuncts for
	// equality as well as inserting new arcs down the line as they are
	// inserted.
	if n.insertConstraint(pattern, c) {
		// Match against full set of arcs from root, but insert in current vertex.
		// Hypothesis: this may not be necessary. Maybe for closedness.
		// TODO: may need to replicate the closedContext for patterns.
		// Also: Conjuncts for matching other arcs in this node may be different
		// for matching arcs using v.foo?, if we need to ensure that conjuncts
		// from arcs and patterns are grouped under the same vertex.
		// TODO: verify. See test Pattern 1b
		for _, a := range n.node.Arcs {
			if matchPattern(n.ctx, pattern, a.Label) {
				// TODO: is it necessary to check for uniqueness here?
				n.addConstraint(a, a.ArcType, c, true)
			}
		}
	}

	if n.node.HasEllipsis {
		return
	}

	// TODO: we could still try to accumulate patterns.
}

// isTotal reports whether pattern value p represents a full domain, that is,
// whether it is of type BasicType or Top.
func isTotal(p Value) bool {
	switch p.(type) {
	case *BasicType:
		return true
	case *Top:
		return true
	}
	return false
}

func (ctx *OpContext) addPositions(c Conjunct) {
	if x, ok := c.x.(*ConjunctGroup); ok {
		for _, c := range *x {
			ctx.addPositions(c)
		}
	}
	if pos := c.Field(); pos != nil {
		ctx.AddPosition(pos)
	}
}

// notAllowedError reports a field not allowed error in n and sets the value
// for arc f to that error.
func (ctx *OpContext) notAllowedError(arc *Vertex) *Bottom {
	defer ctx.PopArc(ctx.PushArc(arc))

	defer ctx.ReleasePositions(ctx.MarkPositions())

	for _, c := range arc.Conjuncts {
		ctx.addPositions(c)
	}
	// TODO(0.7): Find another way to get this provenance information. Not
	// currently stored in new evaluator.
	// for _, s := range x.Structs {
	//  s.AddPositions(ctx)
	// }

	// TODO: use the arcType from the closeContext.
	if arc.ArcType == ArcPending {
		// arc.ArcType = ArcNotPresent
		// We do not know yet whether the arc will be present or not. Checking
		// this will be deferred until this is known, after the comprehension
		// has been evaluated.
		return nil
	}
	ctx.Assertf(ctx.pos(), !allowedInClosed(arc.Label), "unexpected disallowed definition, let, or hidden field")
	if ctx.HasErr() {
		// The next error will override this error when not run in Strict mode.
		return nil
	}

	// TODO: setting arc instead of n.node eliminates subfields. This may be
	// desirable or not, but it differs, at least from <=v0.6 behavior.
	err := ctx.NewErrf("field not allowed")
	err.CloseCheck = true
	arc.SetValue(ctx, err)
	if arc.state != nil {
		arc.state.kind = 0
	}

	// TODO: remove? We are now setting it on both fields, which seems to be
	// necessary for now. But we should remove this as it often results in
	// a duplicate error.
	// v.SetValue(ctx, ctx.NewErrf("field not allowed"))

	// TODO: create a special kind of error that gets the positions
	// of the relevant locations upon request from the arc.
	return err
}

// mergeConjunctions combines two values into one. It never modifies an
// existing conjunction.
//
// TODO: this was used in the closeContext code. We can still use it to
// construct pattern constraint conjunction in the future. This is currently
// unimplemented.
func mergeConjunctions(a, b Value) Value {
	if a == nil {
		return b
	}
	if b == nil {
		return a
	}
	ca, _ := a.(*Conjunction)
	cb, _ := b.(*Conjunction)
	n := 2
	if ca != nil {
		n += len(ca.Values) - 1
	}
	if cb != nil {
		n += len(cb.Values) - 1
	}
	vs := make([]Value, 0, n)
	if ca != nil {
		vs = append(vs, ca.Values...)
	} else {
		vs = append(vs, a)
	}
	if cb != nil {
		vs = append(vs, cb.Values...)
	} else {
		vs = append(vs, b)
	}
	// TODO: potentially order conjuncts to make matching more likely.
	return &Conjunction{Values: vs}
}