go.fuchsia.dev/infra@v0.0.0-20240507153436-9b593402251b/cmd/submodule_update/submodule/submodule.go

// Copyright 2023 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Package submodule handles analyzing and updating git submodule states.
package submodule

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path"
	"regexp"
	"sort"
	"strings"

	"github.com/google/subcommands"
	"go.fuchsia.dev/infra/cmd/submodule_update/gitutil"
)

// Submodule represents the status of  a git submodule.
type Submodule struct {
	// Name is the name of the submodule in jiri projects.
	Name string `xml:"name,attr,omitempty"`
	// Revision is the revision the submodule.
	Revision string `xml:"revision,attr,omitempty"`
	// Path is the relative path starting from the superproject root.
	Path string `xml:"path,attr,omitempty"`
	// Remote is the remote for a submodule.
	Remote string `xml:"remote,attr,omitempty"`
}

// Submodules maps Keys to Submodules.
type Submodules map[Key]Submodule

// Key is a map key for a submodule.
type Key string

// Key returns the unique Key for the project.
func (s Submodule) Key() Key {
	return Key(s.Path)
}

var submoduleStatusRe = regexp.MustCompile(`(?m)^[+\-U\s]?([0-9a-f]{40})\s([a-zA-Z0-9_.\-\/]+).*$`)

func gitSubmodules(g *gitutil.Git, cached bool) (Submodules, error) {
	gitSubmoduleStatus, err := g.SubmoduleStatus(gitutil.CachedOpt(cached))
	if err != nil {
		return nil, err
	}
	return gitSubmoduleStatusToSubmodule(g, gitSubmoduleStatus)
}

func gitSubmoduleStatusToSubmodule(g *gitutil.Git, status string) (Submodules, error) {
	var subModules = Submodules{}
	subStatus := submoduleStatusRe.FindAllStringSubmatch(status, -1)
	for _, status := range subStatus {
		// Regex fields are
		// - Full match (field 0)
		// - SHA1 (field 1)
		// - Path (field 2)
		subM := Submodule{
			Path:     status[2],
			Revision: status[1],
		}
		url, err := g.ConfigGetKeyFromFile(
			fmt.Sprintf("submodule.%s.url", subM.Path),
			".gitmodules")
		if err != nil {
			return nil, err
		}
		subM.Remote = url
		name, err := g.ConfigGetKeyFromFile(
			fmt.Sprintf("submodule.%s.name", subM.Path),
			".gitmodules")
		if err != nil {
			return nil, err
		}
		subM.Name = name
		subModules[subM.Key()] = subM
	}
	return subModules, nil
}

func addIgnoreToSubmodulesConfig(g *gitutil.Git, s Submodule) error {
	configKey := fmt.Sprintf("submodule.%s.ignore", s.Path)
	return g.ConfigAddKeyToFile(configKey, ".gitmodules", "all")
}

func addProjectNameToSubmodulesConfig(g *gitutil.Git, s Submodule) error {
	configKey := fmt.Sprintf("submodule.%s.name", s.Path)
	return g.ConfigAddKeyToFile(configKey, ".gitmodules", s.Name)
}

// jiriProjectInfo defines jiri JSON format for 'project info' output.
type jiriProjectInfo struct {
	Name string `json:"name"`
	Path string `json:"path"`

	// Relative path w.r.t to root
	RelativePath   string   `json:"relativePath"`
	Remote         string   `json:"remote"`
	Revision       string   `json:"revision"`
	CurrentBranch  string   `json:"current_branch,omitempty"`
	Branches       []string `json:"branches,omitempty"`
	Manifest       string   `json:"manifest,omitempty"`
	GitSubmoduleOf string   `json:"gitsubmoduleof,omitempty"`
}

func jiriProjectsToSubmodule(path string) (Submodules, error) {

	jiriProjectsRaw, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	var jiriProjects []jiriProjectInfo
	err = json.Unmarshal(jiriProjectsRaw, &jiriProjects)
	if err != nil {
		return nil, err
	}

	var subModules = Submodules{}
	for _, project := range jiriProjects {
		// If the project name is empty that means it's a source-of-truth
		// submodule that's not actually known to Jiri.
		if project.Name == "" && project.GitSubmoduleOf != "" {
			continue
		}
		// Drop "integration" and "fuchsia" (relative path "'") and not a submodule of fuchsia
		if project.RelativePath == "." || project.RelativePath == "integration" || project.GitSubmoduleOf != "fuchsia" {
			continue
		}
		subM := Submodule{
			Name:     project.Name,
			Path:     project.RelativePath,
			Revision: project.Revision,
			Remote:   project.Remote,
		}
		subModules[subM.Key()] = subM
	}
	return subModules, nil
}

// DiffSubmodule structure defines the difference between the status of two submodules
type DiffSubmodule struct {
	Name        string `json:"name,omitempty"`
	Path        string `json:"path"`
	OldPath     string `json:"old_path,omitempty"`
	Revision    string `json:"revision"`
	OldRevision string `json:"old_revision,omitempty"`
	Remote      string `json:"remote,omitempty"`
}

type diffSubmodulesByPath []DiffSubmodule

func (p diffSubmodulesByPath) Len() int {
	return len(p)
}
func (p diffSubmodulesByPath) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}
func (p diffSubmodulesByPath) Less(i, j int) bool {
	return p[i].Path < p[j].Path
}

// Diff structure enumerates the new, deleted and updated submodules when diffing between a set of submodules.
type Diff struct {
	NewSubmodules     []DiffSubmodule `json:"new_submodules"`
	DeletedSubmodules []DiffSubmodule `json:"deleted_submodules"`
	UpdatedSubmodules []DiffSubmodule `json:"updated_submodules"`
}

func (d Diff) sort() Diff {
	sort.Sort(diffSubmodulesByPath(d.NewSubmodules))
	sort.Sort(diffSubmodulesByPath(d.DeletedSubmodules))
	sort.Sort(diffSubmodulesByPath(d.UpdatedSubmodules))
	return d
}

func deleteSubmodules(g *gitutil.Git, diff []DiffSubmodule) error {
	if len(diff) == 0 {
		return nil
	}
	var submodulePaths []string
	for _, subM := range diff {
		submodulePaths = append(submodulePaths, subM.Path)
	}
	return g.Remove(submodulePaths...)
}

func addSubmodules(g *gitutil.Git, diff []DiffSubmodule) error {
	if len(diff) == 0 {
		return nil
	}
	for _, subMDiff := range diff {
		if err := g.SubmoduleAdd(subMDiff.Remote, subMDiff.Path); err != nil {
			return err
		}
		subM := Submodule{
			Name:   subMDiff.Name,
			Path:   subMDiff.Path,
			Remote: subMDiff.Remote,
		}
		// Make sure all new git submodules have project name included in config.
		if err := addProjectNameToSubmodulesConfig(g, subM); err != nil {
			return err
		}
		// Add ignore all to git submodules config to avoid project drift
		if err := addIgnoreToSubmodulesConfig(g, subM); err != nil {
			return err
		}
	}
	// Checkout all added submodules at given revision
	gs := *g
	for _, subM := range diff {
		gs.Update(gitutil.SubmoduleDirOpt(subM.Path))
		if err := gs.CheckoutBranch(subM.Revision, false); err != nil {
			return err
		}
	}
	return nil
}

func updateSubmodules(g *gitutil.Git, diff []DiffSubmodule, superprojectRoot string) error {
	if len(diff) == 0 {
		return nil
	}
	var submodulePaths []string
	for _, subM := range diff {
		// We need to fetch for every submodule that needs updating.
		subMPath := path.Join(superprojectRoot, subM.Path)
		g := gitutil.New(gitutil.RootDirOpt(subMPath))
		if err := g.Fetch("origin"); err != nil {
			return err
		}
		submodulePaths = append(submodulePaths, subM.Path)
	}
	if err := g.SubmoduleUpdate(submodulePaths, gitutil.InitOpt(true)); err != nil {
		return err
	}

	gs := *g
	for _, subM := range diff {
		gs.Update(gitutil.SubmoduleDirOpt(subM.Path))
		if err := gs.CheckoutBranch(subM.Revision, false); err != nil {
			return err
		}
	}
	return nil
}

func updateCommitMessage(message string) string {
	// Replace [roll] with [superproject] to differentiate commit message.
	const RollPrefix = "[roll] "
	// Only substitute if [roll] is at beginning of message.
	if strings.Index(message, RollPrefix) == 0 {
		return strings.Replace(message, RollPrefix, "[superproject] ", 1)
	}
	return message
}

func updateSuperprojectSubmodules(g *gitutil.Git, diff Diff, superprojectRoot string) error {
	if err := deleteSubmodules(g, diff.DeletedSubmodules); err != nil {
		return err
	}
	if err := addSubmodules(g, diff.NewSubmodules); err != nil {
		return err
	}
	if err := updateSubmodules(g, diff.UpdatedSubmodules, superprojectRoot); err != nil {
		return err
	}

	return nil
}

// Add project name to all submodules.
func updateSubmodulesName(g *gitutil.Git, gitSubMs, jiriSubMs Submodules) error {
	for key, gitSubM := range gitSubMs {
		if _, ok := jiriSubMs[key]; ok {
			gitSubM.Name = jiriSubMs[key].Name
			if err := addProjectNameToSubmodulesConfig(g, gitSubM); err != nil {
				return err
			}
		}
	}
	return nil
}

// Add ignore = all to all submodules.
func updateSubmodulesIgnore(g *gitutil.Git, gitSubMs, jiriSubMs Submodules) error {
	for key, gitSubM := range gitSubMs {
		if _, ok := jiriSubMs[key]; ok {
			if err := addIgnoreToSubmodulesConfig(g, gitSubM); err != nil {
				return err
			}
		}
	}
	return nil
}

func getDiff(gitSubmodules, jiriSubmodules Submodules) (Diff, error) {
	diff := Diff{}
	// Get deleted submodules
	for key, s1 := range gitSubmodules {
		if s1.Name == "" {
			// Submodules that are source-of-truth in the superproject will not
			// have the `name` field set. It is only set for submodules that
			// correspond to Jiri projects. Submodules that intentionally do not
			// correspond to any Jiri project should not be deleted.
			continue
		}
		if _, ok := jiriSubmodules[key]; !ok {
			diff.DeletedSubmodules = append(diff.DeletedSubmodules, DiffSubmodule{
				Path:     s1.Path,
				Revision: s1.Revision,
				Remote:   s1.Remote,
			})
		}
	}

	// Get new and updated submodules
	for key, s2 := range jiriSubmodules {
		if s1, ok := gitSubmodules[key]; !ok {
			diff.NewSubmodules = append(diff.NewSubmodules, DiffSubmodule{
				Name:     s2.Name,
				Path:     s2.Path,
				Revision: s2.Revision,
				Remote:   s2.Remote,
			})
		} else if s1.Remote != s2.Remote {
			// If remote has changed we need to treat it as a delete/add pair.
			// Delete old submodule (with old remote)
			diff.DeletedSubmodules = append(diff.DeletedSubmodules, DiffSubmodule{
				Path:     s1.Path,
				Revision: s1.Revision,
				Remote:   s1.Remote,
			})
			// Add new submodule (with new remote)
			diff.NewSubmodules = append(diff.NewSubmodules, DiffSubmodule{
				Name:     s2.Name,
				Path:     s2.Path,
				Revision: s2.Revision,
				Remote:   s2.Remote,
			})
		} else if s1.Revision != s2.Revision {
			// Revision changed, update to new revision.
			diff.UpdatedSubmodules = append(diff.UpdatedSubmodules, DiffSubmodule{
				Name:        s2.Name,
				Path:        s2.Path,
				Revision:    s2.Revision,
				OldRevision: s1.Revision,
			})
		}
	}
	return diff.sort(), nil
}

func copyFile(srcPath, dstPath string) error {
	sourceFileStat, err := os.Stat(srcPath)
	if err != nil {
		return err
	}

	if !sourceFileStat.Mode().IsRegular() {
		return fmt.Errorf("%s is not a regular file", srcPath)
	}

	data, err := os.ReadFile(srcPath)
	if err != nil {
		return err
	}
	return os.WriteFile(dstPath, data, 0644)
}

func copyCIPDEnsureToSuperproject(snapshotPaths map[string]string, destination string) error {
	for _, srcPath := range snapshotPaths {
		if srcPath == "" {
			continue
		}
		dstPath := path.Join(destination, path.Base(srcPath))
		if err := copyFile(srcPath, dstPath); err != nil {
			return err
		}
	}
	return nil
}

// UpdateSuperproject updates the submodules at superProjectRoot
// to match the jiri project state
func UpdateSuperproject(g *gitutil.Git, message string, jiriProjectsPath string, snapshotPaths map[string]string, outputJSONPath string, noCommit bool, superprojectRoot string) subcommands.ExitStatus {

	gitSubmodules, err := gitSubmodules(g, true)
	if err != nil {
		log.Printf("Error getting git submodules %s", err)
		return subcommands.ExitFailure
	}

	jiriSubmodules, err := jiriProjectsToSubmodule(jiriProjectsPath)
	if err != nil {
		log.Printf("Error parsing jiri projects %s", err)
		return subcommands.ExitFailure
	}

	if err := updateSubmodulesName(g, gitSubmodules, jiriSubmodules); err != nil {
		log.Printf("Error adding project name to submodule config %s", err)
		return subcommands.ExitFailure
	}

	if err := updateSubmodulesIgnore(g, gitSubmodules, jiriSubmodules); err != nil {
		log.Printf("Error adding ignore=diry to submodule config %s", err)
		return subcommands.ExitFailure
	}

	submoduleDiff, err := getDiff(gitSubmodules, jiriSubmodules)
	if err != nil {
		log.Printf("Error diffing submodules: %s", err)
		return subcommands.ExitFailure
	}

	fmt.Printf("Submodule Diff:\n%+v", submoduleDiff)
	// Export submodule diff json to output json
	submoduleDiffJSON, err := json.MarshalIndent(submoduleDiff, "", "  ")
	if err != nil {
		log.Printf("failed to marshal submodule diff to JSON: %s", err)
		return subcommands.ExitFailure
	}

	if err := os.WriteFile(outputJSONPath, submoduleDiffJSON, 0644); err != nil {
		log.Printf("Error writing submoduleDiffJSON to jsonoutput: %s", err)
		return subcommands.ExitFailure
	}

	if err := updateSuperprojectSubmodules(g, submoduleDiff, superprojectRoot); err != nil {
		log.Printf("Error updating superproject: %s", err)
		return subcommands.ExitFailure
	}

	// Skip add files and commit for noCommit Flag
	// auto_roller api expects unstaged changes.
	if !noCommit {
		if err := g.AddAllFiles(); err != nil {
			log.Printf("Error adding files to commit %s", err)
			return subcommands.ExitFailure
		}

		// Make sure there are files to commit
		// This prevents empty commits when, for example, only fuchsia.git is updated.
		files, err := g.FilesWithUncommittedChanges()
		if err != nil {
			log.Printf("Error checking for uncommitted files %s", err)
			return subcommands.ExitFailure
		}

		if len(files) != 0 {
			if err := g.CommitWithMessage(updateCommitMessage(message)); err != nil {
				log.Printf("Error committing to superproject %s", err)
				return subcommands.ExitFailure
			}

		}
	}
	return subcommands.ExitSuccess
}