code.gitea.io/gitea@v1.22.3/services/packages/rpm/repository.go

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package rpm

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	packages_model "code.gitea.io/gitea/models/packages"
	rpm_model "code.gitea.io/gitea/models/packages/rpm"
	user_model "code.gitea.io/gitea/models/user"
	"code.gitea.io/gitea/modules/json"
	packages_module "code.gitea.io/gitea/modules/packages"
	rpm_module "code.gitea.io/gitea/modules/packages/rpm"
	"code.gitea.io/gitea/modules/util"
	packages_service "code.gitea.io/gitea/services/packages"

	"github.com/keybase/go-crypto/openpgp"
	"github.com/keybase/go-crypto/openpgp/armor"
	"github.com/keybase/go-crypto/openpgp/packet"
)

// GetOrCreateRepositoryVersion gets or creates the internal repository package
// The RPM registry needs multiple metadata files which are stored in this package.
func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
	return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeRpm, rpm_module.RepositoryPackage, rpm_module.RepositoryVersion)
}

// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files
func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, error) {
	priv, err := user_model.GetSetting(ctx, ownerID, rpm_module.SettingKeyPrivate)
	if err != nil && !errors.Is(err, util.ErrNotExist) {
		return "", "", err
	}

	pub, err := user_model.GetSetting(ctx, ownerID, rpm_module.SettingKeyPublic)
	if err != nil && !errors.Is(err, util.ErrNotExist) {
		return "", "", err
	}

	if priv == "" || pub == "" {
		priv, pub, err = generateKeypair()
		if err != nil {
			return "", "", err
		}

		if err := user_model.SetUserSetting(ctx, ownerID, rpm_module.SettingKeyPrivate, priv); err != nil {
			return "", "", err
		}

		if err := user_model.SetUserSetting(ctx, ownerID, rpm_module.SettingKeyPublic, pub); err != nil {
			return "", "", err
		}
	}

	return priv, pub, nil
}

func generateKeypair() (string, string, error) {
	e, err := openpgp.NewEntity("", "RPM Registry", "", nil)
	if err != nil {
		return "", "", err
	}

	var priv strings.Builder
	var pub strings.Builder

	w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
	if err != nil {
		return "", "", err
	}
	if err := e.SerializePrivate(w, nil); err != nil {
		return "", "", err
	}
	w.Close()

	w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
	if err != nil {
		return "", "", err
	}
	if err := e.Serialize(w); err != nil {
		return "", "", err
	}
	w.Close()

	return priv.String(), pub.String(), nil
}

// BuildAllRepositoryFiles (re)builds all repository files for every available group
func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
	pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
	if err != nil {
		return err
	}

	// 1. Delete all existing repository files
	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
	if err != nil {
		return err
	}

	for _, pf := range pfs {
		if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
			return err
		}
	}

	// 2. (Re)Build repository files for existing packages
	groups, err := rpm_model.GetGroups(ctx, ownerID)
	if err != nil {
		return err
	}
	for _, group := range groups {
		if err := BuildSpecificRepositoryFiles(ctx, ownerID, group); err != nil {
			return fmt.Errorf("failed to build repository files [%s]: %w", group, err)
		}
	}

	return nil
}

type repoChecksum struct {
	Value string `xml:",chardata"`
	Type  string `xml:"type,attr"`
}

type repoLocation struct {
	Href string `xml:"href,attr"`
}

type repoData struct {
	Type         string       `xml:"type,attr"`
	Checksum     repoChecksum `xml:"checksum"`
	OpenChecksum repoChecksum `xml:"open-checksum"`
	Location     repoLocation `xml:"location"`
	Timestamp    int64        `xml:"timestamp"`
	Size         int64        `xml:"size"`
	OpenSize     int64        `xml:"open-size"`
}

type packageData struct {
	Package         *packages_model.Package
	Version         *packages_model.PackageVersion
	Blob            *packages_model.PackageBlob
	VersionMetadata *rpm_module.VersionMetadata
	FileMetadata    *rpm_module.FileMetadata
}

type packageCache = map[*packages_model.PackageFile]*packageData

// BuildSpecificRepositoryFiles builds metadata files for the repository
func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, group string) error {
	pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
	if err != nil {
		return err
	}

	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
		OwnerID:      ownerID,
		PackageType:  packages_model.TypeRpm,
		Query:        "%.rpm",
		CompositeKey: group,
	})
	if err != nil {
		return err
	}

	// Delete the repository files if there are no packages
	if len(pfs) == 0 {
		pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
		if err != nil {
			return err
		}
		for _, pf := range pfs {
			if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
				return err
			}
		}

		return nil
	}

	// Cache data needed for all repository files
	cache := make(packageCache)
	for _, pf := range pfs {
		pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
		if err != nil {
			return err
		}
		p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
		if err != nil {
			return err
		}
		pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
		if err != nil {
			return err
		}
		pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, rpm_module.PropertyMetadata)
		if err != nil {
			return err
		}

		pd := &packageData{
			Package: p,
			Version: pv,
			Blob:    pb,
		}

		if err := json.Unmarshal([]byte(pv.MetadataJSON), &pd.VersionMetadata); err != nil {
			return err
		}
		if len(pps) > 0 {
			if err := json.Unmarshal([]byte(pps[0].Value), &pd.FileMetadata); err != nil {
				return err
			}
		}

		cache[pf] = pd
	}

	primary, err := buildPrimary(ctx, pv, pfs, cache, group)
	if err != nil {
		return err
	}
	filelists, err := buildFilelists(ctx, pv, pfs, cache, group)
	if err != nil {
		return err
	}
	other, err := buildOther(ctx, pv, pfs, cache, group)
	if err != nil {
		return err
	}

	return buildRepomd(
		ctx,
		pv,
		ownerID,
		[]*repoData{
			primary,
			filelists,
			other,
		},
		group,
	)
}

// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#repomd-xml
func buildRepomd(ctx context.Context, pv *packages_model.PackageVersion, ownerID int64, data []*repoData, group string) error {
	type Repomd struct {
		XMLName  xml.Name    `xml:"repomd"`
		Xmlns    string      `xml:"xmlns,attr"`
		XmlnsRpm string      `xml:"xmlns:rpm,attr"`
		Data     []*repoData `xml:"data"`
	}

	var buf bytes.Buffer
	buf.WriteString(xml.Header)
	if err := xml.NewEncoder(&buf).Encode(&Repomd{
		Xmlns:    "http://linux.duke.edu/metadata/repo",
		XmlnsRpm: "http://linux.duke.edu/metadata/rpm",
		Data:     data,
	}); err != nil {
		return err
	}

	priv, _, err := GetOrCreateKeyPair(ctx, ownerID)
	if err != nil {
		return err
	}

	block, err := armor.Decode(strings.NewReader(priv))
	if err != nil {
		return err
	}

	e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
	if err != nil {
		return err
	}

	repomdAscContent, _ := packages_module.NewHashedBuffer()
	defer repomdAscContent.Close()

	if err := openpgp.ArmoredDetachSign(repomdAscContent, e, bytes.NewReader(buf.Bytes()), nil); err != nil {
		return err
	}

	repomdContent, _ := packages_module.CreateHashedBufferFromReader(&buf)
	defer repomdContent.Close()

	for _, file := range []struct {
		Name string
		Data packages_module.HashedSizeReader
	}{
		{"repomd.xml", repomdContent},
		{"repomd.xml.asc", repomdAscContent},
	} {
		_, err = packages_service.AddFileToPackageVersionInternal(
			ctx,
			pv,
			&packages_service.PackageFileCreationInfo{
				PackageFileInfo: packages_service.PackageFileInfo{
					Filename:     file.Name,
					CompositeKey: group,
				},
				Creator:           user_model.NewGhostUser(),
				Data:              file.Data,
				IsLead:            false,
				OverwriteExisting: true,
			},
		)
		if err != nil {
			return err
		}
	}

	return nil
}

// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#primary-xml
func buildPrimary(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) {
	type Version struct {
		Epoch   string `xml:"epoch,attr"`
		Version string `xml:"ver,attr"`
		Release string `xml:"rel,attr"`
	}

	type Checksum struct {
		Checksum string `xml:",chardata"`
		Type     string `xml:"type,attr"`
		Pkgid    string `xml:"pkgid,attr"`
	}

	type Times struct {
		File  uint64 `xml:"file,attr"`
		Build uint64 `xml:"build,attr"`
	}

	type Sizes struct {
		Package   int64  `xml:"package,attr"`
		Installed uint64 `xml:"installed,attr"`
		Archive   uint64 `xml:"archive,attr"`
	}

	type Location struct {
		Href string `xml:"href,attr"`
	}

	type EntryList struct {
		Entries []*rpm_module.Entry `xml:"rpm:entry"`
	}

	type Format struct {
		License   string             `xml:"rpm:license"`
		Vendor    string             `xml:"rpm:vendor"`
		Group     string             `xml:"rpm:group"`
		Buildhost string             `xml:"rpm:buildhost"`
		Sourcerpm string             `xml:"rpm:sourcerpm"`
		Provides  EntryList          `xml:"rpm:provides"`
		Requires  EntryList          `xml:"rpm:requires"`
		Conflicts EntryList          `xml:"rpm:conflicts"`
		Obsoletes EntryList          `xml:"rpm:obsoletes"`
		Files     []*rpm_module.File `xml:"file"`
	}

	type Package struct {
		XMLName      xml.Name `xml:"package"`
		Type         string   `xml:"type,attr"`
		Name         string   `xml:"name"`
		Architecture string   `xml:"arch"`
		Version      Version  `xml:"version"`
		Checksum     Checksum `xml:"checksum"`
		Summary      string   `xml:"summary"`
		Description  string   `xml:"description"`
		Packager     string   `xml:"packager"`
		URL          string   `xml:"url"`
		Time         Times    `xml:"time"`
		Size         Sizes    `xml:"size"`
		Location     Location `xml:"location"`
		Format       Format   `xml:"format"`
	}

	type Metadata struct {
		XMLName      xml.Name   `xml:"metadata"`
		Xmlns        string     `xml:"xmlns,attr"`
		XmlnsRpm     string     `xml:"xmlns:rpm,attr"`
		PackageCount int        `xml:"packages,attr"`
		Packages     []*Package `xml:"package"`
	}

	packages := make([]*Package, 0, len(pfs))
	for _, pf := range pfs {
		pd := c[pf]

		files := make([]*rpm_module.File, 0, 3)
		for _, f := range pd.FileMetadata.Files {
			if f.IsExecutable {
				files = append(files, f)
			}
		}
		packageVersion := fmt.Sprintf("%s-%s", pd.FileMetadata.Version, pd.FileMetadata.Release)
		packages = append(packages, &Package{
			Type:         "rpm",
			Name:         pd.Package.Name,
			Architecture: pd.FileMetadata.Architecture,
			Version: Version{
				Epoch:   pd.FileMetadata.Epoch,
				Version: pd.FileMetadata.Version,
				Release: pd.FileMetadata.Release,
			},
			Checksum: Checksum{
				Type:     "sha256",
				Checksum: pd.Blob.HashSHA256,
				Pkgid:    "YES",
			},
			Summary:     pd.VersionMetadata.Summary,
			Description: pd.VersionMetadata.Description,
			Packager:    pd.FileMetadata.Packager,
			URL:         pd.VersionMetadata.ProjectURL,
			Time: Times{
				File:  pd.FileMetadata.FileTime,
				Build: pd.FileMetadata.BuildTime,
			},
			Size: Sizes{
				Package:   pd.Blob.Size,
				Installed: pd.FileMetadata.InstalledSize,
				Archive:   pd.FileMetadata.ArchiveSize,
			},
			Location: Location{
				Href: fmt.Sprintf("package/%s/%s/%s/%s-%s.%s.rpm", pd.Package.Name, packageVersion, pd.FileMetadata.Architecture, pd.Package.Name, packageVersion, pd.FileMetadata.Architecture),
			},
			Format: Format{
				License:   pd.VersionMetadata.License,
				Vendor:    pd.FileMetadata.Vendor,
				Group:     pd.FileMetadata.Group,
				Buildhost: pd.FileMetadata.BuildHost,
				Sourcerpm: pd.FileMetadata.SourceRpm,
				Provides: EntryList{
					Entries: pd.FileMetadata.Provides,
				},
				Requires: EntryList{
					Entries: pd.FileMetadata.Requires,
				},
				Conflicts: EntryList{
					Entries: pd.FileMetadata.Conflicts,
				},
				Obsoletes: EntryList{
					Entries: pd.FileMetadata.Obsoletes,
				},
				Files: files,
			},
		})
	}

	return addDataAsFileToRepo(ctx, pv, "primary", &Metadata{
		Xmlns:        "http://linux.duke.edu/metadata/common",
		XmlnsRpm:     "http://linux.duke.edu/metadata/rpm",
		PackageCount: len(pfs),
		Packages:     packages,
	}, group)
}

// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#filelists-xml
func buildFilelists(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl
	type Version struct {
		Epoch   string `xml:"epoch,attr"`
		Version string `xml:"ver,attr"`
		Release string `xml:"rel,attr"`
	}

	type Package struct {
		Pkgid        string             `xml:"pkgid,attr"`
		Name         string             `xml:"name,attr"`
		Architecture string             `xml:"arch,attr"`
		Version      Version            `xml:"version"`
		Files        []*rpm_module.File `xml:"file"`
	}

	type Filelists struct {
		XMLName      xml.Name   `xml:"filelists"`
		Xmlns        string     `xml:"xmlns,attr"`
		PackageCount int        `xml:"packages,attr"`
		Packages     []*Package `xml:"package"`
	}

	packages := make([]*Package, 0, len(pfs))
	for _, pf := range pfs {
		pd := c[pf]

		packages = append(packages, &Package{
			Pkgid:        pd.Blob.HashSHA256,
			Name:         pd.Package.Name,
			Architecture: pd.FileMetadata.Architecture,
			Version: Version{
				Epoch:   pd.FileMetadata.Epoch,
				Version: pd.FileMetadata.Version,
				Release: pd.FileMetadata.Release,
			},
			Files: pd.FileMetadata.Files,
		})
	}

	return addDataAsFileToRepo(ctx, pv, "filelists", &Filelists{
		Xmlns:        "http://linux.duke.edu/metadata/other",
		PackageCount: len(pfs),
		Packages:     packages,
	}, group)
}

// https://docs.pulpproject.org/en/2.19/plugins/pulp_rpm/tech-reference/rpm.html#other-xml
func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*packages_model.PackageFile, c packageCache, group string) (*repoData, error) { //nolint:dupl
	type Version struct {
		Epoch   string `xml:"epoch,attr"`
		Version string `xml:"ver,attr"`
		Release string `xml:"rel,attr"`
	}

	type Package struct {
		Pkgid        string                  `xml:"pkgid,attr"`
		Name         string                  `xml:"name,attr"`
		Architecture string                  `xml:"arch,attr"`
		Version      Version                 `xml:"version"`
		Changelogs   []*rpm_module.Changelog `xml:"changelog"`
	}

	type Otherdata struct {
		XMLName      xml.Name   `xml:"otherdata"`
		Xmlns        string     `xml:"xmlns,attr"`
		PackageCount int        `xml:"packages,attr"`
		Packages     []*Package `xml:"package"`
	}

	packages := make([]*Package, 0, len(pfs))
	for _, pf := range pfs {
		pd := c[pf]

		packages = append(packages, &Package{
			Pkgid:        pd.Blob.HashSHA256,
			Name:         pd.Package.Name,
			Architecture: pd.FileMetadata.Architecture,
			Version: Version{
				Epoch:   pd.FileMetadata.Epoch,
				Version: pd.FileMetadata.Version,
				Release: pd.FileMetadata.Release,
			},
			Changelogs: pd.FileMetadata.Changelogs,
		})
	}

	return addDataAsFileToRepo(ctx, pv, "other", &Otherdata{
		Xmlns:        "http://linux.duke.edu/metadata/other",
		PackageCount: len(pfs),
		Packages:     packages,
	}, group)
}

// writtenCounter counts all written bytes
type writtenCounter struct {
	written int64
}

func (wc *writtenCounter) Write(buf []byte) (int, error) {
	n := len(buf)

	wc.written += int64(n)

	return n, nil
}

func (wc *writtenCounter) Written() int64 {
	return wc.written
}

func addDataAsFileToRepo(ctx context.Context, pv *packages_model.PackageVersion, filetype string, obj any, group string) (*repoData, error) {
	content, _ := packages_module.NewHashedBuffer()
	defer content.Close()

	gzw := gzip.NewWriter(content)
	wc := &writtenCounter{}
	h := sha256.New()

	w := io.MultiWriter(gzw, wc, h)
	_, _ = w.Write([]byte(xml.Header))

	if err := xml.NewEncoder(w).Encode(obj); err != nil {
		return nil, err
	}

	if err := gzw.Close(); err != nil {
		return nil, err
	}

	filename := filetype + ".xml.gz"

	_, err := packages_service.AddFileToPackageVersionInternal(
		ctx,
		pv,
		&packages_service.PackageFileCreationInfo{
			PackageFileInfo: packages_service.PackageFileInfo{
				Filename:     filename,
				CompositeKey: group,
			},
			Creator:           user_model.NewGhostUser(),
			Data:              content,
			IsLead:            false,
			OverwriteExisting: true,
		},
	)
	if err != nil {
		return nil, err
	}

	_, _, hashSHA256, _ := content.Sums()

	return &repoData{
		Type: filetype,
		Checksum: repoChecksum{
			Type:  "sha256",
			Value: hex.EncodeToString(hashSHA256),
		},
		OpenChecksum: repoChecksum{
			Type:  "sha256",
			Value: hex.EncodeToString(h.Sum(nil)),
		},
		Location: repoLocation{
			Href: "repodata/" + filename,
		},
		Timestamp: time.Now().Unix(),
		Size:      content.Size(),
		OpenSize:  wc.Written(),
	}, nil
}