github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imagebuilder/builder/processManifest.go (about)

     1  package builder
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/Cloud-Foundations/Dominator/lib/filesystem/util"
    13  	"github.com/Cloud-Foundations/Dominator/lib/format"
    14  	"github.com/Cloud-Foundations/Dominator/lib/fsutil"
    15  	"github.com/Cloud-Foundations/Dominator/lib/json"
    16  	"github.com/Cloud-Foundations/Dominator/lib/srpc"
    17  	"github.com/Cloud-Foundations/Dominator/lib/verstr"
    18  )
    19  
    20  func deleteDirectories(directoriesToDelete []string) error {
    21  	for index := len(directoriesToDelete) - 1; index >= 0; index-- {
    22  		if err := os.Remove(directoriesToDelete[index]); err != nil {
    23  			return err
    24  		}
    25  	}
    26  	return nil
    27  }
    28  
    29  func makeDirectory(directory string, directoriesToDelete []string,
    30  	directoriesWhichExist map[string]struct{},
    31  	bindMountDirectories map[string]struct{},
    32  	buildLog io.Writer) ([]string, error) {
    33  	if _, ok := directoriesWhichExist[directory]; ok {
    34  		return directoriesToDelete, nil
    35  	} else if fi, err := os.Stat(directory); err != nil {
    36  		if !os.IsNotExist(err) {
    37  			return directoriesToDelete, err
    38  		}
    39  		var err error
    40  		directoriesToDelete, err = makeDirectory(filepath.Dir(directory),
    41  			directoriesToDelete, directoriesWhichExist, bindMountDirectories,
    42  			buildLog)
    43  		if err != nil {
    44  			return directoriesToDelete, err
    45  		}
    46  		if _, ok := bindMountDirectories[directory]; ok {
    47  			fmt.Fprintf(buildLog, "Making bind mount point: %s\n", directory)
    48  		} else {
    49  			fmt.Fprintf(buildLog,
    50  				"Making intermediate directory for bind mount: %s\n",
    51  				directory)
    52  		}
    53  		if err := os.Mkdir(directory, fsutil.DirPerms); err != nil {
    54  			return nil, err
    55  		}
    56  		directoriesToDelete = append(directoriesToDelete, directory)
    57  		directoriesWhichExist[directory] = struct{}{}
    58  		return directoriesToDelete, nil
    59  	} else if !fi.IsDir() {
    60  		return directoriesToDelete,
    61  			fmt.Errorf("%s is not a directory", directory)
    62  	} else {
    63  		directoriesWhichExist[directory] = struct{}{}
    64  		return directoriesToDelete, nil
    65  	}
    66  }
    67  
    68  func makeMountPoints(rootDir string, bindMounts []string,
    69  	buildLog io.Writer) ([]string, error) {
    70  	var directoriesToDelete []string
    71  	directoriesWhichExist := make(map[string]struct{})
    72  	defer deleteDirectories(directoriesToDelete)
    73  	bindMountDirectories := make(map[string]struct{}, len(bindMounts))
    74  	for _, bindMount := range bindMounts {
    75  		bindMountDirectories[filepath.Join(rootDir, bindMount)] = struct{}{}
    76  	}
    77  	for _, bindMount := range bindMounts {
    78  		directory := filepath.Join(rootDir, bindMount)
    79  		var err error
    80  		directoriesToDelete, err = makeDirectory(directory, directoriesToDelete,
    81  			directoriesWhichExist, bindMountDirectories, buildLog)
    82  		if err != nil {
    83  			return nil, err
    84  		}
    85  	}
    86  	retval := directoriesToDelete
    87  	directoriesToDelete = nil // Do not clean up in the defer.
    88  	return retval, nil
    89  }
    90  
    91  func unpackImageAndProcessManifest(client *srpc.Client, manifestDir string,
    92  	rootDir string, bindMounts []string, applyFilter bool,
    93  	envGetter environmentGetter, buildLog io.Writer) (manifestType, error) {
    94  	manifestFile := filepath.Join(manifestDir, "manifest")
    95  	var manifestConfig manifestConfigType
    96  	if err := json.ReadFromFile(manifestFile, &manifestConfig); err != nil {
    97  		return manifestType{},
    98  			errors.New("error reading manifest file: " + err.Error())
    99  	}
   100  	sourceImageName := manifestConfig.SourceImage
   101  	if envGetter != nil {
   102  		sourceImageName = os.Expand(sourceImageName, func(name string) string {
   103  			return envGetter.getenv()[name]
   104  		})
   105  	}
   106  	sourceImageInfo, err := unpackImage(client, sourceImageName,
   107  		0, 0, rootDir, buildLog)
   108  	if err != nil {
   109  		return manifestType{},
   110  			errors.New("error unpacking image: " + err.Error())
   111  	}
   112  	startTime := time.Now()
   113  	err = processManifest(manifestDir, rootDir, bindMounts, envGetter, buildLog)
   114  	if err != nil {
   115  		return manifestType{},
   116  			errors.New("error processing manifest: " + err.Error())
   117  	}
   118  	if applyFilter && manifestConfig.Filter != nil {
   119  		err := util.DeletedFilteredFiles(rootDir, manifestConfig.Filter)
   120  		if err != nil {
   121  			return manifestType{}, err
   122  		}
   123  	}
   124  	fmt.Fprintf(buildLog, "Processed manifest in %s\n",
   125  		format.Duration(time.Since(startTime)))
   126  	return manifestType{manifestConfig.Filter, sourceImageInfo}, nil
   127  }
   128  
   129  func processManifest(manifestDir, rootDir string, bindMounts []string,
   130  	envGetter environmentGetter, buildLog io.Writer) error {
   131  	// Copy in system /etc/resolv.conf
   132  	file, err := os.Open("/etc/resolv.conf")
   133  	if err != nil {
   134  		return err
   135  	}
   136  	defer file.Close()
   137  	err = runInTarget(file, buildLog, rootDir, envGetter, packagerPathname,
   138  		"copy-in", "/etc/resolv.conf")
   139  	if err != nil {
   140  		return fmt.Errorf("error copying in /etc/resolv.conf: %s", err)
   141  	}
   142  	if err := copyFiles(manifestDir, "files", rootDir, buildLog); err != nil {
   143  		return err
   144  	}
   145  	for index, bindMount := range bindMounts {
   146  		bindMounts[index] = filepath.Clean(bindMount)
   147  	}
   148  	directoriesToDelete, err := makeMountPoints(rootDir, bindMounts, buildLog)
   149  	if err != nil {
   150  		return err
   151  	}
   152  	defer deleteDirectories(directoriesToDelete)
   153  	err = runScripts(manifestDir, "pre-install-scripts", rootDir, bindMounts,
   154  		envGetter, buildLog)
   155  	if err != nil {
   156  		return err
   157  	}
   158  	packageList, err := fsutil.LoadLines(filepath.Join(manifestDir,
   159  		"package-list"))
   160  	if err != nil {
   161  		if !os.IsNotExist(err) {
   162  			return err
   163  		}
   164  	}
   165  	if len(packageList) > 0 {
   166  		err := updatePackageDatabase(rootDir, bindMounts, envGetter, buildLog)
   167  		if err != nil {
   168  			return err
   169  		}
   170  	}
   171  	err = installPackages(packageList, rootDir, bindMounts, envGetter, buildLog)
   172  	if err != nil {
   173  		return errors.New("error installing packages: " + err.Error())
   174  	}
   175  	err = copyFiles(manifestDir, "post-install-files", rootDir, buildLog)
   176  	if err != nil {
   177  		return err
   178  	}
   179  	err = runScripts(manifestDir, "scripts", rootDir, bindMounts, envGetter,
   180  		buildLog)
   181  	if err != nil {
   182  		return err
   183  	}
   184  	if err := cleanPackages(rootDir, buildLog); err != nil {
   185  		return err
   186  	}
   187  	if err := clearResolvConf(buildLog, rootDir); err != nil {
   188  		return err
   189  	}
   190  	err = copyFiles(manifestDir, "post-scripts-files", rootDir, buildLog)
   191  	if err != nil {
   192  		return err
   193  	}
   194  	return deleteDirectories(directoriesToDelete)
   195  }
   196  
   197  func copyFiles(manifestDir, dirname, rootDir string, buildLog io.Writer) error {
   198  	startTime := time.Now()
   199  	sourceDir := filepath.Join(manifestDir, dirname)
   200  	cf := func(destFilename, sourceFilename string, mode os.FileMode) error {
   201  		return copyFile(destFilename, sourceFilename, mode, len(manifestDir)+1,
   202  			buildLog)
   203  	}
   204  	if err := fsutil.CopyTreeWithCopyFunc(rootDir, sourceDir, cf); err != nil {
   205  		return fmt.Errorf("error copying %s: %s", dirname, err)
   206  	}
   207  	fmt.Fprintf(buildLog, "\nCopied %s tree in %s\n",
   208  		dirname, format.Duration(time.Since(startTime)))
   209  	return nil
   210  }
   211  
   212  func copyFile(destFilename, sourceFilename string, mode os.FileMode,
   213  	prefixLength int, buildLog io.Writer) error {
   214  	same, err := fsutil.CompareFiles(destFilename, sourceFilename)
   215  	if err != nil && !os.IsNotExist(err) {
   216  		return err
   217  	}
   218  	if same {
   219  		fmt.Fprintf(buildLog, "Same contents for: %s\n",
   220  			sourceFilename[prefixLength:])
   221  		return nil
   222  	}
   223  	return fsutil.CopyFile(destFilename, sourceFilename, mode)
   224  }
   225  
   226  func installPackages(packageList []string, rootDir string, bindMounts []string,
   227  	envGetter environmentGetter, buildLog io.Writer) error {
   228  	if len(packageList) < 1 { // Nothing to do.
   229  		fmt.Fprintln(buildLog, "\nNo packages to install")
   230  		return nil
   231  	}
   232  	fmt.Fprintln(buildLog, "\nUpgrading packages:")
   233  	startTime := time.Now()
   234  	err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts,
   235  		envGetter, packagerPathname, "upgrade")
   236  	if err != nil {
   237  		return errors.New("error upgrading: " + err.Error())
   238  	}
   239  	fmt.Fprintf(buildLog, "Package upgrade took: %s\n",
   240  		format.Duration(time.Since(startTime)))
   241  
   242  	fmt.Fprintln(buildLog, "\nInstalling packages:",
   243  		strings.Join(packageList, " "))
   244  	startTime = time.Now()
   245  	args := []string{"install"}
   246  	args = append(args, packageList...)
   247  	err = runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts,
   248  		envGetter, packagerPathname, args...)
   249  	if err != nil {
   250  		return errors.New("error installing: " + err.Error())
   251  	}
   252  	fmt.Fprintf(buildLog, "Package install took: %s\n",
   253  		format.Duration(time.Since(startTime)))
   254  	return nil
   255  }
   256  
   257  func runScripts(manifestDir, dirname, rootDir string, bindMounts []string,
   258  	envGetter environmentGetter, buildLog io.Writer) error {
   259  	scriptsDir := filepath.Join(manifestDir, dirname)
   260  	file, err := os.Open(scriptsDir)
   261  	if err != nil {
   262  		if os.IsNotExist(err) {
   263  			fmt.Fprintf(buildLog, "No %s directory\n", dirname)
   264  			return nil
   265  		}
   266  		return err
   267  	}
   268  	names, err := file.Readdirnames(-1)
   269  	file.Close()
   270  	if err != nil {
   271  		return err
   272  	}
   273  	if len(names) < 1 {
   274  		fmt.Fprintln(buildLog, "\nNo scripts to run")
   275  		return nil
   276  	}
   277  	verstr.Sort(names)
   278  	fmt.Fprintf(buildLog, "\nRunning scripts in: %s\n", dirname)
   279  	scriptsStartTime := time.Now()
   280  	tmpDir := filepath.Join(rootDir, ".scripts")
   281  	if err := os.Mkdir(tmpDir, dirPerms); err != nil {
   282  		return err
   283  	}
   284  	defer os.RemoveAll(tmpDir)
   285  	for _, name := range names {
   286  		if len(name) > 0 && name[0] == '.' {
   287  			continue // Skip hidden paths.
   288  		}
   289  		err := fsutil.CopyFile(filepath.Join(tmpDir, name),
   290  			filepath.Join(scriptsDir, name),
   291  			dirPerms)
   292  		if err != nil {
   293  			return err
   294  		}
   295  	}
   296  	for _, name := range names {
   297  		fmt.Fprintf(buildLog, "Running script: %s\n", name)
   298  		startTime := time.Now()
   299  		err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts,
   300  			envGetter, packagerPathname, "run",
   301  			filepath.Join("/.scripts", name))
   302  		if err != nil {
   303  			return errors.New("error running script: " + name + ": " +
   304  				err.Error())
   305  		}
   306  		timeTaken := time.Since(startTime)
   307  		fmt.Fprintf(buildLog, "Script: %s took %s\n",
   308  			name, format.Duration(timeTaken))
   309  		fmt.Fprintln(buildLog,
   310  			"=================================================================")
   311  	}
   312  	timeTaken := time.Since(scriptsStartTime)
   313  	fmt.Fprintf(buildLog, "Ran scripts in %s\n", format.Duration(timeTaken))
   314  	return nil
   315  }
   316  
   317  func updatePackageDatabase(rootDir string, bindMounts []string,
   318  	envGetter environmentGetter, buildLog io.Writer) error {
   319  	fmt.Fprintln(buildLog, "\nUpdating package database:")
   320  	startTime := time.Now()
   321  	err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts,
   322  		envGetter, packagerPathname, "update")
   323  	if err != nil {
   324  		return errors.New("error updating: " + err.Error())
   325  	}
   326  	fmt.Fprintf(buildLog, "Package databse update took: %s\n",
   327  		format.Duration(time.Since(startTime)))
   328  	return nil
   329  }