github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/imagebuilder/builder/processManifest.go (about) 1 package builder 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "strings" 10 "time" 11 12 "github.com/Cloud-Foundations/Dominator/lib/filesystem/util" 13 "github.com/Cloud-Foundations/Dominator/lib/format" 14 "github.com/Cloud-Foundations/Dominator/lib/fsutil" 15 "github.com/Cloud-Foundations/Dominator/lib/json" 16 "github.com/Cloud-Foundations/Dominator/lib/srpc" 17 "github.com/Cloud-Foundations/Dominator/lib/verstr" 18 ) 19 20 func deleteDirectories(directoriesToDelete []string) error { 21 for index := len(directoriesToDelete) - 1; index >= 0; index-- { 22 if err := os.Remove(directoriesToDelete[index]); err != nil { 23 return err 24 } 25 } 26 return nil 27 } 28 29 func makeDirectory(directory string, directoriesToDelete []string, 30 directoriesWhichExist map[string]struct{}, 31 bindMountDirectories map[string]struct{}, 32 buildLog io.Writer) ([]string, error) { 33 if _, ok := directoriesWhichExist[directory]; ok { 34 return directoriesToDelete, nil 35 } else if fi, err := os.Stat(directory); err != nil { 36 if !os.IsNotExist(err) { 37 return directoriesToDelete, err 38 } 39 var err error 40 directoriesToDelete, err = makeDirectory(filepath.Dir(directory), 41 directoriesToDelete, directoriesWhichExist, bindMountDirectories, 42 buildLog) 43 if err != nil { 44 return directoriesToDelete, err 45 } 46 if _, ok := bindMountDirectories[directory]; ok { 47 fmt.Fprintf(buildLog, "Making bind mount point: %s\n", directory) 48 } else { 49 fmt.Fprintf(buildLog, 50 "Making intermediate directory for bind mount: %s\n", 51 directory) 52 } 53 if err := os.Mkdir(directory, fsutil.DirPerms); err != nil { 54 return nil, err 55 } 56 directoriesToDelete = append(directoriesToDelete, directory) 57 directoriesWhichExist[directory] = struct{}{} 58 return directoriesToDelete, nil 59 } else if !fi.IsDir() { 60 return directoriesToDelete, 61 fmt.Errorf("%s is not a directory", directory) 62 } else { 63 directoriesWhichExist[directory] = struct{}{} 64 return directoriesToDelete, nil 65 } 66 } 67 68 func makeMountPoints(rootDir string, bindMounts []string, 69 buildLog io.Writer) ([]string, error) { 70 var directoriesToDelete []string 71 directoriesWhichExist := make(map[string]struct{}) 72 defer deleteDirectories(directoriesToDelete) 73 bindMountDirectories := make(map[string]struct{}, len(bindMounts)) 74 for _, bindMount := range bindMounts { 75 bindMountDirectories[filepath.Join(rootDir, bindMount)] = struct{}{} 76 } 77 for _, bindMount := range bindMounts { 78 directory := filepath.Join(rootDir, bindMount) 79 var err error 80 directoriesToDelete, err = makeDirectory(directory, directoriesToDelete, 81 directoriesWhichExist, bindMountDirectories, buildLog) 82 if err != nil { 83 return nil, err 84 } 85 } 86 retval := directoriesToDelete 87 directoriesToDelete = nil // Do not clean up in the defer. 88 return retval, nil 89 } 90 91 func unpackImageAndProcessManifest(client *srpc.Client, manifestDir string, 92 rootDir string, bindMounts []string, applyFilter bool, 93 envGetter environmentGetter, buildLog io.Writer) (manifestType, error) { 94 manifestFile := filepath.Join(manifestDir, "manifest") 95 var manifestConfig manifestConfigType 96 if err := json.ReadFromFile(manifestFile, &manifestConfig); err != nil { 97 return manifestType{}, 98 errors.New("error reading manifest file: " + err.Error()) 99 } 100 sourceImageName := manifestConfig.SourceImage 101 if envGetter != nil { 102 sourceImageName = os.Expand(sourceImageName, func(name string) string { 103 return envGetter.getenv()[name] 104 }) 105 } 106 sourceImageInfo, err := unpackImage(client, sourceImageName, 107 0, 0, rootDir, buildLog) 108 if err != nil { 109 return manifestType{}, 110 errors.New("error unpacking image: " + err.Error()) 111 } 112 startTime := time.Now() 113 err = processManifest(manifestDir, rootDir, bindMounts, envGetter, buildLog) 114 if err != nil { 115 return manifestType{}, 116 errors.New("error processing manifest: " + err.Error()) 117 } 118 if applyFilter && manifestConfig.Filter != nil { 119 err := util.DeletedFilteredFiles(rootDir, manifestConfig.Filter) 120 if err != nil { 121 return manifestType{}, err 122 } 123 } 124 fmt.Fprintf(buildLog, "Processed manifest in %s\n", 125 format.Duration(time.Since(startTime))) 126 return manifestType{manifestConfig.Filter, sourceImageInfo}, nil 127 } 128 129 func processManifest(manifestDir, rootDir string, bindMounts []string, 130 envGetter environmentGetter, buildLog io.Writer) error { 131 // Copy in system /etc/resolv.conf 132 file, err := os.Open("/etc/resolv.conf") 133 if err != nil { 134 return err 135 } 136 defer file.Close() 137 err = runInTarget(file, buildLog, rootDir, envGetter, packagerPathname, 138 "copy-in", "/etc/resolv.conf") 139 if err != nil { 140 return fmt.Errorf("error copying in /etc/resolv.conf: %s", err) 141 } 142 if err := copyFiles(manifestDir, "files", rootDir, buildLog); err != nil { 143 return err 144 } 145 for index, bindMount := range bindMounts { 146 bindMounts[index] = filepath.Clean(bindMount) 147 } 148 directoriesToDelete, err := makeMountPoints(rootDir, bindMounts, buildLog) 149 if err != nil { 150 return err 151 } 152 defer deleteDirectories(directoriesToDelete) 153 err = runScripts(manifestDir, "pre-install-scripts", rootDir, bindMounts, 154 envGetter, buildLog) 155 if err != nil { 156 return err 157 } 158 packageList, err := fsutil.LoadLines(filepath.Join(manifestDir, 159 "package-list")) 160 if err != nil { 161 if !os.IsNotExist(err) { 162 return err 163 } 164 } 165 if len(packageList) > 0 { 166 err := updatePackageDatabase(rootDir, bindMounts, envGetter, buildLog) 167 if err != nil { 168 return err 169 } 170 } 171 err = installPackages(packageList, rootDir, bindMounts, envGetter, buildLog) 172 if err != nil { 173 return errors.New("error installing packages: " + err.Error()) 174 } 175 err = copyFiles(manifestDir, "post-install-files", rootDir, buildLog) 176 if err != nil { 177 return err 178 } 179 err = runScripts(manifestDir, "scripts", rootDir, bindMounts, envGetter, 180 buildLog) 181 if err != nil { 182 return err 183 } 184 if err := cleanPackages(rootDir, buildLog); err != nil { 185 return err 186 } 187 if err := clearResolvConf(buildLog, rootDir); err != nil { 188 return err 189 } 190 err = copyFiles(manifestDir, "post-scripts-files", rootDir, buildLog) 191 if err != nil { 192 return err 193 } 194 return deleteDirectories(directoriesToDelete) 195 } 196 197 func copyFiles(manifestDir, dirname, rootDir string, buildLog io.Writer) error { 198 startTime := time.Now() 199 sourceDir := filepath.Join(manifestDir, dirname) 200 cf := func(destFilename, sourceFilename string, mode os.FileMode) error { 201 return copyFile(destFilename, sourceFilename, mode, len(manifestDir)+1, 202 buildLog) 203 } 204 if err := fsutil.CopyTreeWithCopyFunc(rootDir, sourceDir, cf); err != nil { 205 return fmt.Errorf("error copying %s: %s", dirname, err) 206 } 207 fmt.Fprintf(buildLog, "\nCopied %s tree in %s\n", 208 dirname, format.Duration(time.Since(startTime))) 209 return nil 210 } 211 212 func copyFile(destFilename, sourceFilename string, mode os.FileMode, 213 prefixLength int, buildLog io.Writer) error { 214 same, err := fsutil.CompareFiles(destFilename, sourceFilename) 215 if err != nil && !os.IsNotExist(err) { 216 return err 217 } 218 if same { 219 fmt.Fprintf(buildLog, "Same contents for: %s\n", 220 sourceFilename[prefixLength:]) 221 return nil 222 } 223 return fsutil.CopyFile(destFilename, sourceFilename, mode) 224 } 225 226 func installPackages(packageList []string, rootDir string, bindMounts []string, 227 envGetter environmentGetter, buildLog io.Writer) error { 228 if len(packageList) < 1 { // Nothing to do. 229 fmt.Fprintln(buildLog, "\nNo packages to install") 230 return nil 231 } 232 fmt.Fprintln(buildLog, "\nUpgrading packages:") 233 startTime := time.Now() 234 err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts, 235 envGetter, packagerPathname, "upgrade") 236 if err != nil { 237 return errors.New("error upgrading: " + err.Error()) 238 } 239 fmt.Fprintf(buildLog, "Package upgrade took: %s\n", 240 format.Duration(time.Since(startTime))) 241 242 fmt.Fprintln(buildLog, "\nInstalling packages:", 243 strings.Join(packageList, " ")) 244 startTime = time.Now() 245 args := []string{"install"} 246 args = append(args, packageList...) 247 err = runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts, 248 envGetter, packagerPathname, args...) 249 if err != nil { 250 return errors.New("error installing: " + err.Error()) 251 } 252 fmt.Fprintf(buildLog, "Package install took: %s\n", 253 format.Duration(time.Since(startTime))) 254 return nil 255 } 256 257 func runScripts(manifestDir, dirname, rootDir string, bindMounts []string, 258 envGetter environmentGetter, buildLog io.Writer) error { 259 scriptsDir := filepath.Join(manifestDir, dirname) 260 file, err := os.Open(scriptsDir) 261 if err != nil { 262 if os.IsNotExist(err) { 263 fmt.Fprintf(buildLog, "No %s directory\n", dirname) 264 return nil 265 } 266 return err 267 } 268 names, err := file.Readdirnames(-1) 269 file.Close() 270 if err != nil { 271 return err 272 } 273 if len(names) < 1 { 274 fmt.Fprintln(buildLog, "\nNo scripts to run") 275 return nil 276 } 277 verstr.Sort(names) 278 fmt.Fprintf(buildLog, "\nRunning scripts in: %s\n", dirname) 279 scriptsStartTime := time.Now() 280 tmpDir := filepath.Join(rootDir, ".scripts") 281 if err := os.Mkdir(tmpDir, dirPerms); err != nil { 282 return err 283 } 284 defer os.RemoveAll(tmpDir) 285 for _, name := range names { 286 if len(name) > 0 && name[0] == '.' { 287 continue // Skip hidden paths. 288 } 289 err := fsutil.CopyFile(filepath.Join(tmpDir, name), 290 filepath.Join(scriptsDir, name), 291 dirPerms) 292 if err != nil { 293 return err 294 } 295 } 296 for _, name := range names { 297 fmt.Fprintf(buildLog, "Running script: %s\n", name) 298 startTime := time.Now() 299 err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts, 300 envGetter, packagerPathname, "run", 301 filepath.Join("/.scripts", name)) 302 if err != nil { 303 return errors.New("error running script: " + name + ": " + 304 err.Error()) 305 } 306 timeTaken := time.Since(startTime) 307 fmt.Fprintf(buildLog, "Script: %s took %s\n", 308 name, format.Duration(timeTaken)) 309 fmt.Fprintln(buildLog, 310 "=================================================================") 311 } 312 timeTaken := time.Since(scriptsStartTime) 313 fmt.Fprintf(buildLog, "Ran scripts in %s\n", format.Duration(timeTaken)) 314 return nil 315 } 316 317 func updatePackageDatabase(rootDir string, bindMounts []string, 318 envGetter environmentGetter, buildLog io.Writer) error { 319 fmt.Fprintln(buildLog, "\nUpdating package database:") 320 startTime := time.Now() 321 err := runInTargetWithBindMounts(nil, buildLog, rootDir, bindMounts, 322 envGetter, packagerPathname, "update") 323 if err != nil { 324 return errors.New("error updating: " + err.Error()) 325 } 326 fmt.Fprintf(buildLog, "Package databse update took: %s\n", 327 format.Duration(time.Since(startTime))) 328 return nil 329 }