github.com/canonical/ubuntu-image@v0.0.0-20240430122802-2202fe98b290/internal/statemachine/classic.go (about) 1 package statemachine 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 9 "github.com/invopop/jsonschema" 10 "github.com/xeipuuv/gojsonschema" 11 "gopkg.in/yaml.v2" 12 13 "github.com/canonical/ubuntu-image/internal/commands" 14 "github.com/canonical/ubuntu-image/internal/imagedefinition" 15 ) 16 17 var rootfsSeedStates = []stateFunc{ 18 germinateState, 19 createChrootState, 20 } 21 22 var imageCreationStates = []stateFunc{ 23 calculateRootfsSizeState, 24 populateBootfsContentsState, 25 populatePreparePartitionsState, 26 } 27 28 // ClassicStateMachine embeds StateMachine and adds the command line flags specific to classic images 29 type ClassicStateMachine struct { 30 StateMachine 31 ImageDef imagedefinition.ImageDefinition 32 Args commands.ClassicArgs 33 } 34 35 // Setup assigns variables and calls other functions that must be executed before Run() 36 func (classicStateMachine *ClassicStateMachine) Setup() error { 37 // set the parent pointer of the embedded struct 38 classicStateMachine.parent = classicStateMachine 39 40 classicStateMachine.states = make([]stateFunc, 0) 41 42 if err := classicStateMachine.setConfDefDir(classicStateMachine.parent.(*ClassicStateMachine).Args.ImageDefinition); err != nil { 43 return err 44 } 45 46 // do the validation common to all image types 47 if err := classicStateMachine.validateInput(); err != nil { 48 return err 49 } 50 51 if err := classicStateMachine.parseImageDefinition(); err != nil { 52 return err 53 } 54 55 if err := classicStateMachine.calculateStates(); err != nil { 56 return err 57 } 58 59 // validate values of until and thru 60 if err := classicStateMachine.validateUntilThru(); err != nil { 61 return err 62 } 63 64 // if --resume was passed, figure out where to start 65 if err := classicStateMachine.readMetadata(metadataStateFile); err != nil { 66 return err 67 } 68 69 classicStateMachine.displayStates() 70 71 if classicStateMachine.commonFlags.DryRun { 72 return nil 73 } 74 75 if err := classicStateMachine.makeTemporaryDirectories(); err != nil { 76 return err 77 } 78 79 return classicStateMachine.determineOutputDirectory() 80 } 81 82 // parseImageDefinition parses the provided yaml file and ensures it is valid 83 func (stateMachine *StateMachine) parseImageDefinition() error { 84 classicStateMachine := stateMachine.parent.(*ClassicStateMachine) 85 86 imageDefinition, err := readImageDefinition(classicStateMachine.Args.ImageDefinition) 87 if err != nil { 88 return err 89 } 90 91 if imageDefinition.Rootfs != nil && imageDefinition.Rootfs.SourcesListDeb822 == nil { 92 fmt.Print("WARNING: rootfs.sources-list-deb822 was not set. Please explicitly set the format desired for sources list in your image definition.\n") 93 } 94 95 // populate the default values for imageDefinition if they were not provided in 96 // the image definition YAML file 97 if err := helperSetDefaults(imageDefinition); err != nil { 98 return err 99 } 100 101 if imageDefinition.Rootfs != nil && *imageDefinition.Rootfs.SourcesListDeb822 { 102 fmt.Print("WARNING: rootfs.sources-list-deb822 is set to true. The DEB822 format will be used to manage sources list. Please make sure you are not building an image older than noble.\n") 103 } else { 104 fmt.Print("WARNING: rootfs.sources-list-deb822 is set to false. The deprecated format will be used to manage sources list. Please if possible adopt the new format.\n") 105 } 106 107 err = validateImageDefinition(imageDefinition) 108 if err != nil { 109 return err 110 } 111 112 classicStateMachine.ImageDef = *imageDefinition 113 114 return nil 115 } 116 117 func readImageDefinition(imageDefPath string) (*imagedefinition.ImageDefinition, error) { 118 imageDefinition := &imagedefinition.ImageDefinition{} 119 imageFile, err := os.Open(imageDefPath) 120 if err != nil { 121 return nil, fmt.Errorf("Error opening image definition file: %s", err.Error()) 122 } 123 defer imageFile.Close() 124 if err := yaml.NewDecoder(imageFile).Decode(imageDefinition); err != nil { 125 return nil, err 126 } 127 128 return imageDefinition, nil 129 } 130 131 // validateImageDefinition validates the given imageDefinition 132 // The official standard for YAML schemas states that they are an extension of 133 // JSON schema draft 4. We therefore validate the decoded YAML against a JSON 134 // schema. The workflow is as follows: 135 // 1. Use the jsonschema library to generate a schema from the struct definition 136 // 2. Load the created schema and parsed yaml into types defined by gojsonschema 137 // 3. Use the gojsonschema library to validate the parsed YAML against the schema 138 func validateImageDefinition(imageDefinition *imagedefinition.ImageDefinition) error { 139 var jsonReflector jsonschema.Reflector 140 141 // 1. parse the ImageDefinition struct into a schema using the jsonschema tags 142 schema := jsonReflector.Reflect(imagedefinition.ImageDefinition{}) 143 144 // 2. load the schema and parsed YAML data into types understood by gojsonschema 145 schemaLoader := gojsonschema.NewGoLoader(schema) 146 imageDefinitionLoader := gojsonschema.NewGoLoader(imageDefinition) 147 148 // 3. validate the parsed data against the schema 149 result, err := gojsonschemaValidate(schemaLoader, imageDefinitionLoader) 150 if err != nil { 151 return fmt.Errorf("Schema validation returned an error: %s", err.Error()) 152 } 153 154 err = validateGadget(imageDefinition, result) 155 if err != nil { 156 return err 157 } 158 159 err = validateCustomization(imageDefinition, result) 160 if err != nil { 161 return err 162 } 163 164 // TODO: I've created a PR upstream in xeipuuv/gojsonschema 165 // https://github.com/xeipuuv/gojsonschema/pull/352 166 // if it gets merged this can be removed 167 err = helperCheckEmptyFields(imageDefinition, result, schema) 168 if err != nil { 169 return err 170 } 171 172 if !result.Valid() { 173 return fmt.Errorf("Schema validation failed: %s", result.Errors()) 174 } 175 176 return nil 177 } 178 179 // validateCustomization validates the Gadget section of the image definition 180 func validateGadget(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result) error { 181 // Do custom validation for gadgetURL being required if gadget is not pre-built 182 if imageDefinition.Gadget != nil { 183 if imageDefinition.Gadget.GadgetType != "prebuilt" && imageDefinition.Gadget.GadgetURL == "" { 184 jsonContext := gojsonschema.NewJsonContext("gadget_validation", nil) 185 errDetail := gojsonschema.ErrorDetails{ 186 "key": "gadget:type", 187 "value": imageDefinition.Gadget.GadgetType, 188 } 189 result.AddError( 190 imagedefinition.NewMissingURLError( 191 gojsonschema.NewJsonContext("missingURL", jsonContext), 192 52, 193 errDetail, 194 ), 195 errDetail, 196 ) 197 } 198 } else { 199 diskUsed, err := helperCheckTags(imageDefinition.Artifacts, "is_disk") 200 if err != nil { 201 return fmt.Errorf("Error checking struct tags for Artifacts: \"%s\"", err.Error()) 202 } 203 if diskUsed != "" { 204 jsonContext := gojsonschema.NewJsonContext("image_without_gadget", nil) 205 errDetail := gojsonschema.ErrorDetails{ 206 "key1": diskUsed, 207 "key2": "gadget:", 208 } 209 result.AddError( 210 imagedefinition.NewDependentKeyError( 211 gojsonschema.NewJsonContext("dependentKey", jsonContext), 212 52, 213 errDetail, 214 ), 215 errDetail, 216 ) 217 } 218 } 219 220 return nil 221 } 222 223 // validateCustomization validates the Customization section of the image definition 224 func validateCustomization(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result) error { 225 if imageDefinition.Customization == nil { 226 return nil 227 } 228 229 validateExtraPPAs(imageDefinition, result) 230 if imageDefinition.Customization.Manual != nil { 231 jsonContext := gojsonschema.NewJsonContext("manual_path_validation", nil) 232 validateManualMakeDirs(imageDefinition, result, jsonContext) 233 validateManualCopyFile(imageDefinition, result, jsonContext) 234 validateManualTouchFile(imageDefinition, result, jsonContext) 235 } 236 237 return nil 238 } 239 240 // validateExtraPPAs validates the Customization.ExtraPPAs section of the image definition 241 func validateExtraPPAs(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result) { 242 for _, p := range imageDefinition.Customization.ExtraPPAs { 243 if p.Auth != "" && p.Fingerprint == "" { 244 jsonContext := gojsonschema.NewJsonContext("ppa_validation", nil) 245 errDetail := gojsonschema.ErrorDetails{ 246 "ppaName": p.Name, 247 } 248 result.AddError( 249 imagedefinition.NewInvalidPPAError( 250 gojsonschema.NewJsonContext("missingPrivatePPAFingerprint", 251 jsonContext), 252 52, 253 errDetail, 254 ), 255 errDetail, 256 ) 257 } 258 } 259 } 260 261 // validateManualMakeDirs validates the Customization.Manual.MakeDirs section of the image definition 262 func validateManualMakeDirs(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result, jsonContext *gojsonschema.JsonContext) { 263 if imageDefinition.Customization.Manual.MakeDirs == nil { 264 return 265 } 266 for _, mkdir := range imageDefinition.Customization.Manual.MakeDirs { 267 validateAbsolutePath(mkdir.Path, "customization:manual:mkdir:destination", result, jsonContext) 268 } 269 } 270 271 // validateManualCopyFile validates the Customization.Manual.CopyFile section of the image definition 272 func validateManualCopyFile(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result, jsonContext *gojsonschema.JsonContext) { 273 if imageDefinition.Customization.Manual.CopyFile == nil { 274 return 275 } 276 for _, copy := range imageDefinition.Customization.Manual.CopyFile { 277 validateAbsolutePath(copy.Dest, "customization:manual:copy-file:destination", result, jsonContext) 278 } 279 } 280 281 // validateManualTouchFile validates the Customization.Manual.TouchFile section of the image definition 282 func validateManualTouchFile(imageDefinition *imagedefinition.ImageDefinition, result *gojsonschema.Result, jsonContext *gojsonschema.JsonContext) { 283 if imageDefinition.Customization.Manual.TouchFile == nil { 284 return 285 } 286 for _, touch := range imageDefinition.Customization.Manual.TouchFile { 287 validateAbsolutePath(touch.TouchPath, "customization:manual:touch-file:path", result, jsonContext) 288 } 289 } 290 291 // validateAbsolutePath validates the 292 func validateAbsolutePath(path string, errorKey string, result *gojsonschema.Result, jsonContext *gojsonschema.JsonContext) { 293 // XXX: filepath.IsAbs() does returns true for paths like ../../../something 294 // and those are NOT absolute paths. 295 if !filepath.IsAbs(path) || strings.Contains(path, "/../") { 296 errDetail := gojsonschema.ErrorDetails{ 297 "key": errorKey, 298 "value": path, 299 } 300 result.AddError( 301 imagedefinition.NewPathNotAbsoluteError( 302 gojsonschema.NewJsonContext("nonAbsoluteManualPath", 303 jsonContext), 304 52, 305 errDetail, 306 ), 307 errDetail, 308 ) 309 } 310 } 311 312 // calculateStates dynamically calculates all the states 313 // needed to build the image, as defined by the image-definition file 314 // that was loaded previously. 315 // If a new possible state is added to the classic build state machine, it 316 // should be added here (usually basing on contents of the image definition) 317 func (s *StateMachine) calculateStates() error { 318 c := s.parent.(*ClassicStateMachine) 319 320 var rootfsCreationStates []stateFunc 321 322 if c.ImageDef.Gadget != nil { 323 s.addGadgetStates(&rootfsCreationStates) 324 } 325 326 if c.ImageDef.Artifacts != nil { 327 // if artifacts are specified, verify the correctness and store them in the struct 328 diskUsed, err := helperCheckTags(c.ImageDef.Artifacts, "is_disk") 329 if err != nil { 330 return fmt.Errorf("Error checking struct tags for Artifacts: \"%s\"", err.Error()) 331 } 332 if diskUsed != "" { 333 rootfsCreationStates = append(rootfsCreationStates, verifyArtifactNamesState) 334 } 335 } 336 337 // determine the states needed for preparing the rootfs. 338 // The rootfs is either created from a seed, from 339 // archive-tasks or as a prebuilt tarball. These 340 // options are mutually exclusive and have been validated 341 // by the schema already 342 if c.ImageDef.Rootfs.Tarball != nil { 343 s.addRootfsFromTarballStates(&rootfsCreationStates) 344 } else if c.ImageDef.Rootfs.Seed != nil { 345 s.addRootfsFromSeedStates(&rootfsCreationStates) 346 } else { 347 rootfsCreationStates = append(rootfsCreationStates, buildRootfsFromTasksState) 348 } 349 350 // Before customization, make sure we clean unwanted secrets/values that 351 // are supposed to be unique per machine 352 rootfsCreationStates = append(rootfsCreationStates, cleanRootfsState) 353 354 rootfsCreationStates = append(rootfsCreationStates, customizeSourcesListState) 355 356 if c.ImageDef.Customization != nil { 357 s.addCustomizationStates(&rootfsCreationStates) 358 } 359 360 // Make sure that the rootfs has the correct locale set 361 rootfsCreationStates = append(rootfsCreationStates, setDefaultLocaleState) 362 363 // The rootfs is laid out in a staging area, now populate it in the correct location 364 rootfsCreationStates = append(rootfsCreationStates, populateClassicRootfsContentsState) 365 366 if s.commonFlags.DiskInfo != "" { 367 rootfsCreationStates = append(rootfsCreationStates, generateDiskInfoState) 368 } 369 370 s.addArtifactsStates(c, &rootfsCreationStates) 371 372 // Append the newly calculated states to the slice of funcs in the parent struct 373 s.states = append(s.states, rootfsCreationStates...) 374 375 return nil 376 } 377 378 func (s *StateMachine) addGadgetStates(states *[]stateFunc) { 379 c := s.parent.(*ClassicStateMachine) 380 381 // determine the states needed for preparing the gadget 382 switch c.ImageDef.Gadget.GadgetType { 383 case "git", "directory": 384 *states = append(*states, buildGadgetTreeState) 385 fallthrough 386 case "prebuilt": 387 *states = append(*states, prepareGadgetTreeState) 388 } 389 390 // Load the gadget yaml after the gadget is built 391 *states = append(*states, loadGadgetYamlState) 392 } 393 394 func (s *StateMachine) addRootfsFromTarballStates(states *[]stateFunc) { 395 c := s.parent.(*ClassicStateMachine) 396 397 *states = append(*states, extractRootfsTarState) 398 if c.ImageDef.Customization == nil { 399 return 400 } 401 402 if len(c.ImageDef.Customization.ExtraPPAs) > 0 { 403 *states = append(*states, 404 []stateFunc{ 405 addExtraPPAsState, 406 installPackagesState, 407 cleanExtraPPAsState, 408 }...) 409 } else if len(c.ImageDef.Customization.ExtraPackages) > 0 { 410 *states = append(*states, installPackagesState) 411 } 412 413 if len(c.ImageDef.Customization.ExtraSnaps) > 0 { 414 *states = append(*states, 415 []stateFunc{ 416 prepareClassicImageState, 417 preseedClassicImageState, 418 }...) 419 } 420 } 421 422 func (s *StateMachine) addRootfsFromSeedStates(states *[]stateFunc) { 423 c := s.parent.(*ClassicStateMachine) 424 425 *states = append(*states, rootfsSeedStates...) 426 427 if c.ImageDef.Customization == nil { 428 *states = append(*states, installPackagesState) 429 } else if len(c.ImageDef.Customization.ExtraPPAs) > 0 { 430 *states = append(*states, 431 []stateFunc{ 432 addExtraPPAsState, 433 installPackagesState, 434 cleanExtraPPAsState, 435 }...) 436 } else { 437 *states = append(*states, installPackagesState) 438 } 439 440 *states = append(*states, 441 []stateFunc{ 442 prepareClassicImageState, 443 preseedClassicImageState, 444 }..., 445 ) 446 } 447 448 // addCustomizationStates determines any customization that needs to run before the image 449 // is created 450 // TODO: installer image customization... eventually. 451 func (s *StateMachine) addCustomizationStates(states *[]stateFunc) { 452 c := s.parent.(*ClassicStateMachine) 453 454 if c.ImageDef.Customization.CloudInit != nil { 455 *states = append(*states, customizeCloudInitState) 456 } 457 if len(c.ImageDef.Customization.Fstab) > 0 { 458 *states = append(*states, customizeFstabState) 459 } 460 if c.ImageDef.Customization.Manual != nil { 461 *states = append(*states, manualCustomizationState) 462 } 463 } 464 465 // addArtifactsStates adds the needed states to generates the artifacts 466 func (s *StateMachine) addArtifactsStates(c *ClassicStateMachine, states *[]stateFunc) { 467 if c.ImageDef.Artifacts == nil { 468 return 469 } 470 if c.ImageDef.Gadget != nil { 471 s.addImgStates(states) 472 } 473 474 if c.ImageDef.Artifacts.Qcow2 != nil { 475 s.addQcow2States(states) 476 } 477 478 if c.ImageDef.Artifacts.Manifest != nil { 479 *states = append(*states, generatePackageManifestState) 480 } 481 482 if c.ImageDef.Artifacts.Filelist != nil { 483 *states = append(*states, generateFilelistState) 484 } 485 486 if c.ImageDef.Artifacts.RootfsTar != nil { 487 *states = append(*states, generateRootfsTarballState) 488 } 489 } 490 491 func (s *StateMachine) addImgStates(states *[]stateFunc) { 492 c := s.parent.(*ClassicStateMachine) 493 *states = append(*states, imageCreationStates...) 494 495 if c.ImageDef.Artifacts.Img == nil { 496 return 497 } 498 499 *states = append(*states, 500 makeDiskState, 501 updateBootloaderState, 502 ) 503 } 504 505 func (s *StateMachine) addQcow2States(states *[]stateFunc) { 506 // Only run make_disk once 507 found := false 508 for _, stateFunc := range *states { 509 if stateFunc.name == makeDiskState.name { 510 found = true 511 } 512 } 513 if !found { 514 *states = append(*states, 515 makeDiskState, 516 updateBootloaderState, 517 ) 518 } 519 *states = append(*states, makeQcow2ImgState) 520 }