github.com/turbot/steampipe@v1.7.0-rc.0.0.20240517123944-7cef272d4458/pkg/steampipeconfig/load_mod.go (about) 1 package steampipeconfig 2 3 import ( 4 "context" 5 "fmt" 6 "log" 7 "os" 8 "path/filepath" 9 "strings" 10 11 filehelpers "github.com/turbot/go-kit/files" 12 "github.com/turbot/go-kit/helpers" 13 "github.com/turbot/steampipe-plugin-sdk/v5/plugin" 14 "github.com/turbot/steampipe/pkg/constants" 15 "github.com/turbot/steampipe/pkg/error_helpers" 16 "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" 17 "github.com/turbot/steampipe/pkg/steampipeconfig/parse" 18 "github.com/turbot/steampipe/pkg/steampipeconfig/versionmap" 19 ) 20 21 // LoadMod parses all hcl files in modPath and returns a single mod 22 // if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions 23 // NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable 24 // - a default mod will be created assuming there are any resource files 25 func LoadMod(ctx context.Context, modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings error_helpers.ErrorAndWarnings) { 26 defer func() { 27 if r := recover(); r != nil { 28 errorsAndWarnings = error_helpers.NewErrorsAndWarning(helpers.ToError(r)) 29 } 30 }() 31 32 mod, loadModResult := loadModDefinition(ctx, modPath, parseCtx) 33 if loadModResult.Error != nil { 34 return nil, loadModResult 35 } 36 37 // if this is a dependency mod, initialise the dependency config 38 if parseCtx.DependencyConfig != nil { 39 parseCtx.DependencyConfig.SetModProperties(mod) 40 } 41 42 // set the current mod on the run context 43 if err := parseCtx.SetCurrentMod(mod); err != nil { 44 return nil, error_helpers.NewErrorsAndWarning(err) 45 } 46 47 // load the mod dependencies 48 if err := loadModDependencies(ctx, mod, parseCtx); err != nil { 49 return nil, error_helpers.NewErrorsAndWarning(err) 50 } 51 52 // populate the resource maps of the current mod using the dependency mods 53 mod.ResourceMaps = parseCtx.GetResourceMaps() 54 // now load the mod resource hcl ( 55 mod, errorsAndWarnings = loadModResources(ctx, mod, parseCtx) 56 57 // add in any warnings from mod load 58 errorsAndWarnings.AddWarning(loadModResult.Warnings...) 59 return mod, errorsAndWarnings 60 } 61 62 func loadModDefinition(ctx context.Context, modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errorsAndWarnings error_helpers.ErrorAndWarnings) { 63 errorsAndWarnings = error_helpers.ErrorAndWarnings{} 64 // verify the mod folder exists 65 _, err := os.Stat(modPath) 66 if os.IsNotExist(err) { 67 return nil, error_helpers.NewErrorsAndWarning(fmt.Errorf("mod folder %s does not exist", modPath)) 68 } 69 70 modFilePath, exists := parse.ModfileExists(modPath) 71 if exists { 72 // load the mod definition to get the dependencies 73 var res *parse.DecodeResult 74 mod, res = parse.ParseModDefinition(modFilePath, parseCtx.EvalCtx) 75 errorsAndWarnings = error_helpers.DiagsToErrorsAndWarnings("mod load failed", res.Diags) 76 if res.Diags.HasErrors() { 77 return nil, errorsAndWarnings 78 } 79 } else { 80 // so there is no mod file - should we create a default? 81 if !parseCtx.ShouldCreateDefaultMod() { 82 errorsAndWarnings.Error = fmt.Errorf("mod folder %s does not contain a mod resource definition", modPath) 83 // ShouldCreateDefaultMod flag NOT set - fail 84 return nil, errorsAndWarnings 85 } 86 // just create a default mod 87 mod = modconfig.CreateDefaultMod(modPath) 88 89 } 90 return mod, errorsAndWarnings 91 } 92 93 func loadModDependencies(ctx context.Context, parent *modconfig.Mod, parseCtx *parse.ModParseContext) error { 94 var errors []error 95 if parent.Require != nil { 96 // now ensure there is a lock file - if we have any mod dependnecies there MUST be a lock file - 97 // otherwise 'steampipe install' must be run 98 if err := parseCtx.EnsureWorkspaceLock(parent); err != nil { 99 return err 100 } 101 102 for _, requiredModVersion := range parent.Require.Mods { 103 // get the locked version ofd this dependency 104 lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersion(requiredModVersion, parent) 105 if err != nil { 106 return err 107 } 108 if lockedVersion == nil { 109 return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'") 110 } 111 if err := loadModDependency(ctx, lockedVersion, parseCtx); err != nil { 112 errors = append(errors, err) 113 } 114 } 115 } 116 117 return error_helpers.CombineErrors(errors...) 118 } 119 120 func loadModDependency(ctx context.Context, modDependency *versionmap.ResolvedVersionConstraint, parseCtx *parse.ModParseContext) error { 121 // dependency mods are installed to <mod path>/<mod nam>@version 122 // for example workspace_folder/.steampipe/mods/github.com/turbot/steampipe-mod-aws-compliance@v1.0 123 124 // we need to list all mod folder in the parent folder: workspace_folder/.steampipe/mods/github.com/turbot/ 125 // for each folder we parse the mod name and version and determine whether it meets the version constraint 126 127 // search the parent folder for a mod installation which satisfied the given mod dependency 128 dependencyDir, err := parseCtx.WorkspaceLock.FindInstalledDependency(modDependency) 129 if err != nil { 130 return err 131 } 132 133 // we need to modify the ListOptions to ensure we include hidden files - these are excluded by default 134 prevExclusions := parseCtx.ListOptions.Exclude 135 parseCtx.ListOptions.Exclude = nil 136 defer func() { parseCtx.ListOptions.Exclude = prevExclusions }() 137 138 childParseCtx := parse.NewChildModParseContext(parseCtx, modDependency, dependencyDir) 139 // NOTE: pass in the version and dependency path of the mod - these must be set before it loads its dependencies 140 dependencyMod, errAndWarnings := LoadMod(ctx, dependencyDir, childParseCtx) 141 if errAndWarnings.GetError() != nil { 142 return errAndWarnings.GetError() 143 } 144 145 // update loaded dependency mods 146 parseCtx.AddLoadedDependencyMod(dependencyMod) 147 // TODO IS THIS NEEDED???? 148 if parseCtx.ParentParseCtx != nil { 149 // add mod resources to parent parse context 150 parseCtx.ParentParseCtx.AddModResources(dependencyMod) 151 } 152 return nil 153 154 } 155 156 func loadModResources(ctx context.Context, mod *modconfig.Mod, parseCtx *parse.ModParseContext) (*modconfig.Mod, error_helpers.ErrorAndWarnings) { 157 // if flag is set, create pseudo resources by mapping files 158 var pseudoResources []modconfig.MappableResource 159 var err error 160 if parseCtx.CreatePseudoResources() { 161 // now execute any pseudo-resource creations based on file mappings 162 pseudoResources, err = createPseudoResources(ctx, mod, parseCtx) 163 if err != nil { 164 return nil, error_helpers.NewErrorsAndWarning(err) 165 } 166 } 167 168 // get the source files 169 sourcePaths, err := getSourcePaths(ctx, mod.ModPath, parseCtx.ListOptions) 170 if err != nil { 171 log.Printf("[WARN] LoadMod: failed to get mod file paths: %v\n", err) 172 return nil, error_helpers.NewErrorsAndWarning(err) 173 } 174 175 // load the raw file data 176 fileData, diags := parse.LoadFileData(sourcePaths...) 177 if diags.HasErrors() { 178 return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod files", diags)) 179 } 180 181 // parse all hcl files (NOTE - this reads the CurrentMod out of ParseContext and adds to it) 182 mod, errAndWarnings := parse.ParseMod(ctx, fileData, pseudoResources, parseCtx) 183 184 return mod, errAndWarnings 185 } 186 187 // LoadModResourceNames parses all hcl files in modPath and returns the names of all resources 188 func LoadModResourceNames(ctx context.Context, mod *modconfig.Mod, parseCtx *parse.ModParseContext) (resources *modconfig.WorkspaceResources, err error) { 189 defer func() { 190 if r := recover(); r != nil { 191 err = helpers.ToError(r) 192 } 193 }() 194 195 resources = modconfig.NewWorkspaceResources() 196 if parseCtx == nil { 197 parseCtx = &parse.ModParseContext{} 198 } 199 // verify the mod folder exists 200 if _, err := os.Stat(mod.ModPath); os.IsNotExist(err) { 201 return nil, fmt.Errorf("mod folder %s does not exist", mod.ModPath) 202 } 203 204 // now execute any pseudo-resource creations based on file mappings 205 pseudoResources, err := createPseudoResources(ctx, mod, parseCtx) 206 if err != nil { 207 return nil, err 208 } 209 210 // add pseudo resources to result 211 for _, r := range pseudoResources { 212 if strings.HasPrefix(r.Name(), "query.") || strings.HasPrefix(r.Name(), "local.query.") { 213 resources.Query[r.Name()] = true 214 } 215 } 216 217 sourcePaths, err := getSourcePaths(ctx, mod.ModPath, parseCtx.ListOptions) 218 if err != nil { 219 log.Printf("[WARN] LoadModResourceNames: failed to get mod file paths: %v\n", err) 220 return nil, err 221 } 222 223 fileData, diags := parse.LoadFileData(sourcePaths...) 224 if diags.HasErrors() { 225 return nil, plugin.DiagsToError("Failed to load all mod files", diags) 226 } 227 228 parsedResourceNames, err := parse.ParseModResourceNames(fileData) 229 if err != nil { 230 return nil, err 231 } 232 return resources.Merge(parsedResourceNames), nil 233 } 234 235 // GetModFileExtensions returns list of all file extensions we care about 236 // this will be the mod data extension, plus any registered extensions registered in fileToResourceMap 237 func GetModFileExtensions() []string { 238 res := append(modconfig.RegisteredFileExtensions(), constants.ModDataExtensions...) 239 return append(res, constants.VariablesExtensions...) 240 } 241 242 // build list of all filepaths we need to parse/load the mod 243 // this will include hcl files (with .sp extension) 244 // as well as any other files with extensions that have been registered for pseudo resource creation 245 // (see steampipeconfig/modconfig/resource_type_map.go) 246 func getSourcePaths(ctx context.Context, modPath string, listOpts *filehelpers.ListOptions) ([]string, error) { 247 sourcePaths, err := filehelpers.ListFilesWithContext(ctx, modPath, listOpts) 248 if err != nil { 249 return nil, err 250 } 251 return sourcePaths, nil 252 } 253 254 // create pseudo-resources for any files whose extensions are registered 255 func createPseudoResources(ctx context.Context, mod *modconfig.Mod, parseCtx *parse.ModParseContext) ([]modconfig.MappableResource, error) { 256 // create list options to find pseudo resources 257 listOpts := &filehelpers.ListOptions{ 258 Flags: parseCtx.ListOptions.Flags, 259 Include: filehelpers.InclusionsFromExtensions(modconfig.RegisteredFileExtensions()), 260 Exclude: parseCtx.ListOptions.Exclude, 261 } 262 // list all registered files 263 sourcePaths, err := getSourcePaths(ctx, mod.ModPath, listOpts) 264 if err != nil { 265 return nil, err 266 } 267 268 var errors []error 269 var res []modconfig.MappableResource 270 271 // for every source path: 272 // - if it is NOT a registered type, skip 273 // [- if an existing resource has already referred directly to this file, skip] *not yet* 274 for _, path := range sourcePaths { 275 factory, ok := modconfig.ResourceTypeMap[filepath.Ext(path)] 276 if !ok { 277 continue 278 } 279 resource, fileData, err := factory(mod.ModPath, path, parseCtx.CurrentMod) 280 if err != nil { 281 errors = append(errors, err) 282 continue 283 } 284 if resource != nil { 285 metadata, err := getPseudoResourceMetadata(mod, resource.Name(), path, fileData) 286 if err != nil { 287 return nil, err 288 } 289 resource.SetMetadata(metadata) 290 res = append(res, resource) 291 } 292 } 293 294 // show errors as trace logging 295 if len(errors) > 0 { 296 for _, err := range errors { 297 log.Printf("[TRACE] failed to convert local file into resource: %v", err) 298 } 299 } 300 301 return res, nil 302 } 303 304 func getPseudoResourceMetadata(mod *modconfig.Mod, resourceName string, path string, fileData []byte) (*modconfig.ResourceMetadata, error) { 305 sourceDefinition := string(fileData) 306 split := strings.Split(sourceDefinition, "\n") 307 lineCount := len(split) 308 309 // convert the name into a short name 310 parsedName, err := modconfig.ParseResourceName(resourceName) 311 if err != nil { 312 return nil, err 313 } 314 315 m := &modconfig.ResourceMetadata{ 316 ResourceName: parsedName.Name, 317 FileName: path, 318 StartLineNumber: 1, 319 EndLineNumber: lineCount, 320 IsAutoGenerated: true, 321 SourceDefinition: sourceDefinition, 322 } 323 m.SetMod(mod) 324 325 return m, nil 326 }