github.com/juju/charm/v11@v11.2.0/bundledatasrc.go (about) 1 // Copyright 2019 Canonical Ltd. 2 // Licensed under the LGPLv3, see LICENCE file for details. 3 4 package charm 5 6 import ( 7 "bytes" 8 "io" 9 "io/ioutil" 10 "os" 11 "path/filepath" 12 "strings" 13 14 "github.com/juju/errors" 15 "gopkg.in/yaml.v2" 16 ) 17 18 // FieldPresenceMap indicates which keys of a parsed bundle yaml document were 19 // present when the document was parsed. This map is used by the overlay merge 20 // code to figure out whether empty/nil field values were actually specified as 21 // such in the yaml document. 22 type FieldPresenceMap map[interface{}]interface{} 23 24 func (fpm FieldPresenceMap) fieldPresent(fieldName string) bool { 25 _, exists := fpm[fieldName] 26 return exists 27 } 28 29 func (fpm FieldPresenceMap) forField(fieldName string) FieldPresenceMap { 30 v, exists := fpm[fieldName] 31 if !exists { 32 return nil 33 } 34 35 // Always returns a FieldPresenceMap even if the underlying type is empty. 36 // As the only way to interact with the map is through the use of the two 37 // methods, then it will allow you to walk over the map in a much saner way. 38 asMap, _ := v.(FieldPresenceMap) 39 if asMap == nil { 40 return FieldPresenceMap{} 41 } 42 return asMap 43 } 44 45 // BundleDataPart combines a parsed BundleData instance with a nested map that 46 // can be used to discriminate between fields that are missing from the data 47 // and those that are present but defined to be empty. 48 type BundleDataPart struct { 49 Data *BundleData 50 PresenceMap FieldPresenceMap 51 UnmarshallError error 52 } 53 54 // BundleDataSource is implemented by types that can parse bundle data into a 55 // list of composable parts. 56 type BundleDataSource interface { 57 Parts() []*BundleDataPart 58 BasePath() string 59 ResolveInclude(path string) ([]byte, error) 60 } 61 62 type resolvedBundleDataSource struct { 63 basePath string 64 parts []*BundleDataPart 65 } 66 67 func (s *resolvedBundleDataSource) Parts() []*BundleDataPart { 68 return s.parts 69 } 70 71 func (s *resolvedBundleDataSource) BasePath() string { 72 return s.basePath 73 } 74 75 func (s *resolvedBundleDataSource) ResolveInclude(path string) ([]byte, error) { 76 absPath := path 77 if !filepath.IsAbs(absPath) { 78 var err error 79 absPath, err = filepath.Abs(filepath.Clean(filepath.Join(s.basePath, absPath))) 80 if err != nil { 81 return nil, errors.Annotatef(err, "resolving relative include %q", path) 82 } 83 } 84 85 info, err := os.Stat(absPath) 86 if err != nil { 87 if isNotExistsError(err) { 88 return nil, errors.NotFoundf("include file %q", absPath) 89 } 90 91 return nil, errors.Annotatef(err, "stat failed for %q", absPath) 92 } 93 94 if info.IsDir() { 95 return nil, errors.Errorf("include path %q resolves to a folder", absPath) 96 } 97 98 data, err := ioutil.ReadFile(absPath) 99 if err != nil { 100 return nil, errors.Annotatef(err, "reading include file at %q", absPath) 101 } 102 103 return data, nil 104 } 105 106 // LocalBundleDataSource reads a (potentially multi-part) bundle from path and 107 // returns a BundleDataSource for it. Path may point to a yaml file, a bundle 108 // directory or a bundle archive. 109 func LocalBundleDataSource(path string) (BundleDataSource, error) { 110 info, err := os.Stat(path) 111 if err != nil { 112 if isNotExistsError(err) { 113 return nil, errors.NotFoundf("%q", path) 114 } 115 116 return nil, errors.Annotatef(err, "stat failed for %q", path) 117 } 118 119 // Treat as an exploded bundle archive directory 120 if info.IsDir() { 121 path = filepath.Join(path, "bundle.yaml") 122 } 123 124 // Try parsing as a yaml file first 125 f, err := os.Open(path) 126 if err != nil { 127 if isNotExistsError(err) { 128 return nil, errors.NotFoundf("%q", path) 129 } 130 return nil, errors.Annotatef(err, "access bundle data at %q", path) 131 } 132 defer func() { _ = f.Close() }() 133 134 parts, pErr := parseBundleParts(f) 135 if pErr == nil { 136 absPath, err := filepath.Abs(path) 137 if err != nil { 138 return nil, errors.Annotatef(err, "resolve absolute path to %s", path) 139 } 140 return &resolvedBundleDataSource{ 141 basePath: filepath.Dir(absPath), 142 parts: parts, 143 }, nil 144 } 145 146 // As a fallback, try to parse as a bundle archive 147 zo := newZipOpenerFromPath(path) 148 zrc, err := zo.openZip() 149 if err != nil { 150 // Not a zip file; return the original parse error 151 return nil, errors.NewNotValid(pErr, "cannot unmarshal bundle contents") 152 } 153 defer func() { _ = zrc.Close() }() 154 155 r, err := zipOpenFile(zrc, "bundle.yaml") 156 if err != nil { 157 // It is a zip file but not one that contains a bundle.yaml 158 return nil, errors.NotFoundf("interpret bundle contents as a bundle archive: %v", err) 159 } 160 defer func() { _ = r.Close() }() 161 162 if parts, pErr = parseBundleParts(r); pErr == nil { 163 return &resolvedBundleDataSource{ 164 basePath: "", // use empty base path for archives 165 parts: parts, 166 }, nil 167 } 168 169 return nil, errors.NewNotValid(pErr, "cannot unmarshal bundle contents") 170 } 171 172 func isNotExistsError(err error) bool { 173 if os.IsNotExist(err) { 174 return true 175 } 176 // On Windows, we get a path error due to a GetFileAttributesEx syscall. 177 // To avoid being too proscriptive, we'll simply check for the error 178 // type and not any content. 179 if _, ok := err.(*os.PathError); ok { 180 return true 181 } 182 return false 183 } 184 185 // StreamBundleDataSource reads a (potentially multi-part) bundle from r and 186 // returns a BundleDataSource for it. 187 func StreamBundleDataSource(r io.Reader, basePath string) (BundleDataSource, error) { 188 parts, err := parseBundleParts(r) 189 if err != nil { 190 return nil, errors.NotValidf("cannot unmarshal bundle contents: %v", err) 191 } 192 193 return &resolvedBundleDataSource{parts: parts, basePath: basePath}, nil 194 } 195 196 func parseBundleParts(r io.Reader) ([]*BundleDataPart, error) { 197 b, err := ioutil.ReadAll(r) 198 if err != nil { 199 return nil, err 200 } 201 202 var ( 203 // Ideally, we would be using a single reader and we would 204 // rewind it to read each block in structured and raw mode. 205 // Unfortunately, the yaml parser seems to parse all documents 206 // at once so we need to use two decoders. The third is to allow 207 // for validation of the yaml by using strict decoding. However 208 // we still want to return non strict bundle parts so that 209 // force may be used in deploy. 210 structDec = yaml.NewDecoder(bytes.NewReader(b)) 211 strictDec = yaml.NewDecoder(bytes.NewReader(b)) 212 rawDec = yaml.NewDecoder(bytes.NewReader(b)) 213 parts []*BundleDataPart 214 ) 215 216 for docIdx := 0; ; docIdx++ { 217 var part BundleDataPart 218 219 err = structDec.Decode(&part.Data) 220 if err == io.EOF { 221 break 222 } else if err != nil && !strings.HasPrefix(err.Error(), "yaml: unmarshal errors:") { 223 return nil, errors.Annotatef(err, "unmarshal document %d", docIdx) 224 } 225 226 var data *BundleData 227 strictDec.SetStrict(true) 228 err = strictDec.Decode(&data) 229 if err == io.EOF { 230 break 231 } else if err != nil { 232 if strings.HasPrefix(err.Error(), "yaml: unmarshal errors:") { 233 friendlyErrors := userFriendlyUnmarshalErrors(err) 234 part.UnmarshallError = errors.Annotatef(friendlyErrors, "unmarshal document %d", docIdx) 235 } else { 236 return nil, errors.Annotatef(err, "unmarshal document %d", docIdx) 237 } 238 } 239 240 // We have already checked for errors for the previous unmarshal attempt 241 _ = rawDec.Decode(&part.PresenceMap) 242 parts = append(parts, &part) 243 } 244 245 return parts, nil 246 } 247 248 func userFriendlyUnmarshalErrors(err error) error { 249 logger.Tracef("developer friendly error message: \n%s", err.Error()) 250 friendlyText := err.Error() 251 friendlyText = strings.ReplaceAll(friendlyText, "type charm.ApplicationSpec", "applications") 252 friendlyText = strings.ReplaceAll(friendlyText, "type charm.legacyBundleData", "bundle") 253 friendlyText = strings.ReplaceAll(friendlyText, "type charm.RelationSpec", "relations") 254 friendlyText = strings.ReplaceAll(friendlyText, "type charm.MachineSpec", "machines") 255 friendlyText = strings.ReplaceAll(friendlyText, "type charm.SaasSpec", "saas") 256 return errors.New(friendlyText) 257 }