github.com/jenkins-x/jx/v2@v2.1.155/pkg/versionstream/version_data.go (about) 1 package versionstream 2 3 import ( 4 "fmt" 5 "sort" 6 7 "github.com/blang/semver" 8 "github.com/jenkins-x/jx-logging/pkg/log" 9 "github.com/jenkins-x/jx/v2/pkg/util" 10 "github.com/pkg/errors" 11 "sigs.k8s.io/yaml" 12 13 "io/ioutil" 14 "os" 15 "path/filepath" 16 "regexp" 17 "strings" 18 ) 19 20 // Callback a callback function for processing version information. Return true to continue processing 21 // or false to terminate the loop 22 type Callback func(kind VersionKind, name string, version *StableVersion) (bool, error) 23 24 // VersionKind represents the kind of version 25 type VersionKind string 26 27 const ( 28 // KindChart represents a chart version 29 KindChart VersionKind = "charts" 30 31 // KindPackage represents a package version 32 KindPackage VersionKind = "packages" 33 34 // KindDocker represents a docker resolveImage version 35 KindDocker VersionKind = "docker" 36 37 // KindGit represents a git repository (e.g. for jx boot configuration or a build pack) 38 KindGit VersionKind = "git" 39 ) 40 41 var ( 42 // Kinds all the version kinds 43 Kinds = []VersionKind{ 44 KindChart, 45 KindPackage, 46 KindDocker, 47 KindGit, 48 } 49 50 // KindStrings all the kinds as strings for validating CLI arguments 51 KindStrings = []string{ 52 string(KindChart), 53 string(KindPackage), 54 string(KindDocker), 55 string(KindGit), 56 } 57 ) 58 59 // StableVersion stores the stable version information 60 type StableVersion struct { 61 // Version the default version to use 62 Version string `json:"version,omitempty"` 63 // VersionUpperLimit represents the upper limit which indicates a version which is too new. 64 65 // e.g. for packages we could use: `{ version: "1.10.1", upperLimit: "1.14.0"}` which would mean these 66 // versions are all valid `["1.11.5", "1.13.1234"]` but these are invalid `["1.14.0", "1.14.1"]` 67 UpperLimit string `json:"upperLimit,omitempty"` 68 // GitURL the URL to the source code 69 GitURL string `json:"gitUrl,omitempty"` 70 // Component is the component inside the git URL 71 Component string `json:"component,omitempty"` 72 // URL the URL for the documentation 73 URL string `json:"url,omitempty"` 74 } 75 76 // VerifyPackage verifies the current version of the package is valid 77 func (data *StableVersion) VerifyPackage(name string, currentVersion string, workDir string) error { 78 currentVersion = convertToVersion(currentVersion) 79 if currentVersion == "" { 80 return nil 81 } 82 version := convertToVersion(data.Version) 83 if version == "" { 84 log.Logger().Warnf("could not find a stable package version for %s from %s\nFor background see: https://jenkins-x.io/about/concepts/version-stream/", name, workDir) 85 log.Logger().Infof("Please lock this version down via the command: %s", util.ColorInfo(fmt.Sprintf("jx step create pr versions -k package -n %s", name))) 86 return nil 87 } 88 89 currentSem, err := semver.Make(currentVersion) 90 if err != nil { 91 return errors.Wrapf(err, "failed to parse semantic version for current version %s for package %s", currentVersion, name) 92 } 93 94 minSem, err := semver.Make(version) 95 if err != nil { 96 return errors.Wrapf(err, "failed to parse required semantic version %s for package %s", version, name) 97 } 98 99 upperLimitText := convertToVersion(data.UpperLimit) 100 if upperLimitText == "" { 101 if minSem.Equals(currentSem) { 102 return nil 103 } 104 return verifyError(name, fmt.Errorf("package %s is on version %s but the version stream requires version %s", name, currentVersion, version)) 105 } 106 107 // lets make sure the current version is in the range 108 if currentSem.LT(minSem) { 109 return verifyError(name, fmt.Errorf("package %s is an old version %s. The version stream requires at least %s", name, currentVersion, version)) 110 } 111 112 limitSem, err := semver.Make(upperLimitText) 113 if err != nil { 114 return errors.Wrapf(err, "failed to parse upper limit version %s for package %s", upperLimitText, name) 115 } 116 117 if currentSem.GE(limitSem) { 118 return verifyError(name, fmt.Errorf("package %s is using version %s which is too new. The version stream requires a version earlier than %s", name, currentVersion, upperLimitText)) 119 } 120 return nil 121 } 122 123 // verifyError allows package verify errors to be disabled in development via environment variables 124 func verifyError(name string, err error) error { 125 envVar := "JX_DISABLE_VERIFY_" + strings.ToUpper(name) 126 value := os.Getenv(envVar) 127 if strings.ToLower(value) == "true" { 128 log.Logger().Warnf("$%s is true so disabling verify of %s: %s\n", envVar, name, err.Error()) 129 return nil 130 } 131 return err 132 } 133 134 // convertToVersion extracts a semantic version from the specified string. 135 // If no semantic version is contained in the specified string the string is returned unmodified. 136 func convertToVersion(text string) string { 137 // Some apps might not exactly follow semver, like for example Git for Windows: 2.23.0.windows.1 138 // we're trimming everything after a semver from the answer 139 // to avoid error described in issue #6825 140 r := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`) 141 if !r.Match([]byte(text)) { 142 return text 143 } 144 return r.FindString(text) 145 } 146 147 // LoadStableVersion loads the stable version data from the version configuration directory returning an empty object if there is 148 // no specific stable version configuration available 149 func LoadStableVersion(wrkDir string, kind VersionKind, name string) (*StableVersion, error) { 150 if kind == KindGit { 151 name = GitURLToName(name) 152 } 153 path := filepath.Join(wrkDir, string(kind), name+".yml") 154 return LoadStableVersionFile(path) 155 } 156 157 // GitURLToName lets trim any URL scheme and trailing .git or / from a git URL 158 func GitURLToName(name string) string { 159 // lets trim the URL scheme 160 idx := strings.Index(name, "://") 161 if idx > 0 { 162 name = name[idx+3:] 163 } 164 name = strings.TrimSuffix(name, ".git") 165 name = strings.TrimSuffix(name, "/") 166 return name 167 } 168 169 // LoadStableVersionFile loads the stable version data from the given file name 170 func LoadStableVersionFile(path string) (*StableVersion, error) { 171 version := &StableVersion{} 172 exists, err := util.FileExists(path) 173 if err != nil { 174 return version, errors.Wrapf(err, "failed to check if file exists %s", path) 175 } 176 if !exists { 177 return version, nil 178 } 179 data, err := ioutil.ReadFile(path) 180 if err != nil { 181 return version, errors.Wrapf(err, "failed to load YAML file %s", path) 182 } 183 version, err = LoadStableVersionFromData(data) 184 if err != nil { 185 return version, errors.Wrapf(err, "failed to unmarshal YAML for file %s", path) 186 } 187 return version, err 188 } 189 190 // LoadStableVersionFromData loads the stable version data from the given the data 191 func LoadStableVersionFromData(data []byte) (*StableVersion, error) { 192 version := &StableVersion{} 193 err := yaml.Unmarshal(data, version) 194 if err != nil { 195 return version, errors.Wrapf(err, "failed to unmarshal YAML") 196 } 197 return version, err 198 } 199 200 // LoadStableVersionNumber loads just the stable version number for the given kind and name 201 func LoadStableVersionNumber(wrkDir string, kind VersionKind, name string) (string, error) { 202 data, err := LoadStableVersion(wrkDir, kind, name) 203 if err != nil { 204 return "", err 205 } 206 version := data.Version 207 if version != "" { 208 log.Logger().Debugf("using stable version %s from %s of %s from %s", util.ColorInfo(version), string(kind), util.ColorInfo(name), wrkDir) 209 } else { 210 // lets not warn if building current dir chart 211 if kind == KindChart && name == "." { 212 return version, err 213 } 214 log.Logger().Warnf("could not find a stable version from %s of %s from %s\nFor background see: https://jenkins-x.io/about/concepts/version-stream/", string(kind), name, wrkDir) 215 log.Logger().Infof("Please lock this version down via the command: %s", util.ColorInfo(fmt.Sprintf("jx step create pr versions -k %s -n %s", string(kind), name))) 216 } 217 return version, err 218 } 219 220 // SaveStableVersion saves the version file 221 func SaveStableVersion(wrkDir string, kind VersionKind, name string, stableVersion *StableVersion) error { 222 path := filepath.Join(wrkDir, string(kind), name+".yml") 223 return SaveStableVersionFile(path, stableVersion) 224 } 225 226 // SaveStableVersionFile saves the stabe version to the given file name 227 func SaveStableVersionFile(path string, stableVersion *StableVersion) error { 228 data, err := yaml.Marshal(stableVersion) 229 if err != nil { 230 return errors.Wrapf(err, "failed to marshal data to YAML %#v", stableVersion) 231 } 232 dir, _ := filepath.Split(path) 233 err = os.MkdirAll(dir, util.DefaultWritePermissions) 234 if err != nil { 235 return errors.Wrapf(err, "failed to create directory %s", dir) 236 } 237 238 err = ioutil.WriteFile(path, data, util.DefaultWritePermissions) 239 if err != nil { 240 return errors.Wrapf(err, "failed to write file %s", path) 241 } 242 return nil 243 } 244 245 // ResolveDockerImage resolves the version of the specified image against the version stream defined in versionsDir. 246 // If there is a version defined for the image in the version stream 'image:<version>' is returned, otherwise the 247 // passed image name is returned as is. 248 func ResolveDockerImage(versionsDir, image string) (string, error) { 249 // lets check if we already have a version 250 path := strings.SplitN(image, ":", 2) 251 if len(path) == 2 && path[1] != "" { 252 return image, nil 253 } 254 info, err := LoadStableVersion(versionsDir, KindDocker, image) 255 if err != nil { 256 return image, err 257 } 258 if info.Version == "" { 259 // lets check if there is a docker.io prefix and if so lets try fetch without the docker prefix 260 prefix := "docker.io/" 261 if strings.HasPrefix(image, prefix) { 262 image = strings.TrimPrefix(image, prefix) 263 info, err = LoadStableVersion(versionsDir, KindDocker, image) 264 if err != nil { 265 return image, err 266 } 267 } 268 } 269 if info.Version == "" { 270 log.Logger().Warnf("could not find a stable version for Docker image: %s in %s", image, versionsDir) 271 log.Logger().Warn("for background see: https://jenkins-x.io/about/concepts/version-stream/") 272 log.Logger().Infof("please lock this version down via the command: %s", util.ColorInfo(fmt.Sprintf("jx step create pr versions -k docker -n %s -v 1.2.3", image))) 273 return image, nil 274 } 275 prefix := strings.TrimSuffix(strings.TrimSpace(image), ":") 276 return prefix + ":" + info.Version, nil 277 } 278 279 // UpdateStableVersionFiles applies an update to the stable version files matched by globPattern, updating to version 280 func UpdateStableVersionFiles(globPattern string, version string, excludeFiles ...string) ([]string, error) { 281 files, err := filepath.Glob(globPattern) 282 if err != nil { 283 return nil, errors.Wrapf(err, "failed to create glob from pattern %s", globPattern) 284 } 285 answer := make([]string, 0) 286 287 for _, path := range files { 288 _, name := filepath.Split(path) 289 if util.StringArrayIndex(excludeFiles, name) >= 0 { 290 continue 291 } 292 data, err := LoadStableVersionFile(path) 293 if err != nil { 294 return nil, errors.Wrapf(err, "failed to load oldVersion info for %s", path) 295 } 296 if data.Version == "" || data.Version == version { 297 continue 298 } 299 answer = append(answer, data.Version) 300 data.Version = version 301 err = SaveStableVersionFile(path, data) 302 if err != nil { 303 return nil, errors.Wrapf(err, "failed to save oldVersion info for %s", path) 304 } 305 } 306 return answer, nil 307 } 308 309 // UpdateStableVersion applies an update to the stable version file in dir/kindStr/name.yml, updating to version 310 func UpdateStableVersion(dir string, kindStr string, name string, version string) ([]string, error) { 311 answer := make([]string, 0) 312 kind := VersionKind(kindStr) 313 data, err := LoadStableVersion(dir, kind, name) 314 if err != nil { 315 return nil, err 316 } 317 if data.Version == version { 318 return nil, nil 319 } 320 answer = append(answer, data.Version) 321 data.Version = version 322 323 err = SaveStableVersion(dir, kind, name, data) 324 if err != nil { 325 return nil, errors.Wrapf(err, "failed to save versionstream file") 326 } 327 return answer, nil 328 } 329 330 // GetRepositoryPrefixes loads the repository prefixes for the version stream 331 func GetRepositoryPrefixes(dir string) (*RepositoryPrefixes, error) { 332 answer := &RepositoryPrefixes{} 333 fileName := filepath.Join(dir, "charts", "repositories.yml") 334 exists, err := util.FileExists(fileName) 335 if err != nil { 336 return answer, errors.Wrapf(err, "failed to find file %s", fileName) 337 } 338 if !exists { 339 return answer, nil 340 } 341 data, err := ioutil.ReadFile(fileName) 342 if err != nil { 343 return answer, errors.Wrapf(err, "failed to load file %s", fileName) 344 } 345 err = yaml.Unmarshal(data, answer) 346 if err != nil { 347 return answer, errors.Wrapf(err, "failed to unmarshal YAML in file %s", fileName) 348 } 349 return answer, nil 350 } 351 352 // GetQuickStarts loads the quickstarts from the version stream 353 func GetQuickStarts(dir string) (*QuickStarts, error) { 354 answer := &QuickStarts{} 355 fileName := filepath.Join(dir, "quickstarts.yml") 356 exists, err := util.FileExists(fileName) 357 if err != nil { 358 return answer, errors.Wrapf(err, "failed to find file %s", fileName) 359 } 360 if !exists { 361 return answer, nil 362 } 363 data, err := ioutil.ReadFile(fileName) 364 if err != nil { 365 return answer, errors.Wrapf(err, "failed to load file %s", fileName) 366 } 367 err = yaml.Unmarshal(data, &answer) 368 if err != nil { 369 return answer, errors.Wrapf(err, "failed to unmarshal YAML in file %s", fileName) 370 } 371 return answer, nil 372 } 373 374 // SaveQuickStarts saves the modified quickstarts in the version stream dir 375 func SaveQuickStarts(dir string, qs *QuickStarts) error { 376 data, err := yaml.Marshal(qs) 377 if err != nil { 378 return errors.Wrapf(err, "failed to marshal quickstarts to YAML") 379 } 380 fileName := filepath.Join(dir, "quickstarts.yml") 381 err = ioutil.WriteFile(fileName, data, util.DefaultWritePermissions) 382 if err != nil { 383 return errors.Wrapf(err, "failed to save file %s", fileName) 384 } 385 return nil 386 } 387 388 // RepositoryPrefixes maps repository prefixes to URLs 389 type RepositoryPrefixes struct { 390 Repositories []RepositoryURLs `json:"repositories"` 391 urlToPrefix map[string]string `json:"-"` 392 prefixToURLs map[string][]string `json:"-"` 393 } 394 395 // RepositoryURLs contains the prefix and URLS for a repository 396 type RepositoryURLs struct { 397 Prefix string `json:"prefix"` 398 URLs []string `json:"urls"` 399 } 400 401 // QuickStart the configuration of a quickstart in the version stream 402 type QuickStart struct { 403 ID string `json:"id,omitempty"` 404 Owner string `json:"owner,omitempty"` 405 Name string `json:"name,omitempty"` 406 Version string `json:"version,omitempty"` 407 Language string `json:"language,omitempty"` 408 Framework string `json:"framework,omitempty"` 409 Tags []string `json:"tags,omitempty"` 410 DownloadZipURL string `json:"downloadZipURL,omitempty"` 411 } 412 413 // QuickStarts the configuration of a the quickstarts in the version stream 414 type QuickStarts struct { 415 QuickStarts []*QuickStart `json:"quickstarts"` 416 DefaultOwner string `json:"defaultOwner"` 417 } 418 419 // DefaultMissingValues defaults any missing values such as ID which is a combination of owner and name 420 func (qs *QuickStarts) DefaultMissingValues() { 421 for _, q := range qs.QuickStarts { 422 q.defaultMissingValues(qs) 423 } 424 } 425 426 // Sort sorts the quickstarts into name order 427 func (qs *QuickStarts) Sort() { 428 sort.Sort(quickStartOrder(qs.QuickStarts)) 429 } 430 431 type quickStartOrder []*QuickStart 432 433 // Len returns the length of the order 434 func (a quickStartOrder) Len() int { return len(a) } 435 436 // Swap swaps 2 items in the slice 437 func (a quickStartOrder) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 438 439 // Less returns trtue if an itetm is less than the order 440 func (a quickStartOrder) Less(i, j int) bool { 441 r1 := a[i] 442 r2 := a[j] 443 444 n1 := r1.Name 445 n2 := r2.Name 446 if n1 != n2 { 447 return n1 < n2 448 } 449 o1 := r1.Owner 450 o2 := r2.Owner 451 return o1 < o2 452 } 453 454 func (q *QuickStart) defaultMissingValues(qs *QuickStarts) { 455 if qs.DefaultOwner == "" { 456 qs.DefaultOwner = "jenkins-x-quickstarts" 457 } 458 if q.Owner == "" { 459 q.Owner = qs.DefaultOwner 460 } 461 if q.ID == "" { 462 q.ID = fmt.Sprintf("%s/%s", q.Owner, q.Name) 463 } 464 if q.DownloadZipURL == "" { 465 q.DownloadZipURL = fmt.Sprintf("https://codeload.github.com/%s/%s/zip/master", q.Owner, q.Name) 466 } 467 } 468 469 // PrefixForURL returns the repository prefix for the given URL 470 func (p *RepositoryPrefixes) PrefixForURL(u string) string { 471 if p.urlToPrefix == nil { 472 p.urlToPrefix = map[string]string{} 473 474 for _, repo := range p.Repositories { 475 for _, url := range repo.URLs { 476 p.urlToPrefix[url] = repo.Prefix 477 } 478 } 479 } 480 return p.urlToPrefix[u] 481 } 482 483 // URLsForPrefix returns the repository URLs for the given prefix 484 func (p *RepositoryPrefixes) URLsForPrefix(prefix string) []string { 485 if p.prefixToURLs == nil { 486 p.prefixToURLs = make(map[string][]string) 487 for _, repo := range p.Repositories { 488 p.prefixToURLs[repo.Prefix] = repo.URLs 489 } 490 } 491 return p.prefixToURLs[prefix] 492 } 493 494 // NameFromPath converts a path into a name for use with stable versions 495 func NameFromPath(basepath string, path string) (string, error) { 496 name, err := filepath.Rel(basepath, path) 497 if err != nil { 498 return "", errors.Wrapf(err, "failed to extract base path from %s", path) 499 } 500 ext := filepath.Ext(name) 501 if ext != "" { 502 name = strings.TrimSuffix(name, ext) 503 } 504 return name, nil 505 }