github.com/redhat-appstudio/e2e-tests@v0.0.0-20230619105049-9a422b2094d7/magefiles/utils.go (about) 1 package main 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io" 8 "net/http" 9 "os" 10 "regexp" 11 "strings" 12 "sync" 13 "text/template" 14 "time" 15 16 "k8s.io/klog/v2" 17 18 sprig "github.com/go-task/slim-sprig" 19 "github.com/magefile/mage/sh" 20 "github.com/redhat-appstudio/image-controller/pkg/quay" 21 ) 22 23 const quayPrefixesToDeleteRegexp = "e2e-demos|has-e2e|multi-comp|build-e2e" 24 25 func getRemoteAndBranchNameFromPRLink(url string) (remote, branchName string, err error) { 26 ghRes := &GithubPRInfo{} 27 if err := sendHttpRequestAndParseResponse(url, "GET", ghRes); err != nil { 28 return "", "", err 29 } 30 31 if ghRes.Head.Label == "" { 32 return "", "", fmt.Errorf("failed to get an information about the remote and branch name from PR %s", url) 33 } 34 35 split := strings.Split(ghRes.Head.Label, ":") 36 remote, branchName = split[0], split[1] 37 38 return remote, branchName, nil 39 } 40 41 func gitCheckoutRemoteBranch(remoteName, branchName string) error { 42 var git = sh.RunCmd("git") 43 for _, arg := range [][]string{ 44 {"remote", "add", remoteName, fmt.Sprintf("https://github.com/%s/e2e-tests.git", remoteName)}, 45 {"fetch", remoteName}, 46 {"checkout", branchName}, 47 } { 48 if err := git(arg...); err != nil { 49 return fmt.Errorf("error when checkout out remote branch %s from remote %s: %v", branchName, remoteName, err) 50 } 51 } 52 return nil 53 } 54 55 func sendHttpRequestAndParseResponse(url, method string, v interface{}) error { 56 req, err := http.NewRequestWithContext(context.Background(), method, url, nil) 57 if err != nil { 58 return err 59 } 60 req.Header.Set("Authorization", fmt.Sprintf("token %s", os.Getenv("GITHUB_TOKEN"))) 61 res, err := http.DefaultClient.Do(req) 62 if err != nil { 63 return fmt.Errorf("error when sending request to '%s': %+v", url, err) 64 } 65 defer res.Body.Close() 66 body, err := io.ReadAll(res.Body) 67 if err != nil { 68 return fmt.Errorf("error when reading the response body from URL '%s': %+v", url, err) 69 } 70 if res.StatusCode > 299 { 71 return fmt.Errorf("unexpected status code: %d, response body: %s", res.StatusCode, string(body)) 72 } 73 74 if err := json.Unmarshal(body, v); err != nil { 75 return fmt.Errorf("error when unmarshalling the response body from URL '%s': %+v", url, err) 76 } 77 78 return nil 79 } 80 81 func retry(f func() error, attempts int, delay time.Duration) error { 82 var err error 83 for i := 0; i < attempts; i++ { 84 if i > 0 { 85 klog.Infof("got an error: %+v - will retry in %v", err, delay) 86 time.Sleep(delay) 87 } 88 err = f() 89 if err != nil { 90 continue 91 } else { 92 return nil 93 } 94 } 95 return fmt.Errorf("reached maximum number of attempts (%d). error: %+v", attempts, err) 96 } 97 98 func goFmt(path string) error { 99 err := sh.RunV("go", "fmt", path) 100 if err != nil { 101 return fmt.Errorf(fmt.Sprintf("Could not fmt:\n%s\n", path), err) 102 } 103 return nil 104 } 105 106 func fileExists(path string) bool { 107 _, err := os.Stat(path) 108 return err == nil 109 } 110 111 func renderTemplate(destination, templatePath string, templateData interface{}, appendDestination bool) error { 112 113 var templateText string 114 var f *os.File 115 var err error 116 117 /* This decision logic feels a little clunky cause initially I wanted to 118 to have this func create the new file and render the template into the new 119 file. But with the updating the pkg/framework/describe.go use case 120 I wanted to reuse leveraging the txt/template package rather than 121 rendering/updating using strings/regex. 122 */ 123 if appendDestination { 124 125 f, err = os.OpenFile(destination, os.O_APPEND|os.O_WRONLY, 0664) 126 if err != nil { 127 klog.Infof("Failed to open file: %v", err) 128 } 129 } else { 130 131 if fileExists(destination) { 132 return fmt.Errorf("%s already exists", destination) 133 } 134 f, err = os.Create(destination) 135 if err != nil { 136 klog.Infof("Failed to create file: %v", err) 137 } 138 } 139 140 defer f.Close() 141 142 tpl, err := os.ReadFile(templatePath) 143 if err != nil { 144 klog.Infof("error reading file: %v", err) 145 146 } 147 var tmplText = string(tpl) 148 templateText = fmt.Sprintf("\n%s", tmplText) 149 specTemplate, err := template.New("spec").Funcs(sprig.TxtFuncMap()).Parse(templateText) 150 if err != nil { 151 klog.Infof("error parsing template file: %v", err) 152 153 } 154 155 err = specTemplate.Execute(f, templateData) 156 if err != nil { 157 klog.Infof("error rendering template file: %v", err) 158 } 159 160 return nil 161 } 162 163 func cleanupQuayReposAndRobots(quayService quay.QuayService, quayOrg string) error { 164 r, err := regexp.Compile(fmt.Sprintf(`^(%s)`, quayPrefixesToDeleteRegexp)) 165 if err != nil { 166 return err 167 } 168 169 repos, err := quayService.GetAllRepositories(quayOrg) 170 if err != nil { 171 return err 172 } 173 174 // Key is the repo name without slashes which is the same as robot name 175 // Value is the repo name with slashes 176 reposMap := make(map[string]string) 177 178 for _, repo := range repos { 179 if r.MatchString(repo.Name) { 180 sanitizedRepoName := strings.ReplaceAll(repo.Name, "/", "") // repo name without slashes 181 reposMap[sanitizedRepoName] = repo.Name 182 } 183 } 184 185 robots, err := quayService.GetAllRobotAccounts(quayOrg) 186 if err != nil { 187 return err 188 } 189 190 r, err = regexp.Compile(fmt.Sprintf(`^%s\+(%s)`, quayOrg, quayPrefixesToDeleteRegexp)) 191 if err != nil { 192 return err 193 } 194 195 const timeFormat = "Mon, 02 Jan 2006 15:04:05 -0700" 196 197 // Deletes robots and their repos with correct prefix if created more than 24 hours ago 198 for _, robot := range robots { 199 parsed, err := time.Parse(timeFormat, robot.Created) 200 if err != nil { 201 return err 202 } 203 204 // If robot.Name has correct prefix and was created more than 24 hours ago 205 if r.MatchString(robot.Name) && time.Since(parsed) > 24*time.Hour { 206 // Robot name without the name of org which is the same as previous sanitizedRepoName 207 // redhat-appstudio-qe+e2e-demos turns to e2e-demos 208 splitRobotName := strings.Split(robot.Name, "+") 209 if len(splitRobotName) != 2 { 210 return fmt.Errorf("failed to split robot name into 2 parts, got %d parts", len(splitRobotName)) 211 } 212 sanitizedRepoName := splitRobotName[1] // Same as robot shortname 213 if repo, exists := reposMap[sanitizedRepoName]; exists { 214 deleted, err := quayService.DeleteRepository(quayOrg, repo) 215 if err != nil { 216 return fmt.Errorf("failed to delete repository %s, error: %s", repo, err) 217 } 218 if !deleted { 219 fmt.Printf("repository %s has already been deleted, skipping\n", repo) 220 } 221 } 222 // DeleteRobotAccount uses robot shortname, so e2e-demos instead of redhat-appstudio-qe+e2e-demos 223 deleted, err := quayService.DeleteRobotAccount(quayOrg, splitRobotName[1]) 224 if err != nil { 225 return fmt.Errorf("failed to delete robot account %s, error: %s", robot.Name, err) 226 } 227 if !deleted { 228 fmt.Printf("robot account %s has already been deleted, skipping\n", robot.Name) 229 } 230 } 231 } 232 return nil 233 } 234 235 func cleanupQuayTags(quayService quay.QuayService, organization, repository string) error { 236 workerCount := 10 237 var wg sync.WaitGroup 238 239 var allTags []quay.Tag 240 var errors []error 241 242 page := 1 243 for { 244 tags, hasAdditional, err := quayService.GetTagsFromPage(organization, repository, page) 245 page++ 246 if err != nil { 247 errors = append(errors, fmt.Errorf("error getting tags of `%s` repository of `%s` organization on page `%d`, error: %s", repository, organization, page, err)) 248 continue 249 } 250 allTags = append(allTags, tags...) 251 if !hasAdditional { 252 break 253 } 254 } 255 256 wg.Add(workerCount) 257 258 var errorsMutex sync.Mutex 259 for i := 0; i < workerCount; i++ { 260 go func(startIdx int, allTags []quay.Tag, errors []error, errorsMutex *sync.Mutex, wg *sync.WaitGroup) { 261 defer wg.Done() 262 for idx := startIdx; idx < len(allTags); idx += workerCount { 263 tag := allTags[idx] 264 if time.Unix(tag.StartTS, 0).Before(time.Now().AddDate(0, 0, -7)) { 265 deleted, err := quayService.DeleteTag(organization, repository, tag.Name) 266 if err != nil { 267 errorsMutex.Lock() 268 errors = append(errors, fmt.Errorf("error during deletion of tag `%s` in repository `%s` of organization `%s`, error: `%s`\n", tag.Name, repository, organization, err)) 269 errorsMutex.Unlock() 270 } else if !deleted { 271 fmt.Printf("tag `%s` in repository `%s` of organization `%s` was not deleted\n", tag.Name, repository, organization) 272 } 273 } 274 } 275 }(i, allTags, errors, &errorsMutex, &wg) 276 } 277 278 wg.Wait() 279 280 if len(errors) == 0 { 281 return nil 282 } 283 284 var errBuilder strings.Builder 285 for _, err := range errors { 286 errBuilder.WriteString(fmt.Sprintf("%s\n", err)) 287 } 288 return fmt.Errorf("encountered errors during CleanupQuayTags: %s", errBuilder.String()) 289 }