github.com/abayer/test-infra@v0.0.5/velodrome/transform/fetcher.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"fmt"
    21  	"time"
    22  
    23  	"k8s.io/test-infra/velodrome/sql"
    24  
    25  	"github.com/golang/glog"
    26  	"github.com/jinzhu/gorm"
    27  )
    28  
    29  // Fetcher is a utility class used to Fetch all types of events
    30  type Fetcher struct {
    31  	IssuesChannel         chan sql.Issue
    32  	EventsCommentsChannel chan interface{}
    33  
    34  	lastIssue   time.Time
    35  	lastEvent   time.Time
    36  	lastComment time.Time
    37  	repository  string
    38  }
    39  
    40  // NewFetcher creates a new Fetcher and initializes the output channels
    41  func NewFetcher(repository string) *Fetcher {
    42  	return &Fetcher{
    43  		IssuesChannel:         make(chan sql.Issue, 100),
    44  		EventsCommentsChannel: make(chan interface{}, 100),
    45  		repository:            repository,
    46  	}
    47  }
    48  
    49  // fetchRecentIssues retrieves issues from DB, but only fetches issues modified since last call
    50  func (f *Fetcher) fetchRecentIssues(db *gorm.DB) error {
    51  	glog.Infof("Fetching issues updated after %s", f.lastIssue)
    52  
    53  	var issues []sql.Issue
    54  	query := db.
    55  		Where("issue_updated_at >= ?", f.lastIssue).
    56  		Where("repository = ?", f.repository).
    57  		Order("issue_updated_at").
    58  		Preload("Labels").
    59  		Find(&issues)
    60  	if query.Error != nil {
    61  		return query.Error
    62  	}
    63  
    64  	count := len(issues)
    65  	for _, issue := range issues {
    66  		f.IssuesChannel <- issue
    67  		f.lastIssue = issue.IssueUpdatedAt
    68  	}
    69  	glog.Infof("Found and pushed %d updated/new issues", count)
    70  
    71  	return nil
    72  }
    73  
    74  // fetchRecentEventsAndComments retrieves events from DB, but only fetches events created since last call
    75  func (f *Fetcher) fetchRecentEventsAndComments(db *gorm.DB) error {
    76  	glog.Infof("Fetching issue-events with id bigger than %s", f.lastEvent)
    77  	glog.Infof("Fetching comments with id bigger than %s", f.lastComment)
    78  
    79  	eventRows, err := db.
    80  		Model(sql.IssueEvent{}).
    81  		Where("repository = ?", f.repository).
    82  		Where("event_created_at > ?", f.lastEvent).
    83  		Order("event_created_at asc").
    84  		Rows()
    85  	if err != nil {
    86  		return fmt.Errorf("Failed to query events from database: %s", err)
    87  	}
    88  
    89  	commentRows, err := db.
    90  		Model(sql.Comment{}).
    91  		Where("repository = ?", f.repository).
    92  		Where("comment_created_at > ?", f.lastComment).
    93  		Order("comment_created_at asc").
    94  		Rows()
    95  	if err != nil {
    96  		return fmt.Errorf("Failed to query comments from database: %s", err)
    97  	}
    98  
    99  	count := 0
   100  	comment := &sql.Comment{}
   101  	if commentRows.Next() {
   102  		db.ScanRows(commentRows, comment)
   103  	} else {
   104  		comment = nil
   105  	}
   106  	event := &sql.IssueEvent{}
   107  	if eventRows.Next() {
   108  		db.ScanRows(eventRows, event)
   109  	} else {
   110  		event = nil
   111  	}
   112  
   113  	for event != nil || comment != nil {
   114  		if event == nil || (comment != nil && comment.CommentCreatedAt.Before(event.EventCreatedAt)) {
   115  			f.EventsCommentsChannel <- *comment
   116  			f.lastComment = comment.CommentCreatedAt
   117  			if commentRows.Next() {
   118  				db.ScanRows(commentRows, comment)
   119  			} else {
   120  				comment = nil
   121  			}
   122  		} else {
   123  			f.EventsCommentsChannel <- *event
   124  			f.lastEvent = event.EventCreatedAt
   125  			if eventRows.Next() {
   126  				db.ScanRows(eventRows, event)
   127  			} else {
   128  				event = nil
   129  			}
   130  		}
   131  		count++
   132  	}
   133  
   134  	glog.Infof("Found and pushed %d new events/comments", count)
   135  
   136  	return nil
   137  }
   138  
   139  // Fetch retrieves all types of events, and push them to output channels
   140  func (f *Fetcher) Fetch(db *gorm.DB) error {
   141  	if err := f.fetchRecentIssues(db); err != nil {
   142  		return err
   143  	}
   144  	if err := f.fetchRecentEventsAndComments(db); err != nil {
   145  		return err
   146  	}
   147  	return nil
   148  }