bosun.org@v0.0.0-20210513094433-e25bc3e69a1f/cmd/scollector/collectors/google_webmaster.go (about)

     1  package collectors
     2  
     3  import (
     4  	"net/url"
     5  	"strings"
     6  	"time"
     7  
     8  	"bosun.org/cmd/scollector/conf"
     9  	"bosun.org/metadata"
    10  	"bosun.org/opentsdb"
    11  	"bosun.org/slog"
    12  
    13  	"google.golang.org/api/webmasters/v3"
    14  )
    15  
    16  func init() {
    17  	registerInit(func(c *conf.Conf) {
    18  		for _, g := range c.GoogleWebmaster {
    19  			collectors = append(collectors, &IntervalCollector{
    20  				F: func() (opentsdb.MultiDataPoint, error) {
    21  					return c_google_webmaster(g.ClientID, g.Secret, g.Token, g.JSONToken)
    22  				},
    23  				name:     "c_google_webmaster",
    24  				Interval: time.Hour * 1,
    25  			})
    26  		}
    27  	})
    28  }
    29  
    30  func c_google_webmaster(clientID, secret, tokenStr, jsonToken string) (opentsdb.MultiDataPoint, error) {
    31  	c, err := googleAPIClient(clientID, secret, tokenStr, jsonToken, []string{webmasters.WebmastersReadonlyScope})
    32  	if err != nil {
    33  		return nil, err
    34  	}
    35  
    36  	svc, err := webmasters.New(c)
    37  	if err != nil {
    38  		return nil, err
    39  	}
    40  
    41  	md, err := getWebmasterErrorsMetrics(svc)
    42  	if err != nil {
    43  		return nil, err
    44  	}
    45  
    46  	return *md, nil
    47  }
    48  
    49  // getWebmasterErrorsMetric utilizes the webmasters API to list all sites
    50  // associated with an authenticated account, fetch the time-series data error
    51  // metrics for each of those sites, and builds opentsdb datapoints from that
    52  // data.
    53  func getWebmasterErrorsMetrics(svc *webmasters.Service) (*opentsdb.MultiDataPoint, error) {
    54  	md := &opentsdb.MultiDataPoint{}
    55  
    56  	throttle := time.Tick(time.Second / 2)
    57  
    58  	<-throttle
    59  	sites, err := svc.Sites.List().Do()
    60  	if err != nil {
    61  		return nil, err
    62  	}
    63  
    64  	for _, site := range sites.SiteEntry {
    65  		u, err := url.Parse(site.SiteUrl)
    66  		if err != nil {
    67  			return nil, err
    68  		}
    69  		// Webmasters has these new sets with fake URLs like "sc-set:jJfZIHyI4-DY8wg0Ww4l-A".
    70  		// Most API calls we use fail for these sites, so we skip em.
    71  		// TODO: Allow these sites once the API supports em.
    72  		if strings.HasPrefix(site.SiteUrl, "sc-set") {
    73  			continue
    74  		}
    75  		if site.PermissionLevel == "siteUnverifiedUser" {
    76  			slog.Errorf("Lack permission to fetch error metrics for site %s. Skipping.\n", u.Host)
    77  			continue
    78  		}
    79  		tags := opentsdb.TagSet{}
    80  		tags["site"] = u.Host
    81  		tags["scheme"] = u.Scheme
    82  		tags["path"] = u.Path
    83  		<-throttle
    84  		crawlErrors, err := svc.Urlcrawlerrorscounts.Query(site.SiteUrl).LatestCountsOnly(true).Do()
    85  		if err != nil {
    86  			slog.Errorf("Error fetching error counts for site %s: %s", u.Host, err)
    87  			continue
    88  		}
    89  		for _, e := range crawlErrors.CountPerTypes {
    90  			tags["platform"] = e.Platform
    91  			tags["category"] = e.Category
    92  			for _, entry := range e.Entries {
    93  				t, err := time.Parse(time.RFC3339, entry.Timestamp)
    94  				if err != nil {
    95  					return md, err
    96  				}
    97  				AddTS(md, "google.webmaster.errors", t.Unix(), entry.Count, tags, metadata.Gauge, metadata.Error, descGoogleWebmasterErrors)
    98  			}
    99  		}
   100  	}
   101  
   102  	return md, nil
   103  }
   104  
   105  const descGoogleWebmasterErrors = "The number of crawl errors that Google experienced on a given site. Note that if Google webmaster is tracking multiple paths for a given site, then error counts in parent paths (/) will include errors from any child paths (/foo). As such, aggregation across parent and child paths will result in erroneous results."