github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/integration/e2ecortex/client.go (about)

     1  package e2ecortex
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/json"
     7  	"errors"
     8  	"fmt"
     9  	"io"
    10  	"io/ioutil"
    11  	"net/http"
    12  	"net/url"
    13  	"strconv"
    14  	"time"
    15  
    16  	"github.com/gogo/protobuf/proto"
    17  	"github.com/golang/snappy"
    18  	alertConfig "github.com/prometheus/alertmanager/config"
    19  	"github.com/prometheus/alertmanager/types"
    20  	promapi "github.com/prometheus/client_golang/api"
    21  	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
    22  	"github.com/prometheus/common/model"
    23  	"github.com/prometheus/prometheus/pkg/rulefmt"
    24  	"github.com/prometheus/prometheus/prompb"
    25  	yaml "gopkg.in/yaml.v3"
    26  
    27  	"github.com/cortexproject/cortex/pkg/ruler"
    28  )
    29  
    30  var ErrNotFound = errors.New("not found")
    31  
    32  // Client is a client used to interact with Cortex in integration tests
    33  type Client struct {
    34  	alertmanagerClient  promapi.Client
    35  	querierAddress      string
    36  	alertmanagerAddress string
    37  	rulerAddress        string
    38  	distributorAddress  string
    39  	timeout             time.Duration
    40  	httpClient          *http.Client
    41  	querierClient       promv1.API
    42  	orgID               string
    43  }
    44  
    45  // NewClient makes a new Cortex client
    46  func NewClient(
    47  	distributorAddress string,
    48  	querierAddress string,
    49  	alertmanagerAddress string,
    50  	rulerAddress string,
    51  	orgID string,
    52  ) (*Client, error) {
    53  	// Create querier API client
    54  	querierAPIClient, err := promapi.NewClient(promapi.Config{
    55  		Address:      "http://" + querierAddress + "/api/prom",
    56  		RoundTripper: &addOrgIDRoundTripper{orgID: orgID, next: http.DefaultTransport},
    57  	})
    58  	if err != nil {
    59  		return nil, err
    60  	}
    61  
    62  	c := &Client{
    63  		distributorAddress:  distributorAddress,
    64  		querierAddress:      querierAddress,
    65  		alertmanagerAddress: alertmanagerAddress,
    66  		rulerAddress:        rulerAddress,
    67  		timeout:             5 * time.Second,
    68  		httpClient:          &http.Client{},
    69  		querierClient:       promv1.NewAPI(querierAPIClient),
    70  		orgID:               orgID,
    71  	}
    72  
    73  	if alertmanagerAddress != "" {
    74  		alertmanagerAPIClient, err := promapi.NewClient(promapi.Config{
    75  			Address:      "http://" + alertmanagerAddress,
    76  			RoundTripper: &addOrgIDRoundTripper{orgID: orgID, next: http.DefaultTransport},
    77  		})
    78  		if err != nil {
    79  			return nil, err
    80  		}
    81  		c.alertmanagerClient = alertmanagerAPIClient
    82  	}
    83  
    84  	return c, nil
    85  }
    86  
    87  // Push the input timeseries to the remote endpoint
    88  func (c *Client) Push(timeseries []prompb.TimeSeries) (*http.Response, error) {
    89  	// Create write request
    90  	data, err := proto.Marshal(&prompb.WriteRequest{Timeseries: timeseries})
    91  	if err != nil {
    92  		return nil, err
    93  	}
    94  
    95  	// Create HTTP request
    96  	compressed := snappy.Encode(nil, data)
    97  	req, err := http.NewRequest("POST", fmt.Sprintf("http://%s/api/prom/push", c.distributorAddress), bytes.NewReader(compressed))
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	req.Header.Add("Content-Encoding", "snappy")
   103  	req.Header.Set("Content-Type", "application/x-protobuf")
   104  	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
   105  	req.Header.Set("X-Scope-OrgID", c.orgID)
   106  
   107  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   108  	defer cancel()
   109  
   110  	// Execute HTTP request
   111  	res, err := c.httpClient.Do(req.WithContext(ctx))
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  
   116  	defer res.Body.Close()
   117  	return res, nil
   118  }
   119  
   120  // Query runs an instant query.
   121  func (c *Client) Query(query string, ts time.Time) (model.Value, error) {
   122  	value, _, err := c.querierClient.Query(context.Background(), query, ts)
   123  	return value, err
   124  }
   125  
   126  // Query runs a query range.
   127  func (c *Client) QueryRange(query string, start, end time.Time, step time.Duration) (model.Value, error) {
   128  	value, _, err := c.querierClient.QueryRange(context.Background(), query, promv1.Range{
   129  		Start: start,
   130  		End:   end,
   131  		Step:  step,
   132  	})
   133  	return value, err
   134  }
   135  
   136  // QueryRangeRaw runs a ranged query directly against the querier API.
   137  func (c *Client) QueryRangeRaw(query string, start, end time.Time, step time.Duration) (*http.Response, []byte, error) {
   138  	addr := fmt.Sprintf(
   139  		"http://%s/api/prom/api/v1/query_range?query=%s&start=%s&end=%s&step=%s",
   140  		c.querierAddress,
   141  		url.QueryEscape(query),
   142  		FormatTime(start),
   143  		FormatTime(end),
   144  		strconv.FormatFloat(step.Seconds(), 'f', -1, 64),
   145  	)
   146  
   147  	return c.query(addr)
   148  }
   149  
   150  // QueryRaw runs a query directly against the querier API.
   151  func (c *Client) QueryRaw(query string) (*http.Response, []byte, error) {
   152  	addr := fmt.Sprintf("http://%s/api/prom/api/v1/query?query=%s", c.querierAddress, url.QueryEscape(query))
   153  
   154  	return c.query(addr)
   155  }
   156  
   157  func (c *Client) query(addr string) (*http.Response, []byte, error) {
   158  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   159  	defer cancel()
   160  
   161  	req, err := http.NewRequestWithContext(ctx, "GET", addr, nil)
   162  	if err != nil {
   163  		return nil, nil, err
   164  	}
   165  
   166  	req.Header.Set("X-Scope-OrgID", c.orgID)
   167  
   168  	res, err := c.httpClient.Do(req)
   169  	if err != nil {
   170  		return nil, nil, err
   171  	}
   172  	defer res.Body.Close()
   173  
   174  	body, err := ioutil.ReadAll(res.Body)
   175  	if err != nil {
   176  		return nil, nil, err
   177  	}
   178  	return res, body, nil
   179  }
   180  
   181  // Series finds series by label matchers.
   182  func (c *Client) Series(matches []string, start, end time.Time) ([]model.LabelSet, error) {
   183  	result, _, err := c.querierClient.Series(context.Background(), matches, start, end)
   184  	return result, err
   185  }
   186  
   187  // LabelValues gets label values
   188  func (c *Client) LabelValues(label string, start, end time.Time, matches []string) (model.LabelValues, error) {
   189  	result, _, err := c.querierClient.LabelValues(context.Background(), label, matches, start, end)
   190  	return result, err
   191  }
   192  
   193  // LabelNames gets label names
   194  func (c *Client) LabelNames(start, end time.Time) ([]string, error) {
   195  	result, _, err := c.querierClient.LabelNames(context.Background(), nil, start, end)
   196  	return result, err
   197  }
   198  
   199  type addOrgIDRoundTripper struct {
   200  	orgID string
   201  	next  http.RoundTripper
   202  }
   203  
   204  func (r *addOrgIDRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
   205  	req.Header.Set("X-Scope-OrgID", r.orgID)
   206  
   207  	return r.next.RoundTrip(req)
   208  }
   209  
   210  // ServerStatus represents a Alertmanager status response
   211  // TODO: Upgrade to Alertmanager v0.20.0+ and utilize vendored structs
   212  type ServerStatus struct {
   213  	Data struct {
   214  		ConfigYaml string `json:"configYAML"`
   215  	} `json:"data"`
   216  }
   217  
   218  // GetPrometheusRules fetches the rules from the Prometheus endpoint /api/v1/rules.
   219  func (c *Client) GetPrometheusRules() ([]*ruler.RuleGroup, error) {
   220  	// Create HTTP request
   221  	req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/api/prom/api/v1/rules", c.rulerAddress), nil)
   222  	if err != nil {
   223  		return nil, err
   224  	}
   225  	req.Header.Set("X-Scope-OrgID", c.orgID)
   226  
   227  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   228  	defer cancel()
   229  
   230  	// Execute HTTP request
   231  	res, err := c.httpClient.Do(req.WithContext(ctx))
   232  	if err != nil {
   233  		return nil, err
   234  	}
   235  	defer res.Body.Close()
   236  
   237  	body, err := ioutil.ReadAll(res.Body)
   238  	if err != nil {
   239  		return nil, err
   240  	}
   241  
   242  	// Decode the response.
   243  	type response struct {
   244  		Status string              `json:"status"`
   245  		Data   ruler.RuleDiscovery `json:"data"`
   246  	}
   247  
   248  	decoded := &response{}
   249  	if err := json.Unmarshal(body, decoded); err != nil {
   250  		return nil, err
   251  	}
   252  
   253  	if decoded.Status != "success" {
   254  		return nil, fmt.Errorf("unexpected response status '%s'", decoded.Status)
   255  	}
   256  
   257  	return decoded.Data.RuleGroups, nil
   258  }
   259  
   260  // GetRuleGroups gets the configured rule groups from the ruler.
   261  func (c *Client) GetRuleGroups() (map[string][]rulefmt.RuleGroup, error) {
   262  	// Create HTTP request
   263  	req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/api/prom/rules", c.rulerAddress), nil)
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  	req.Header.Set("X-Scope-OrgID", c.orgID)
   268  
   269  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   270  	defer cancel()
   271  
   272  	// Execute HTTP request
   273  	res, err := c.httpClient.Do(req.WithContext(ctx))
   274  	if err != nil {
   275  		return nil, err
   276  	}
   277  
   278  	defer res.Body.Close()
   279  	rgs := map[string][]rulefmt.RuleGroup{}
   280  
   281  	data, err := ioutil.ReadAll(res.Body)
   282  	if err != nil {
   283  		return nil, err
   284  	}
   285  
   286  	err = yaml.Unmarshal(data, rgs)
   287  	if err != nil {
   288  		return nil, err
   289  	}
   290  
   291  	return rgs, nil
   292  }
   293  
   294  // SetRuleGroup configures the provided rulegroup to the ruler.
   295  func (c *Client) SetRuleGroup(rulegroup rulefmt.RuleGroup, namespace string) error {
   296  	// Create write request
   297  	data, err := yaml.Marshal(rulegroup)
   298  	if err != nil {
   299  		return err
   300  	}
   301  
   302  	// Create HTTP request
   303  	req, err := http.NewRequest("POST", fmt.Sprintf("http://%s/api/prom/rules/%s", c.rulerAddress, url.PathEscape(namespace)), bytes.NewReader(data))
   304  	if err != nil {
   305  		return err
   306  	}
   307  
   308  	req.Header.Set("Content-Type", "application/yaml")
   309  	req.Header.Set("X-Scope-OrgID", c.orgID)
   310  
   311  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   312  	defer cancel()
   313  
   314  	// Execute HTTP request
   315  	res, err := c.httpClient.Do(req.WithContext(ctx))
   316  	if err != nil {
   317  		return err
   318  	}
   319  
   320  	defer res.Body.Close()
   321  
   322  	if res.StatusCode != 202 {
   323  		return fmt.Errorf("unexpected status code: %d", res.StatusCode)
   324  	}
   325  
   326  	return nil
   327  }
   328  
   329  // GetRuleGroup gets a rule group.
   330  func (c *Client) GetRuleGroup(namespace string, groupName string) (*http.Response, error) {
   331  	// Create HTTP request
   332  	req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/api/prom/rules/%s/%s", c.rulerAddress, url.PathEscape(namespace), url.PathEscape(groupName)), nil)
   333  	if err != nil {
   334  		return nil, err
   335  	}
   336  
   337  	req.Header.Set("Content-Type", "application/yaml")
   338  	req.Header.Set("X-Scope-OrgID", c.orgID)
   339  
   340  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   341  	defer cancel()
   342  
   343  	// Execute HTTP request
   344  	return c.httpClient.Do(req.WithContext(ctx))
   345  }
   346  
   347  // DeleteRuleGroup deletes a rule group.
   348  func (c *Client) DeleteRuleGroup(namespace string, groupName string) error {
   349  	// Create HTTP request
   350  	req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s/api/prom/rules/%s/%s", c.rulerAddress, url.PathEscape(namespace), url.PathEscape(groupName)), nil)
   351  	if err != nil {
   352  		return err
   353  	}
   354  
   355  	req.Header.Set("Content-Type", "application/yaml")
   356  	req.Header.Set("X-Scope-OrgID", c.orgID)
   357  
   358  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   359  	defer cancel()
   360  
   361  	// Execute HTTP request
   362  	res, err := c.httpClient.Do(req.WithContext(ctx))
   363  	if err != nil {
   364  		return err
   365  	}
   366  
   367  	defer res.Body.Close()
   368  	return nil
   369  }
   370  
   371  // DeleteRuleNamespace deletes all the rule groups (and the namespace itself).
   372  func (c *Client) DeleteRuleNamespace(namespace string) error {
   373  	// Create HTTP request
   374  	req, err := http.NewRequest("DELETE", fmt.Sprintf("http://%s/api/prom/rules/%s", c.rulerAddress, url.PathEscape(namespace)), nil)
   375  	if err != nil {
   376  		return err
   377  	}
   378  
   379  	req.Header.Set("X-Scope-OrgID", c.orgID)
   380  
   381  	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
   382  	defer cancel()
   383  
   384  	// Execute HTTP request
   385  	_, err = c.httpClient.Do(req.WithContext(ctx))
   386  	if err != nil {
   387  		return err
   388  	}
   389  
   390  	return nil
   391  }
   392  
   393  // userConfig is used to communicate a users alertmanager configs
   394  type userConfig struct {
   395  	TemplateFiles      map[string]string `yaml:"template_files"`
   396  	AlertmanagerConfig string            `yaml:"alertmanager_config"`
   397  }
   398  
   399  // GetAlertmanagerStatusPage gets the status page of alertmanager.
   400  func (c *Client) GetAlertmanagerStatusPage(ctx context.Context) ([]byte, error) {
   401  	return c.getRawPage(ctx, "http://"+c.alertmanagerAddress+"/multitenant_alertmanager/status")
   402  }
   403  
   404  func (c *Client) getRawPage(ctx context.Context, url string) ([]byte, error) {
   405  	req, err := http.NewRequest(http.MethodGet, url, nil)
   406  	if err != nil {
   407  		return nil, err
   408  	}
   409  	resp, err := c.httpClient.Do(req.WithContext(ctx))
   410  	if err != nil {
   411  		return nil, err
   412  	}
   413  	defer resp.Body.Close()
   414  
   415  	content, err := ioutil.ReadAll(resp.Body)
   416  	if err != nil {
   417  		return nil, err
   418  	}
   419  	if resp.StatusCode/100 != 2 {
   420  		return nil, fmt.Errorf("fetching page failed with status %d and content %v", resp.StatusCode, string(content))
   421  	}
   422  	return content, nil
   423  }
   424  
   425  // GetAlertmanagerConfig gets the status of an alertmanager instance
   426  func (c *Client) GetAlertmanagerConfig(ctx context.Context) (*alertConfig.Config, error) {
   427  	u := c.alertmanagerClient.URL("/api/prom/api/v1/status", nil)
   428  
   429  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   430  	if err != nil {
   431  		return nil, fmt.Errorf("error creating request: %v", err)
   432  	}
   433  
   434  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   435  	if err != nil {
   436  		return nil, err
   437  	}
   438  
   439  	if resp.StatusCode == http.StatusNotFound {
   440  		return nil, ErrNotFound
   441  	}
   442  
   443  	if resp.StatusCode/100 != 2 {
   444  		return nil, fmt.Errorf("getting config failed with status %d and error %v", resp.StatusCode, string(body))
   445  	}
   446  
   447  	var ss *ServerStatus
   448  	err = json.Unmarshal(body, &ss)
   449  	if err != nil {
   450  		return nil, err
   451  	}
   452  
   453  	cfg := &alertConfig.Config{}
   454  	err = yaml.Unmarshal([]byte(ss.Data.ConfigYaml), cfg)
   455  
   456  	return cfg, err
   457  }
   458  
   459  // SetAlertmanagerConfig gets the status of an alertmanager instance
   460  func (c *Client) SetAlertmanagerConfig(ctx context.Context, amConfig string, templates map[string]string) error {
   461  	u := c.alertmanagerClient.URL("/api/v1/alerts", nil)
   462  
   463  	data, err := yaml.Marshal(&userConfig{
   464  		AlertmanagerConfig: amConfig,
   465  		TemplateFiles:      templates,
   466  	})
   467  	if err != nil {
   468  		return err
   469  	}
   470  
   471  	req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader(data))
   472  	if err != nil {
   473  		return fmt.Errorf("error creating request: %v", err)
   474  	}
   475  
   476  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   477  	if err != nil {
   478  		return err
   479  	}
   480  
   481  	if resp.StatusCode == http.StatusNotFound {
   482  		return ErrNotFound
   483  	}
   484  
   485  	if resp.StatusCode != http.StatusCreated {
   486  		return fmt.Errorf("setting config failed with status %d and error %v", resp.StatusCode, string(body))
   487  	}
   488  
   489  	return nil
   490  }
   491  
   492  // DeleteAlertmanagerConfig gets the status of an alertmanager instance
   493  func (c *Client) DeleteAlertmanagerConfig(ctx context.Context) error {
   494  	u := c.alertmanagerClient.URL("/api/v1/alerts", nil)
   495  	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
   496  	if err != nil {
   497  		return fmt.Errorf("error creating request: %v", err)
   498  	}
   499  
   500  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   501  	if err != nil {
   502  		return err
   503  	}
   504  
   505  	if resp.StatusCode == http.StatusNotFound {
   506  		return ErrNotFound
   507  	}
   508  
   509  	if resp.StatusCode != http.StatusOK {
   510  		return fmt.Errorf("deleting config failed with status %d and error %v", resp.StatusCode, string(body))
   511  	}
   512  
   513  	return nil
   514  }
   515  
   516  // SendAlertToAlermanager sends alerts to the Alertmanager API
   517  func (c *Client) SendAlertToAlermanager(ctx context.Context, alert *model.Alert) error {
   518  	u := c.alertmanagerClient.URL("/api/prom/api/v1/alerts", nil)
   519  
   520  	data, err := json.Marshal([]types.Alert{{Alert: *alert}})
   521  	if err != nil {
   522  		return fmt.Errorf("error marshaling the alert: %v", err)
   523  	}
   524  
   525  	req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader(data))
   526  	if err != nil {
   527  		return fmt.Errorf("error creating request: %v", err)
   528  	}
   529  
   530  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   531  	if err != nil {
   532  		return err
   533  	}
   534  
   535  	if resp.StatusCode != http.StatusOK {
   536  		return fmt.Errorf("sending alert failed with status %d and error %v", resp.StatusCode, string(body))
   537  	}
   538  
   539  	return nil
   540  }
   541  
   542  func (c *Client) GetAlertsV1(ctx context.Context) ([]model.Alert, error) {
   543  	u := c.alertmanagerClient.URL("api/prom/api/v1/alerts", nil)
   544  
   545  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   546  	if err != nil {
   547  		return nil, fmt.Errorf("error creating request: %v", err)
   548  	}
   549  
   550  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   551  	if err != nil {
   552  		return nil, err
   553  	}
   554  
   555  	if resp.StatusCode == http.StatusNotFound {
   556  		return nil, ErrNotFound
   557  	}
   558  
   559  	if resp.StatusCode/100 != 2 {
   560  		return nil, fmt.Errorf("getting alerts failed with status %d and error %v", resp.StatusCode, string(body))
   561  	}
   562  
   563  	type response struct {
   564  		Status string        `json:"status"`
   565  		Data   []model.Alert `json:"data"`
   566  	}
   567  
   568  	decoded := &response{}
   569  	if err := json.Unmarshal(body, decoded); err != nil {
   570  		return nil, err
   571  	}
   572  
   573  	if decoded.Status != "success" {
   574  		return nil, fmt.Errorf("unexpected response status '%s'", decoded.Status)
   575  	}
   576  
   577  	return decoded.Data, nil
   578  }
   579  
   580  func (c *Client) GetAlertsV2(ctx context.Context) ([]model.Alert, error) {
   581  	u := c.alertmanagerClient.URL("api/prom/api/v2/alerts", nil)
   582  
   583  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   584  	if err != nil {
   585  		return nil, fmt.Errorf("error creating request: %v", err)
   586  	}
   587  
   588  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   589  	if err != nil {
   590  		return nil, err
   591  	}
   592  
   593  	if resp.StatusCode == http.StatusNotFound {
   594  		return nil, ErrNotFound
   595  	}
   596  
   597  	if resp.StatusCode/100 != 2 {
   598  		return nil, fmt.Errorf("getting alerts failed with status %d and error %v", resp.StatusCode, string(body))
   599  	}
   600  
   601  	decoded := []model.Alert{}
   602  	if err := json.Unmarshal(body, &decoded); err != nil {
   603  		return nil, err
   604  	}
   605  
   606  	return decoded, nil
   607  }
   608  
   609  type AlertGroup struct {
   610  	Labels model.LabelSet `json:"labels"`
   611  	Alerts []model.Alert  `json:"alerts"`
   612  }
   613  
   614  func (c *Client) GetAlertGroups(ctx context.Context) ([]AlertGroup, error) {
   615  	u := c.alertmanagerClient.URL("api/prom/api/v2/alerts/groups", nil)
   616  
   617  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   618  	if err != nil {
   619  		return nil, fmt.Errorf("error creating request: %v", err)
   620  	}
   621  
   622  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   623  	if err != nil {
   624  		return nil, err
   625  	}
   626  
   627  	if resp.StatusCode == http.StatusNotFound {
   628  		return nil, ErrNotFound
   629  	}
   630  
   631  	if resp.StatusCode/100 != 2 {
   632  		return nil, fmt.Errorf("getting alert groups failed with status %d and error %v", resp.StatusCode, string(body))
   633  	}
   634  
   635  	decoded := []AlertGroup{}
   636  	if err := json.Unmarshal(body, &decoded); err != nil {
   637  		return nil, err
   638  	}
   639  	return decoded, nil
   640  }
   641  
   642  // CreateSilence creates a new silence and returns the unique identifier of the silence.
   643  func (c *Client) CreateSilence(ctx context.Context, silence types.Silence) (string, error) {
   644  	u := c.alertmanagerClient.URL("api/prom/api/v1/silences", nil)
   645  
   646  	data, err := json.Marshal(silence)
   647  	if err != nil {
   648  		return "", fmt.Errorf("error marshaling the silence: %s", err)
   649  	}
   650  
   651  	req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader(data))
   652  	if err != nil {
   653  		return "", fmt.Errorf("error creating request: %v", err)
   654  	}
   655  
   656  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   657  	if err != nil {
   658  		return "", err
   659  	}
   660  
   661  	if resp.StatusCode != http.StatusOK {
   662  		return "", fmt.Errorf("creating the silence failed with status %d and error %v", resp.StatusCode, string(body))
   663  	}
   664  
   665  	type response struct {
   666  		Status string `json:"status"`
   667  		Data   struct {
   668  			SilenceID string `json:"silenceID"`
   669  		} `json:"data"`
   670  	}
   671  
   672  	decoded := &response{}
   673  	if err := json.Unmarshal(body, decoded); err != nil {
   674  		return "", err
   675  	}
   676  
   677  	if decoded.Status != "success" {
   678  		return "", fmt.Errorf("unexpected response status '%s'", decoded.Status)
   679  	}
   680  
   681  	return decoded.Data.SilenceID, nil
   682  }
   683  
   684  func (c *Client) GetSilencesV1(ctx context.Context) ([]types.Silence, error) {
   685  	u := c.alertmanagerClient.URL("api/prom/api/v1/silences", nil)
   686  
   687  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   688  	if err != nil {
   689  		return nil, fmt.Errorf("error creating request: %v", err)
   690  	}
   691  
   692  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   693  	if err != nil {
   694  		return nil, err
   695  	}
   696  
   697  	if resp.StatusCode == http.StatusNotFound {
   698  		return nil, ErrNotFound
   699  	}
   700  
   701  	if resp.StatusCode/100 != 2 {
   702  		return nil, fmt.Errorf("getting silences failed with status %d and error %v", resp.StatusCode, string(body))
   703  	}
   704  
   705  	type response struct {
   706  		Status string          `json:"status"`
   707  		Data   []types.Silence `json:"data"`
   708  	}
   709  
   710  	decoded := &response{}
   711  	if err := json.Unmarshal(body, decoded); err != nil {
   712  		return nil, err
   713  	}
   714  
   715  	if decoded.Status != "success" {
   716  		return nil, fmt.Errorf("unexpected response status '%s'", decoded.Status)
   717  	}
   718  
   719  	return decoded.Data, nil
   720  }
   721  
   722  func (c *Client) GetSilencesV2(ctx context.Context) ([]types.Silence, error) {
   723  	u := c.alertmanagerClient.URL("api/prom/api/v2/silences", nil)
   724  
   725  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   726  	if err != nil {
   727  		return nil, fmt.Errorf("error creating request: %v", err)
   728  	}
   729  
   730  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   731  	if err != nil {
   732  		return nil, err
   733  	}
   734  
   735  	if resp.StatusCode == http.StatusNotFound {
   736  		return nil, ErrNotFound
   737  	}
   738  
   739  	if resp.StatusCode/100 != 2 {
   740  		return nil, fmt.Errorf("getting silences failed with status %d and error %v", resp.StatusCode, string(body))
   741  	}
   742  
   743  	decoded := []types.Silence{}
   744  	if err := json.Unmarshal(body, &decoded); err != nil {
   745  		return nil, err
   746  	}
   747  
   748  	return decoded, nil
   749  }
   750  
   751  func (c *Client) GetSilenceV1(ctx context.Context, id string) (types.Silence, error) {
   752  	u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v1/silence/%s", url.PathEscape(id)), nil)
   753  
   754  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   755  	if err != nil {
   756  		return types.Silence{}, fmt.Errorf("error creating request: %v", err)
   757  	}
   758  
   759  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   760  	if err != nil {
   761  		return types.Silence{}, err
   762  	}
   763  
   764  	if resp.StatusCode == http.StatusNotFound {
   765  		return types.Silence{}, ErrNotFound
   766  	}
   767  
   768  	if resp.StatusCode/100 != 2 {
   769  		return types.Silence{}, fmt.Errorf("getting silence failed with status %d and error %v", resp.StatusCode, string(body))
   770  	}
   771  
   772  	type response struct {
   773  		Status string        `json:"status"`
   774  		Data   types.Silence `json:"data"`
   775  	}
   776  
   777  	decoded := &response{}
   778  	if err := json.Unmarshal(body, decoded); err != nil {
   779  		return types.Silence{}, err
   780  	}
   781  
   782  	if decoded.Status != "success" {
   783  		return types.Silence{}, fmt.Errorf("unexpected response status '%s'", decoded.Status)
   784  	}
   785  
   786  	return decoded.Data, nil
   787  }
   788  
   789  func (c *Client) GetSilenceV2(ctx context.Context, id string) (types.Silence, error) {
   790  	u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v2/silence/%s", url.PathEscape(id)), nil)
   791  
   792  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   793  	if err != nil {
   794  		return types.Silence{}, fmt.Errorf("error creating request: %v", err)
   795  	}
   796  
   797  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   798  	if err != nil {
   799  		return types.Silence{}, err
   800  	}
   801  
   802  	if resp.StatusCode == http.StatusNotFound {
   803  		return types.Silence{}, ErrNotFound
   804  	}
   805  
   806  	if resp.StatusCode/100 != 2 {
   807  		return types.Silence{}, fmt.Errorf("getting silence failed with status %d and error %v", resp.StatusCode, string(body))
   808  	}
   809  
   810  	decoded := types.Silence{}
   811  	if err := json.Unmarshal(body, &decoded); err != nil {
   812  		return types.Silence{}, err
   813  	}
   814  
   815  	return decoded, nil
   816  }
   817  
   818  func (c *Client) DeleteSilence(ctx context.Context, id string) error {
   819  	u := c.alertmanagerClient.URL(fmt.Sprintf("api/prom/api/v1/silence/%s", url.PathEscape(id)), nil)
   820  
   821  	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
   822  	if err != nil {
   823  		return fmt.Errorf("error creating request: %v", err)
   824  	}
   825  
   826  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   827  	if err != nil {
   828  		return err
   829  	}
   830  
   831  	if resp.StatusCode == http.StatusNotFound {
   832  		return ErrNotFound
   833  	}
   834  
   835  	if resp.StatusCode != http.StatusOK {
   836  		return fmt.Errorf("deleting silence failed with status %d and error %v", resp.StatusCode, string(body))
   837  	}
   838  
   839  	return nil
   840  }
   841  
   842  func (c *Client) GetReceivers(ctx context.Context) ([]string, error) {
   843  	u := c.alertmanagerClient.URL("api/prom/api/v1/receivers", nil)
   844  
   845  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   846  	if err != nil {
   847  		return nil, fmt.Errorf("error creating request: %v", err)
   848  	}
   849  
   850  	resp, body, err := c.alertmanagerClient.Do(ctx, req)
   851  	if err != nil {
   852  		return nil, err
   853  	}
   854  
   855  	if resp.StatusCode == http.StatusNotFound {
   856  		return nil, ErrNotFound
   857  	}
   858  
   859  	if resp.StatusCode/100 != 2 {
   860  		return nil, fmt.Errorf("getting receivers failed with status %d and error %v", resp.StatusCode, string(body))
   861  	}
   862  
   863  	type response struct {
   864  		Status string   `json:"status"`
   865  		Data   []string `json:"data"`
   866  	}
   867  
   868  	decoded := &response{}
   869  	if err := json.Unmarshal(body, decoded); err != nil {
   870  		return nil, err
   871  	}
   872  
   873  	if decoded.Status != "success" {
   874  		return nil, fmt.Errorf("unexpected response status '%s'", decoded.Status)
   875  	}
   876  
   877  	return decoded.Data, nil
   878  }
   879  
   880  func (c *Client) PostRequest(url string, body io.Reader) (*http.Response, error) {
   881  	req, err := http.NewRequest("POST", url, body)
   882  	if err != nil {
   883  		return nil, err
   884  	}
   885  
   886  	req.Header.Set("X-Scope-OrgID", c.orgID)
   887  
   888  	client := &http.Client{Timeout: c.timeout}
   889  	return client.Do(req)
   890  }
   891  
   892  // FormatTime converts a time to a string acceptable by the Prometheus API.
   893  func FormatTime(t time.Time) string {
   894  	return strconv.FormatFloat(float64(t.Unix())+float64(t.Nanosecond())/1e9, 'f', -1, 64)
   895  }