github.com/zmap/zcrypto@v0.0.0-20240512203510-0fef58d9a9db/ct/client/logclient.go (about) 1 // Package client is a CT log client implementation and contains types and code 2 // for interacting with RFC6962-compliant CT Log instances. 3 // See http://tools.ietf.org/html/rfc6962 for details 4 package client 5 6 import ( 7 "bytes" 8 "crypto/sha256" 9 "crypto/tls" 10 "encoding/base64" 11 "encoding/json" 12 "errors" 13 "fmt" 14 "io/ioutil" 15 "log" 16 "net/http" 17 "strconv" 18 "time" 19 20 "github.com/mreiferson/go-httpclient" 21 "github.com/zmap/zcrypto/ct" 22 "golang.org/x/net/context" 23 ) 24 25 // URI paths for CT Log endpoints 26 const ( 27 AddChainPath = "/ct/v1/add-chain" 28 AddPreChainPath = "/ct/v1/add-pre-chain" 29 AddJSONPath = "/ct/v1/add-json" 30 GetSTHPath = "/ct/v1/get-sth" 31 GetEntriesPath = "/ct/v1/get-entries" 32 ) 33 34 // LogClient represents a client for a given CT Log instance 35 type LogClient struct { 36 Uri string // the base URI of the log. e.g. http://ct.googleapis/pilot 37 httpClient *http.Client // used to interact with the log via HTTP 38 } 39 40 ////////////////////////////////////////////////////////////////////////////////// 41 // JSON structures follow. 42 // These represent the structures returned by the CT Log server. 43 ////////////////////////////////////////////////////////////////////////////////// 44 45 // addChainRequest represents the JSON request body sent to the add-chain CT 46 // method. 47 type addChainRequest struct { 48 Chain []string `json:"chain"` 49 } 50 51 // addChainResponse represents the JSON response to the add-chain CT method. 52 // An SCT represents a Log's promise to integrate a [pre-]certificate into the 53 // log within a defined period of time. 54 type addChainResponse struct { 55 SCTVersion ct.Version `json:"sct_version"` // SCT structure version 56 ID string `json:"id"` // Log ID 57 Timestamp uint64 `json:"timestamp"` // Timestamp of issuance 58 Extensions string `json:"extensions"` // Holder for any CT extensions 59 Signature string `json:"signature"` // Log signature for this SCT 60 } 61 62 // addJSONRequest represents the JSON request body sent ot the add-json CT 63 // method. 64 type addJSONRequest struct { 65 Data interface{} `json:"data"` 66 } 67 68 // getSTHResponse respresents the JSON response to the get-sth CT method 69 type getSTHResponse struct { 70 TreeSize uint64 `json:"tree_size"` // Number of certs in the current tree 71 Timestamp uint64 `json:"timestamp"` // Time that the tree was created 72 SHA256RootHash string `json:"sha256_root_hash"` // Root hash of the tree 73 TreeHeadSignature string `json:"tree_head_signature"` // Log signature for this STH 74 } 75 76 // base64LeafEntry respresents a Base64 encoded leaf entry 77 type base64LeafEntry struct { 78 LeafInput string `json:"leaf_input"` 79 ExtraData string `json:"extra_data"` 80 } 81 82 // getEntriesReponse respresents the JSON response to the CT get-entries method 83 type getEntriesResponse struct { 84 Entries []base64LeafEntry `json:"entries"` // the list of returned entries 85 } 86 87 // getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method 88 type getConsistencyProofResponse struct { 89 Consistency []string `json:"consistency"` 90 } 91 92 // getAuditProofResponse represents the JSON response to the CT get-audit-proof method 93 type getAuditProofResponse struct { 94 Hash []string `json:"hash"` // the hashes which make up the proof 95 TreeSize uint64 `json:"tree_size"` // the tree size against which this proof is constructed 96 } 97 98 // getAcceptedRootsResponse represents the JSON response to the CT get-roots method. 99 type getAcceptedRootsResponse struct { 100 Certificates []string `json:"certificates"` 101 } 102 103 // getEntryAndProodReponse represents the JSON response to the CT get-entry-and-proof method 104 type getEntryAndProofResponse struct { 105 LeafInput string `json:"leaf_input"` // the entry itself 106 ExtraData string `json:"extra_data"` // any chain provided when the entry was added to the log 107 AuditPath []string `json:"audit_path"` // the corresponding proof 108 } 109 110 // New constructs a new LogClient instance. 111 // |uri| is the base URI of the CT log instance to interact with, e.g. 112 // http://ct.googleapis.com/pilot 113 func New(uri string) *LogClient { 114 var c LogClient 115 c.Uri = uri 116 transport := &httpclient.Transport{ 117 ConnectTimeout: 10 * time.Second, 118 RequestTimeout: 30 * time.Second, 119 ResponseHeaderTimeout: 30 * time.Second, 120 MaxIdleConnsPerHost: 10, 121 DisableKeepAlives: false, 122 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 123 } 124 c.httpClient = &http.Client{Transport: transport} 125 return &c 126 } 127 128 // Makes a HTTP call to |uri|, and attempts to parse the response as a JSON 129 // representation of the structure in |res|. 130 // Returns a non-nil |error| if there was a problem. 131 func (c *LogClient) fetchAndParse(uri string, res interface{}) error { 132 req, err := http.NewRequest("GET", uri, nil) 133 if err != nil { 134 return err 135 } 136 resp, err := c.httpClient.Do(req) 137 var body []byte 138 if resp != nil { 139 if resp.StatusCode > 399 { 140 return errors.New("HTTP error: " + resp.Status) 141 } 142 body, err = ioutil.ReadAll(resp.Body) 143 resp.Body.Close() 144 if err != nil { 145 return err 146 } 147 } 148 if err != nil { 149 return err 150 } 151 if err = json.Unmarshal(body, &res); err != nil { 152 fmt.Println(string(body)) 153 return err 154 } 155 return nil 156 } 157 158 // Makes a HTTP POST call to |uri|, and attempts to parse the response as a JSON 159 // representation of the structure in |res|. 160 // Returns a non-nil |error| if there was a problem. 161 func (c *LogClient) postAndParse(uri string, req interface{}, res interface{}) (*http.Response, string, error) { 162 postBody, err := json.Marshal(req) 163 if err != nil { 164 return nil, "", err 165 } 166 httpReq, err := http.NewRequest("POST", uri, bytes.NewReader(postBody)) 167 if err != nil { 168 return nil, "", err 169 } 170 //httpReq.Header.Set("Keep-Alive", "timeout=15, max=100") 171 httpReq.Header.Set("Content-Type", "application/json") 172 resp, err := c.httpClient.Do(httpReq) 173 // Read all of the body, if there is one, so that the http.Client can do 174 // Keep-Alive: 175 var body []byte 176 if resp != nil { 177 body, err = ioutil.ReadAll(resp.Body) 178 resp.Body.Close() 179 } 180 if err != nil { 181 return resp, string(body), err 182 } 183 if resp.StatusCode == 200 { 184 if err != nil { 185 return resp, string(body), err 186 } 187 if err = json.Unmarshal(body, &res); err != nil { 188 return resp, string(body), err 189 } 190 } 191 return resp, string(body), nil 192 } 193 194 func backoffForRetry(ctx context.Context, d time.Duration) error { 195 backoffTimer := time.NewTimer(d) 196 if ctx != nil { 197 select { 198 case <-ctx.Done(): 199 return ctx.Err() 200 case <-backoffTimer.C: 201 } 202 } else { 203 <-backoffTimer.C 204 } 205 return nil 206 } 207 208 // Attempts to add |chain| to the log, using the api end-point specified by 209 // |path|. If provided context expires before submission is complete an 210 // error will be returned. 211 func (c *LogClient) addChainWithRetry(ctx context.Context, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) { 212 var resp addChainResponse 213 var req addChainRequest 214 for _, link := range chain { 215 req.Chain = append(req.Chain, base64.StdEncoding.EncodeToString(link)) 216 } 217 httpStatus := "Unknown" 218 httpCode := 0 219 backoffSeconds := 0 220 done := false 221 for !done { 222 if backoffSeconds > 0 { 223 log.Printf("Got %s, backing-off %d seconds", httpStatus, backoffSeconds) 224 } 225 err := backoffForRetry(ctx, time.Second*time.Duration(backoffSeconds)) 226 if err != nil { 227 return nil, err, 0 228 } 229 if backoffSeconds > 0 { 230 backoffSeconds = 0 231 } 232 httpResp, errorBody, err := c.postAndParse(c.Uri+path, &req, &resp) 233 if err != nil { 234 backoffSeconds = 10 235 continue 236 } 237 switch { 238 case httpResp.StatusCode == 200: 239 done = true 240 case httpResp.StatusCode == 408: 241 // request timeout, retry immediately 242 case httpResp.StatusCode == 503: 243 // Retry 244 backoffSeconds = 10 245 if retryAfter := httpResp.Header.Get("Retry-After"); retryAfter != "" { 246 if seconds, err := strconv.Atoi(retryAfter); err == nil { 247 backoffSeconds = seconds 248 } 249 } 250 default: 251 return nil, fmt.Errorf("got HTTP Status %s: %s", httpResp.Status, errorBody), httpResp.StatusCode 252 } 253 httpStatus = httpResp.Status 254 httpCode = httpResp.StatusCode 255 } 256 257 rawLogID, err := base64.StdEncoding.DecodeString(resp.ID) 258 if err != nil { 259 return nil, err, httpCode 260 } 261 rawSignature, err := base64.StdEncoding.DecodeString(resp.Signature) 262 if err != nil { 263 return nil, err, httpCode 264 } 265 ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature)) 266 if err != nil { 267 return nil, err, httpCode 268 } 269 var logID ct.SHA256Hash 270 copy(logID[:], rawLogID) 271 return &ct.SignedCertificateTimestamp{ 272 SCTVersion: resp.SCTVersion, 273 LogID: logID, 274 Timestamp: resp.Timestamp, 275 Extensions: ct.CTExtensions(resp.Extensions), 276 Signature: *ds}, nil, httpCode 277 } 278 279 // AddChain adds the (DER represented) X509 |chain| to the log. 280 func (c *LogClient) AddChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) { 281 return c.addChainWithRetry(nil, AddChainPath, chain) 282 } 283 284 // AddPreChain adds the (DER represented) Precertificate |chain| to the log. 285 func (c *LogClient) AddPreChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) { 286 return c.addChainWithRetry(nil, AddPreChainPath, chain) 287 } 288 289 // AddChainWithContext adds the (DER represented) X509 |chain| to the log and 290 // fails if the provided context expires before the chain is submitted. 291 func (c *LogClient) AddChainWithContext(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) { 292 return c.addChainWithRetry(ctx, AddChainPath, chain) 293 } 294 295 func (c *LogClient) AddJSON(data interface{}) (*ct.SignedCertificateTimestamp, error) { 296 req := addJSONRequest{ 297 Data: data, 298 } 299 var resp addChainResponse 300 _, _, err := c.postAndParse(c.Uri+AddJSONPath, &req, &resp) 301 if err != nil { 302 return nil, err 303 } 304 rawLogID, err := base64.StdEncoding.DecodeString(resp.ID) 305 if err != nil { 306 return nil, err 307 } 308 rawSignature, err := base64.StdEncoding.DecodeString(resp.Signature) 309 if err != nil { 310 return nil, err 311 } 312 ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature)) 313 if err != nil { 314 return nil, err 315 } 316 var logID ct.SHA256Hash 317 copy(logID[:], rawLogID) 318 return &ct.SignedCertificateTimestamp{ 319 SCTVersion: resp.SCTVersion, 320 LogID: logID, 321 Timestamp: resp.Timestamp, 322 Extensions: ct.CTExtensions(resp.Extensions), 323 Signature: *ds}, nil 324 } 325 326 // GetSTH retrieves the current STH from the log. 327 // Returns a populated SignedTreeHead, or a non-nil error. 328 func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) { 329 var resp getSTHResponse 330 if err = c.fetchAndParse(c.Uri+GetSTHPath, &resp); err != nil { 331 return 332 } 333 sth = &ct.SignedTreeHead{ 334 TreeSize: resp.TreeSize, 335 Timestamp: resp.Timestamp, 336 } 337 338 rawRootHash, err := base64.StdEncoding.DecodeString(resp.SHA256RootHash) 339 if err != nil { 340 return nil, fmt.Errorf("invalid base64 encoding in sha256_root_hash: %v", err) 341 } 342 if len(rawRootHash) != sha256.Size { 343 return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(rawRootHash)) 344 } 345 copy(sth.SHA256RootHash[:], rawRootHash) 346 347 rawSignature, err := base64.StdEncoding.DecodeString(resp.TreeHeadSignature) 348 if err != nil { 349 return nil, errors.New("invalid base64 encoding in tree_head_signature") 350 } 351 ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature)) 352 if err != nil { 353 return nil, err 354 } 355 // TODO(alcutter): Verify signature 356 sth.TreeHeadSignature = *ds 357 return 358 } 359 360 // GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT 361 // log server. (see section 4.6.) 362 // Returns a slice of LeafInputs or a non-nil error. 363 func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) { 364 if end < 0 { 365 return nil, errors.New("end should be >= 0") 366 } 367 if end < start { 368 return nil, errors.New("start should be <= end") 369 } 370 var resp getEntriesResponse 371 err := c.fetchAndParse(fmt.Sprintf("%s%s?start=%d&end=%d", c.Uri, GetEntriesPath, start, end), &resp) 372 if err != nil { 373 return nil, err 374 } 375 entries := make([]ct.LogEntry, len(resp.Entries)) 376 for index, entry := range resp.Entries { 377 leafBytes, err := base64.StdEncoding.DecodeString(entry.LeafInput) 378 if err != nil { 379 return nil, err 380 } 381 leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewBuffer(leafBytes)) 382 if err != nil { 383 return nil, err 384 } 385 entries[index].Leaf = *leaf 386 chainBytes, err := base64.StdEncoding.DecodeString(entry.ExtraData) 387 if err != nil { 388 return nil, err 389 } 390 391 var chain []ct.ASN1Cert 392 switch leaf.TimestampedEntry.EntryType { 393 case ct.X509LogEntryType: 394 chain, err = ct.UnmarshalX509ChainArray(chainBytes) 395 396 case ct.PrecertLogEntryType: 397 chain, err = ct.UnmarshalPrecertChainArray(chainBytes) 398 399 default: 400 return nil, fmt.Errorf("saw unknown entry type: %v", leaf.TimestampedEntry.EntryType) 401 } 402 if err != nil { 403 return nil, err 404 } 405 entries[index].Chain = chain 406 entries[index].Index = start + int64(index) 407 } 408 return entries, nil 409 }