go.etcd.io/etcd@v3.3.27+incompatible/rafthttp/snapshot_sender.go (about) 1 // Copyright 2015 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package rafthttp 16 17 import ( 18 "bytes" 19 "context" 20 "io" 21 "io/ioutil" 22 "net/http" 23 "time" 24 25 "github.com/coreos/etcd/pkg/httputil" 26 pioutil "github.com/coreos/etcd/pkg/ioutil" 27 "github.com/coreos/etcd/pkg/types" 28 "github.com/coreos/etcd/raft" 29 "github.com/coreos/etcd/snap" 30 "github.com/dustin/go-humanize" 31 ) 32 33 var ( 34 // timeout for reading snapshot response body 35 snapResponseReadTimeout = 5 * time.Second 36 ) 37 38 type snapshotSender struct { 39 from, to types.ID 40 cid types.ID 41 42 tr *Transport 43 picker *urlPicker 44 status *peerStatus 45 r Raft 46 errorc chan error 47 48 stopc chan struct{} 49 } 50 51 func newSnapshotSender(tr *Transport, picker *urlPicker, to types.ID, status *peerStatus) *snapshotSender { 52 return &snapshotSender{ 53 from: tr.ID, 54 to: to, 55 cid: tr.ClusterID, 56 tr: tr, 57 picker: picker, 58 status: status, 59 r: tr.Raft, 60 errorc: tr.ErrorC, 61 stopc: make(chan struct{}), 62 } 63 } 64 65 func (s *snapshotSender) stop() { close(s.stopc) } 66 67 func (s *snapshotSender) send(merged snap.Message) { 68 start := time.Now() 69 70 m := merged.Message 71 to := types.ID(m.To).String() 72 73 body := createSnapBody(merged) 74 defer body.Close() 75 76 u := s.picker.pick() 77 req := createPostRequest(u, RaftSnapshotPrefix, body, "application/octet-stream", s.tr.URLs, s.from, s.cid) 78 79 snapshotTotalSizeVal := uint64(merged.TotalSize) 80 snapshotTotalSize := humanize.Bytes(snapshotTotalSizeVal) 81 plog.Infof("start to send database snapshot [index: %d, to %s, size %s]...", m.Snapshot.Metadata.Index, types.ID(m.To), snapshotTotalSize) 82 snapshotSendInflights.WithLabelValues(to).Inc() 83 defer func() { 84 snapshotSendInflights.WithLabelValues(to).Dec() 85 }() 86 87 err := s.post(req) 88 defer merged.CloseWithError(err) 89 if err != nil { 90 plog.Warningf("database snapshot [index: %d, to: %s] failed to be sent out (%v)", m.Snapshot.Metadata.Index, types.ID(m.To), err) 91 92 // errMemberRemoved is a critical error since a removed member should 93 // always be stopped. So we use reportCriticalError to report it to errorc. 94 if err == errMemberRemoved { 95 reportCriticalError(err, s.errorc) 96 } 97 98 s.picker.unreachable(u) 99 s.status.deactivate(failureType{source: sendSnap, action: "post"}, err.Error()) 100 s.r.ReportUnreachable(m.To) 101 // report SnapshotFailure to raft state machine. After raft state 102 // machine knows about it, it would pause a while and retry sending 103 // new snapshot message. 104 s.r.ReportSnapshot(m.To, raft.SnapshotFailure) 105 sentFailures.WithLabelValues(to).Inc() 106 snapshotSendFailures.WithLabelValues(to).Inc() 107 return 108 } 109 s.status.activate() 110 s.r.ReportSnapshot(m.To, raft.SnapshotFinish) 111 plog.Infof("database snapshot [index: %d, to: %s] sent out successfully", m.Snapshot.Metadata.Index, types.ID(m.To)) 112 113 sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize)) 114 115 snapshotSend.WithLabelValues(to).Inc() 116 snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds()) 117 } 118 119 // post posts the given request. 120 // It returns nil when request is sent out and processed successfully. 121 func (s *snapshotSender) post(req *http.Request) (err error) { 122 ctx, cancel := context.WithCancel(context.Background()) 123 req = req.WithContext(ctx) 124 defer cancel() 125 126 type responseAndError struct { 127 resp *http.Response 128 body []byte 129 err error 130 } 131 result := make(chan responseAndError, 1) 132 133 go func() { 134 resp, err := s.tr.pipelineRt.RoundTrip(req) 135 if err != nil { 136 result <- responseAndError{resp, nil, err} 137 return 138 } 139 140 // close the response body when timeouts. 141 // prevents from reading the body forever when the other side dies right after 142 // successfully receives the request body. 143 time.AfterFunc(snapResponseReadTimeout, func() { httputil.GracefulClose(resp) }) 144 body, err := ioutil.ReadAll(resp.Body) 145 result <- responseAndError{resp, body, err} 146 }() 147 148 select { 149 case <-s.stopc: 150 return errStopped 151 case r := <-result: 152 if r.err != nil { 153 return r.err 154 } 155 return checkPostResponse(r.resp, r.body, req, s.to) 156 } 157 } 158 159 func createSnapBody(merged snap.Message) io.ReadCloser { 160 buf := new(bytes.Buffer) 161 enc := &messageEncoder{w: buf} 162 // encode raft message 163 if err := enc.encode(&merged.Message); err != nil { 164 plog.Panicf("encode message error (%v)", err) 165 } 166 167 return &pioutil.ReaderAndCloser{ 168 Reader: io.MultiReader(buf, merged.ReadCloser), 169 Closer: merged.ReadCloser, 170 } 171 }