go.etcd.io/etcd@v3.3.27+incompatible/rafthttp/pipeline_test.go (about) 1 // Copyright 2015 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package rafthttp 16 17 import ( 18 "errors" 19 "fmt" 20 "io" 21 "io/ioutil" 22 "net/http" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/coreos/etcd/etcdserver/stats" 28 "github.com/coreos/etcd/pkg/testutil" 29 "github.com/coreos/etcd/pkg/types" 30 "github.com/coreos/etcd/raft/raftpb" 31 "github.com/coreos/etcd/version" 32 ) 33 34 // TestPipelineSend tests that pipeline could send data using roundtripper 35 // and increase success count in stats. 36 func TestPipelineSend(t *testing.T) { 37 tr := &roundTripperRecorder{rec: testutil.NewRecorderStream()} 38 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 39 tp := &Transport{pipelineRt: tr} 40 p := startTestPipeline(tp, picker) 41 42 p.msgc <- raftpb.Message{Type: raftpb.MsgApp} 43 tr.rec.Wait(1) 44 p.stop() 45 if p.followerStats.Counts.Success != 1 { 46 t.Errorf("success = %d, want 1", p.followerStats.Counts.Success) 47 } 48 } 49 50 // TestPipelineKeepSendingWhenPostError tests that pipeline can keep 51 // sending messages if previous messages meet post error. 52 func TestPipelineKeepSendingWhenPostError(t *testing.T) { 53 tr := &respRoundTripper{rec: testutil.NewRecorderStream(), err: fmt.Errorf("roundtrip error")} 54 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 55 tp := &Transport{pipelineRt: tr} 56 p := startTestPipeline(tp, picker) 57 defer p.stop() 58 59 for i := 0; i < 50; i++ { 60 p.msgc <- raftpb.Message{Type: raftpb.MsgApp} 61 } 62 63 _, err := tr.rec.Wait(50) 64 if err != nil { 65 t.Errorf("unexpected wait error %v", err) 66 } 67 } 68 69 func TestPipelineExceedMaximumServing(t *testing.T) { 70 rt := newRoundTripperBlocker() 71 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 72 tp := &Transport{pipelineRt: rt} 73 p := startTestPipeline(tp, picker) 74 defer p.stop() 75 76 // keep the sender busy and make the buffer full 77 // nothing can go out as we block the sender 78 for i := 0; i < connPerPipeline+pipelineBufSize; i++ { 79 select { 80 case p.msgc <- raftpb.Message{}: 81 case <-time.After(time.Second): 82 t.Errorf("failed to send out message") 83 } 84 } 85 86 // try to send a data when we are sure the buffer is full 87 select { 88 case p.msgc <- raftpb.Message{}: 89 t.Errorf("unexpected message sendout") 90 default: 91 } 92 93 // unblock the senders and force them to send out the data 94 rt.unblock() 95 96 // It could send new data after previous ones succeed 97 select { 98 case p.msgc <- raftpb.Message{}: 99 case <-time.After(time.Second): 100 t.Errorf("failed to send out message") 101 } 102 } 103 104 // TestPipelineSendFailed tests that when send func meets the post error, 105 // it increases fail count in stats. 106 func TestPipelineSendFailed(t *testing.T) { 107 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 108 rt := newRespRoundTripper(0, errors.New("blah")) 109 rt.rec = testutil.NewRecorderStream() 110 tp := &Transport{pipelineRt: rt} 111 p := startTestPipeline(tp, picker) 112 113 p.msgc <- raftpb.Message{Type: raftpb.MsgApp} 114 if _, err := rt.rec.Wait(1); err != nil { 115 t.Fatal(err) 116 } 117 118 p.stop() 119 120 if p.followerStats.Counts.Fail != 1 { 121 t.Errorf("fail = %d, want 1", p.followerStats.Counts.Fail) 122 } 123 } 124 125 func TestPipelinePost(t *testing.T) { 126 tr := &roundTripperRecorder{rec: &testutil.RecorderBuffered{}} 127 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 128 tp := &Transport{ClusterID: types.ID(1), pipelineRt: tr} 129 p := startTestPipeline(tp, picker) 130 if err := p.post([]byte("some data")); err != nil { 131 t.Fatalf("unexpected post error: %v", err) 132 } 133 act, err := tr.rec.Wait(1) 134 if err != nil { 135 t.Fatal(err) 136 } 137 p.stop() 138 139 req := act[0].Params[0].(*http.Request) 140 141 if g := req.Method; g != "POST" { 142 t.Errorf("method = %s, want %s", g, "POST") 143 } 144 if g := req.URL.String(); g != "http://localhost:2380/raft" { 145 t.Errorf("url = %s, want %s", g, "http://localhost:2380/raft") 146 } 147 if g := req.Header.Get("Content-Type"); g != "application/protobuf" { 148 t.Errorf("content type = %s, want %s", g, "application/protobuf") 149 } 150 if g := req.Header.Get("X-Server-Version"); g != version.Version { 151 t.Errorf("version = %s, want %s", g, version.Version) 152 } 153 if g := req.Header.Get("X-Min-Cluster-Version"); g != version.MinClusterVersion { 154 t.Errorf("min version = %s, want %s", g, version.MinClusterVersion) 155 } 156 if g := req.Header.Get("X-Etcd-Cluster-ID"); g != "1" { 157 t.Errorf("cluster id = %s, want %s", g, "1") 158 } 159 b, err := ioutil.ReadAll(req.Body) 160 if err != nil { 161 t.Fatalf("unexpected ReadAll error: %v", err) 162 } 163 if string(b) != "some data" { 164 t.Errorf("body = %s, want %s", b, "some data") 165 } 166 } 167 168 func TestPipelinePostBad(t *testing.T) { 169 tests := []struct { 170 u string 171 code int 172 err error 173 }{ 174 // RoundTrip returns error 175 {"http://localhost:2380", 0, errors.New("blah")}, 176 // unexpected response status code 177 {"http://localhost:2380", http.StatusOK, nil}, 178 {"http://localhost:2380", http.StatusCreated, nil}, 179 } 180 for i, tt := range tests { 181 picker := mustNewURLPicker(t, []string{tt.u}) 182 tp := &Transport{pipelineRt: newRespRoundTripper(tt.code, tt.err)} 183 p := startTestPipeline(tp, picker) 184 err := p.post([]byte("some data")) 185 p.stop() 186 187 if err == nil { 188 t.Errorf("#%d: err = nil, want not nil", i) 189 } 190 } 191 } 192 193 func TestPipelinePostErrorc(t *testing.T) { 194 tests := []struct { 195 u string 196 code int 197 err error 198 }{ 199 {"http://localhost:2380", http.StatusForbidden, nil}, 200 } 201 for i, tt := range tests { 202 picker := mustNewURLPicker(t, []string{tt.u}) 203 tp := &Transport{pipelineRt: newRespRoundTripper(tt.code, tt.err)} 204 p := startTestPipeline(tp, picker) 205 p.post([]byte("some data")) 206 p.stop() 207 select { 208 case <-p.errorc: 209 default: 210 t.Fatalf("#%d: cannot receive from errorc", i) 211 } 212 } 213 } 214 215 func TestStopBlockedPipeline(t *testing.T) { 216 picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) 217 tp := &Transport{pipelineRt: newRoundTripperBlocker()} 218 p := startTestPipeline(tp, picker) 219 // send many messages that most of them will be blocked in buffer 220 for i := 0; i < connPerPipeline*10; i++ { 221 p.msgc <- raftpb.Message{} 222 } 223 224 done := make(chan struct{}) 225 go func() { 226 p.stop() 227 done <- struct{}{} 228 }() 229 select { 230 case <-done: 231 case <-time.After(time.Second): 232 t.Fatalf("failed to stop pipeline in 1s") 233 } 234 } 235 236 type roundTripperBlocker struct { 237 unblockc chan struct{} 238 mu sync.Mutex 239 cancel map[*http.Request]chan struct{} 240 } 241 242 func newRoundTripperBlocker() *roundTripperBlocker { 243 return &roundTripperBlocker{ 244 unblockc: make(chan struct{}), 245 cancel: make(map[*http.Request]chan struct{}), 246 } 247 } 248 249 func (t *roundTripperBlocker) unblock() { 250 close(t.unblockc) 251 } 252 253 func (t *roundTripperBlocker) CancelRequest(req *http.Request) { 254 t.mu.Lock() 255 defer t.mu.Unlock() 256 if c, ok := t.cancel[req]; ok { 257 c <- struct{}{} 258 delete(t.cancel, req) 259 } 260 } 261 262 type respRoundTripper struct { 263 mu sync.Mutex 264 rec testutil.Recorder 265 266 code int 267 header http.Header 268 err error 269 } 270 271 func newRespRoundTripper(code int, err error) *respRoundTripper { 272 return &respRoundTripper{code: code, err: err} 273 } 274 func (t *respRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 275 t.mu.Lock() 276 defer t.mu.Unlock() 277 if t.rec != nil { 278 t.rec.Record(testutil.Action{Name: "req", Params: []interface{}{req}}) 279 } 280 return &http.Response{StatusCode: t.code, Header: t.header, Body: &nopReadCloser{}}, t.err 281 } 282 283 type roundTripperRecorder struct { 284 rec testutil.Recorder 285 } 286 287 func (t *roundTripperRecorder) RoundTrip(req *http.Request) (*http.Response, error) { 288 if t.rec != nil { 289 t.rec.Record(testutil.Action{Name: "req", Params: []interface{}{req}}) 290 } 291 return &http.Response{StatusCode: http.StatusNoContent, Body: &nopReadCloser{}}, nil 292 } 293 294 type nopReadCloser struct{} 295 296 func (n *nopReadCloser) Read(p []byte) (int, error) { return 0, io.EOF } 297 func (n *nopReadCloser) Close() error { return nil } 298 299 func startTestPipeline(tr *Transport, picker *urlPicker) *pipeline { 300 p := &pipeline{ 301 peerID: types.ID(1), 302 tr: tr, 303 picker: picker, 304 status: newPeerStatus(types.ID(1)), 305 raft: &fakeRaft{}, 306 followerStats: &stats.FollowerStats{}, 307 errorc: make(chan error, 1), 308 } 309 p.start() 310 return p 311 }