k8s.io/apiserver@v0.31.1/pkg/util/proxy/streamtranslator_test.go (about) 1 /* 2 Copyright 2023 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package proxy 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/rand" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io" 27 "math" 28 mrand "math/rand" 29 "net/http" 30 "net/http/httptest" 31 "net/url" 32 "reflect" 33 "strings" 34 "testing" 35 "time" 36 37 v1 "k8s.io/api/core/v1" 38 apierrors "k8s.io/apimachinery/pkg/api/errors" 39 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 "k8s.io/apimachinery/pkg/util/httpstream" 41 "k8s.io/apimachinery/pkg/util/httpstream/spdy" 42 rcconstants "k8s.io/apimachinery/pkg/util/remotecommand" 43 "k8s.io/apimachinery/pkg/util/wait" 44 "k8s.io/apiserver/pkg/util/proxy/metrics" 45 "k8s.io/client-go/rest" 46 "k8s.io/client-go/tools/remotecommand" 47 "k8s.io/client-go/transport" 48 "k8s.io/component-base/metrics/legacyregistry" 49 "k8s.io/component-base/metrics/testutil" 50 ) 51 52 // TestStreamTranslator_LoopbackStdinToStdout returns random data sent on the client's 53 // STDIN channel back onto the client's STDOUT channel. There are two servers in this test: the 54 // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the 55 // data received from the websocket client upstream to the SPDY server (by translating the 56 // websocket data into spdy). The returned data read on the websocket client STDOUT is then 57 // compared the random data sent on STDIN to ensure they are the same. 58 func TestStreamTranslator_LoopbackStdinToStdout(t *testing.T) { 59 metrics.Register() 60 metrics.ResetForTest() 61 t.Cleanup(metrics.ResetForTest) 62 // Create upstream fake SPDY server which copies STDIN back onto STDOUT stream. 63 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 64 ctx, err := createSPDYServerStreams(w, req, Options{ 65 Stdin: true, 66 Stdout: true, 67 }) 68 if err != nil { 69 t.Errorf("error on createHTTPStreams: %v", err) 70 return 71 } 72 defer ctx.conn.Close() 73 // Loopback STDIN data onto STDOUT stream. 74 _, err = io.Copy(ctx.stdoutStream, ctx.stdinStream) 75 if err != nil { 76 t.Fatalf("error copying STDIN to STDOUT: %v", err) 77 } 78 79 })) 80 defer spdyServer.Close() 81 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with 82 // streams STDIN and STDOUT. Create test server from StreamTranslatorHandler. 83 spdyLocation, err := url.Parse(spdyServer.URL) 84 if err != nil { 85 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 86 } 87 spdyTransport, err := fakeTransport() 88 if err != nil { 89 t.Fatalf("Unexpected error creating transport: %v", err) 90 } 91 streams := Options{Stdin: true, Stdout: true} 92 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 93 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 94 streamTranslator.ServeHTTP(w, req) 95 })) 96 defer streamTranslatorServer.Close() 97 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 98 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 99 if err != nil { 100 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 101 } 102 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 103 if err != nil { 104 t.Errorf("unexpected error creating websocket executor: %v", err) 105 } 106 // Generate random data, and set it up to stream on STDIN. The data will be 107 // returned on the STDOUT buffer. 108 randomSize := 1024 * 1024 109 randomData := make([]byte, randomSize) 110 if _, err := rand.Read(randomData); err != nil { 111 t.Errorf("unexpected error reading random data: %v", err) 112 } 113 var stdout bytes.Buffer 114 options := &remotecommand.StreamOptions{ 115 Stdin: bytes.NewReader(randomData), 116 Stdout: &stdout, 117 } 118 errorChan := make(chan error) 119 go func() { 120 // Start the streaming on the WebSocket "exec" client. 121 errorChan <- exec.StreamWithContext(context.Background(), *options) 122 }() 123 124 select { 125 case <-time.After(wait.ForeverTestTimeout): 126 t.Fatalf("expect stream to be closed after connection is closed.") 127 case err := <-errorChan: 128 if err != nil { 129 t.Errorf("unexpected error: %v", err) 130 } 131 } 132 data, err := io.ReadAll(bytes.NewReader(stdout.Bytes())) 133 if err != nil { 134 t.Errorf("error reading the stream: %v", err) 135 return 136 } 137 // Check the random data sent on STDIN was the same returned on STDOUT. 138 if !bytes.Equal(randomData, data) { 139 t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData)) 140 } 141 // Validate the streamtranslator metrics; should be one 200 success. 142 metricNames := []string{"apiserver_stream_translator_requests_total"} 143 expected := ` 144 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 145 # TYPE apiserver_stream_translator_requests_total counter 146 apiserver_stream_translator_requests_total{code="200"} 1 147 ` 148 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 149 t.Fatal(err) 150 } 151 } 152 153 // TestStreamTranslator_LoopbackStdinToStderr returns random data sent on the client's 154 // STDIN channel back onto the client's STDERR channel. There are two servers in this test: the 155 // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the 156 // data received from the websocket client upstream to the SPDY server (by translating the 157 // websocket data into spdy). The returned data read on the websocket client STDERR is then 158 // compared the random data sent on STDIN to ensure they are the same. 159 func TestStreamTranslator_LoopbackStdinToStderr(t *testing.T) { 160 metrics.Register() 161 metrics.ResetForTest() 162 t.Cleanup(metrics.ResetForTest) 163 // Create upstream fake SPDY server which copies STDIN back onto STDERR stream. 164 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 165 ctx, err := createSPDYServerStreams(w, req, Options{ 166 Stdin: true, 167 Stderr: true, 168 }) 169 if err != nil { 170 t.Errorf("error on createHTTPStreams: %v", err) 171 return 172 } 173 defer ctx.conn.Close() 174 // Loopback STDIN data onto STDERR stream. 175 _, err = io.Copy(ctx.stderrStream, ctx.stdinStream) 176 if err != nil { 177 t.Fatalf("error copying STDIN to STDERR: %v", err) 178 } 179 })) 180 defer spdyServer.Close() 181 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with 182 // streams STDIN and STDERR. Create test server from StreamTranslatorHandler. 183 spdyLocation, err := url.Parse(spdyServer.URL) 184 if err != nil { 185 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 186 } 187 spdyTransport, err := fakeTransport() 188 if err != nil { 189 t.Fatalf("Unexpected error creating transport: %v", err) 190 } 191 streams := Options{Stdin: true, Stderr: true} 192 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 193 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 194 streamTranslator.ServeHTTP(w, req) 195 })) 196 defer streamTranslatorServer.Close() 197 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 198 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 199 if err != nil { 200 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 201 } 202 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 203 if err != nil { 204 t.Errorf("unexpected error creating websocket executor: %v", err) 205 } 206 // Generate random data, and set it up to stream on STDIN. The data will be 207 // returned on the STDERR buffer. 208 randomSize := 1024 * 1024 209 randomData := make([]byte, randomSize) 210 if _, err := rand.Read(randomData); err != nil { 211 t.Errorf("unexpected error reading random data: %v", err) 212 } 213 var stderr bytes.Buffer 214 options := &remotecommand.StreamOptions{ 215 Stdin: bytes.NewReader(randomData), 216 Stderr: &stderr, 217 } 218 errorChan := make(chan error) 219 go func() { 220 // Start the streaming on the WebSocket "exec" client. 221 errorChan <- exec.StreamWithContext(context.Background(), *options) 222 }() 223 224 select { 225 case <-time.After(wait.ForeverTestTimeout): 226 t.Fatalf("expect stream to be closed after connection is closed.") 227 case err := <-errorChan: 228 if err != nil { 229 t.Errorf("unexpected error: %v", err) 230 } 231 } 232 data, err := io.ReadAll(bytes.NewReader(stderr.Bytes())) 233 if err != nil { 234 t.Errorf("error reading the stream: %v", err) 235 return 236 } 237 // Check the random data sent on STDIN was the same returned on STDERR. 238 if !bytes.Equal(randomData, data) { 239 t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData)) 240 } 241 // Validate the streamtranslator metrics; should be one 200 success. 242 metricNames := []string{"apiserver_stream_translator_requests_total"} 243 expected := ` 244 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 245 # TYPE apiserver_stream_translator_requests_total counter 246 apiserver_stream_translator_requests_total{code="200"} 1 247 ` 248 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 249 t.Fatal(err) 250 } 251 } 252 253 // Returns a random exit code in the range(1-127). 254 func randomExitCode() int { 255 errorCode := mrand.Intn(127) // Range: (0 - 126) 256 errorCode += 1 // Range: (1 - 127) 257 return errorCode 258 } 259 260 // TestStreamTranslator_ErrorStream tests the error stream by sending an error with a random 261 // exit code, then validating the error arrives on the error stream. 262 func TestStreamTranslator_ErrorStream(t *testing.T) { 263 metrics.Register() 264 metrics.ResetForTest() 265 t.Cleanup(metrics.ResetForTest) 266 expectedExitCode := randomExitCode() 267 // Create upstream fake SPDY server, returning a non-zero exit code 268 // on error stream within the structured error. 269 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 270 ctx, err := createSPDYServerStreams(w, req, Options{ 271 Stdout: true, 272 }) 273 if err != nil { 274 t.Errorf("error on createHTTPStreams: %v", err) 275 return 276 } 277 defer ctx.conn.Close() 278 // Read/discard STDIN data before returning error on error stream. 279 _, err = io.Copy(io.Discard, ctx.stdinStream) 280 if err != nil { 281 t.Fatalf("error copying STDIN to DISCARD: %v", err) 282 } 283 // Force an non-zero exit code error returned on the error stream. 284 err = ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{ 285 Status: metav1.StatusFailure, 286 Reason: rcconstants.NonZeroExitCodeReason, 287 Details: &metav1.StatusDetails{ 288 Causes: []metav1.StatusCause{ 289 { 290 Type: rcconstants.ExitCodeCauseType, 291 Message: fmt.Sprintf("%d", expectedExitCode), 292 }, 293 }, 294 }, 295 }}) 296 if err != nil { 297 t.Fatalf("error writing status: %v", err) 298 } 299 })) 300 defer spdyServer.Close() 301 // Create StreamTranslatorHandler, which points upstream to fake SPDY server, and 302 // create a test server using the StreamTranslatorHandler. 303 spdyLocation, err := url.Parse(spdyServer.URL) 304 if err != nil { 305 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 306 } 307 spdyTransport, err := fakeTransport() 308 if err != nil { 309 t.Fatalf("Unexpected error creating transport: %v", err) 310 } 311 streams := Options{Stdin: true} 312 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 313 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 314 streamTranslator.ServeHTTP(w, req) 315 })) 316 defer streamTranslatorServer.Close() 317 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 318 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 319 if err != nil { 320 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 321 } 322 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 323 if err != nil { 324 t.Errorf("unexpected error creating websocket executor: %v", err) 325 } 326 // Generate random data, and set it up to stream on STDIN. The data will be discarded at 327 // upstream SDPY server. 328 randomSize := 1024 * 1024 329 randomData := make([]byte, randomSize) 330 if _, err := rand.Read(randomData); err != nil { 331 t.Errorf("unexpected error reading random data: %v", err) 332 } 333 options := &remotecommand.StreamOptions{ 334 Stdin: bytes.NewReader(randomData), 335 } 336 errorChan := make(chan error) 337 go func() { 338 // Start the streaming on the WebSocket "exec" client. 339 errorChan <- exec.StreamWithContext(context.Background(), *options) 340 }() 341 342 select { 343 case <-time.After(wait.ForeverTestTimeout): 344 t.Fatalf("expect stream to be closed after connection is closed.") 345 case err := <-errorChan: 346 // Expect exit code error on error stream. 347 if err == nil { 348 t.Errorf("expected error, but received none") 349 } 350 expectedError := fmt.Sprintf("command terminated with exit code %d", expectedExitCode) 351 // Compare expected error with exit code to actual error. 352 if expectedError != err.Error() { 353 t.Errorf("expected error (%s), got (%s)", expectedError, err) 354 } 355 } 356 // Validate the streamtranslator metrics; an exit code error is considered 200 success. 357 metricNames := []string{"apiserver_stream_translator_requests_total"} 358 expected := ` 359 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 360 # TYPE apiserver_stream_translator_requests_total counter 361 apiserver_stream_translator_requests_total{code="200"} 1 362 ` 363 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 364 t.Fatal(err) 365 } 366 } 367 368 // TestStreamTranslator_MultipleReadChannels tests two streams (STDOUT, STDERR) reading from 369 // the connections at the same time. 370 func TestStreamTranslator_MultipleReadChannels(t *testing.T) { 371 metrics.Register() 372 metrics.ResetForTest() 373 t.Cleanup(metrics.ResetForTest) 374 // Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream. 375 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 376 ctx, err := createSPDYServerStreams(w, req, Options{ 377 Stdin: true, 378 Stdout: true, 379 Stderr: true, 380 }) 381 if err != nil { 382 t.Errorf("error on createHTTPStreams: %v", err) 383 return 384 } 385 defer ctx.conn.Close() 386 // TeeReader copies data read on STDIN onto STDERR. 387 stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream) 388 // Also copy STDIN to STDOUT. 389 _, err = io.Copy(ctx.stdoutStream, stdinReader) 390 if err != nil { 391 t.Errorf("error copying STDIN to STDOUT: %v", err) 392 } 393 })) 394 defer spdyServer.Close() 395 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with 396 // streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler. 397 spdyLocation, err := url.Parse(spdyServer.URL) 398 if err != nil { 399 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 400 } 401 spdyTransport, err := fakeTransport() 402 if err != nil { 403 t.Fatalf("Unexpected error creating transport: %v", err) 404 } 405 streams := Options{Stdin: true, Stdout: true, Stderr: true} 406 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 407 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 408 streamTranslator.ServeHTTP(w, req) 409 })) 410 defer streamTranslatorServer.Close() 411 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 412 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 413 if err != nil { 414 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 415 } 416 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 417 if err != nil { 418 t.Errorf("unexpected error creating websocket executor: %v", err) 419 } 420 // Generate random data, and set it up to stream on STDIN. The data will be 421 // returned on the STDOUT and STDERR buffer. 422 randomSize := 1024 * 1024 423 randomData := make([]byte, randomSize) 424 if _, err := rand.Read(randomData); err != nil { 425 t.Errorf("unexpected error reading random data: %v", err) 426 } 427 var stdout, stderr bytes.Buffer 428 options := &remotecommand.StreamOptions{ 429 Stdin: bytes.NewReader(randomData), 430 Stdout: &stdout, 431 Stderr: &stderr, 432 } 433 errorChan := make(chan error) 434 go func() { 435 // Start the streaming on the WebSocket "exec" client. 436 errorChan <- exec.StreamWithContext(context.Background(), *options) 437 }() 438 439 select { 440 case <-time.After(wait.ForeverTestTimeout): 441 t.Fatalf("expect stream to be closed after connection is closed.") 442 case err := <-errorChan: 443 if err != nil { 444 t.Errorf("unexpected error: %v", err) 445 } 446 } 447 stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes())) 448 if err != nil { 449 t.Errorf("error reading the stream: %v", err) 450 return 451 } 452 // Check the random data sent on STDIN was the same returned on STDOUT. 453 if !bytes.Equal(stdoutBytes, randomData) { 454 t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData)) 455 } 456 stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes())) 457 if err != nil { 458 t.Errorf("error reading the stream: %v", err) 459 return 460 } 461 // Check the random data sent on STDIN was the same returned on STDERR. 462 if !bytes.Equal(stderrBytes, randomData) { 463 t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData)) 464 } 465 // Validate the streamtranslator metrics; should have one 200 success. 466 metricNames := []string{"apiserver_stream_translator_requests_total"} 467 expected := ` 468 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 469 # TYPE apiserver_stream_translator_requests_total counter 470 apiserver_stream_translator_requests_total{code="200"} 1 471 ` 472 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 473 t.Fatal(err) 474 } 475 } 476 477 // TestStreamTranslator_ThrottleReadChannels tests two streams (STDOUT, STDERR) using rate limited streams. 478 func TestStreamTranslator_ThrottleReadChannels(t *testing.T) { 479 // Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream. 480 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 481 ctx, err := createSPDYServerStreams(w, req, Options{ 482 Stdin: true, 483 Stdout: true, 484 Stderr: true, 485 }) 486 if err != nil { 487 t.Errorf("error on createHTTPStreams: %v", err) 488 return 489 } 490 defer ctx.conn.Close() 491 // TeeReader copies data read on STDIN onto STDERR. 492 stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream) 493 // Also copy STDIN to STDOUT. 494 _, err = io.Copy(ctx.stdoutStream, stdinReader) 495 if err != nil { 496 t.Errorf("error copying STDIN to STDOUT: %v", err) 497 } 498 })) 499 defer spdyServer.Close() 500 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with 501 // streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler. 502 spdyLocation, err := url.Parse(spdyServer.URL) 503 if err != nil { 504 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 505 } 506 spdyTransport, err := fakeTransport() 507 if err != nil { 508 t.Fatalf("Unexpected error creating transport: %v", err) 509 } 510 streams := Options{Stdin: true, Stdout: true, Stderr: true} 511 maxBytesPerSec := 900 * 1024 // slightly less than the 1MB that is being transferred to exercise throttling. 512 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, int64(maxBytesPerSec), streams) 513 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 514 streamTranslator.ServeHTTP(w, req) 515 })) 516 defer streamTranslatorServer.Close() 517 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 518 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 519 if err != nil { 520 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 521 } 522 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 523 if err != nil { 524 t.Errorf("unexpected error creating websocket executor: %v", err) 525 } 526 // Generate random data, and set it up to stream on STDIN. The data will be 527 // returned on the STDOUT and STDERR buffer. 528 randomSize := 1024 * 1024 529 randomData := make([]byte, randomSize) 530 if _, err := rand.Read(randomData); err != nil { 531 t.Errorf("unexpected error reading random data: %v", err) 532 } 533 var stdout, stderr bytes.Buffer 534 options := &remotecommand.StreamOptions{ 535 Stdin: bytes.NewReader(randomData), 536 Stdout: &stdout, 537 Stderr: &stderr, 538 } 539 errorChan := make(chan error) 540 go func() { 541 // Start the streaming on the WebSocket "exec" client. 542 errorChan <- exec.StreamWithContext(context.Background(), *options) 543 }() 544 545 select { 546 case <-time.After(wait.ForeverTestTimeout): 547 t.Fatalf("expect stream to be closed after connection is closed.") 548 case err := <-errorChan: 549 if err != nil { 550 t.Errorf("unexpected error: %v", err) 551 } 552 } 553 stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes())) 554 if err != nil { 555 t.Errorf("error reading the stream: %v", err) 556 return 557 } 558 // Check the random data sent on STDIN was the same returned on STDOUT. 559 if !bytes.Equal(stdoutBytes, randomData) { 560 t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData)) 561 } 562 stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes())) 563 if err != nil { 564 t.Errorf("error reading the stream: %v", err) 565 return 566 } 567 // Check the random data sent on STDIN was the same returned on STDERR. 568 if !bytes.Equal(stderrBytes, randomData) { 569 t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData)) 570 } 571 } 572 573 // fakeTerminalSizeQueue implements TerminalSizeQueue, returning a random set of 574 // "maxSizes" number of TerminalSizes, storing the TerminalSizes in "sizes" slice. 575 type fakeTerminalSizeQueue struct { 576 maxSizes int 577 terminalSizes []remotecommand.TerminalSize 578 } 579 580 // newTerminalSizeQueue returns a pointer to a fakeTerminalSizeQueue passing 581 // "max" number of random TerminalSizes created. 582 func newTerminalSizeQueue(max int) *fakeTerminalSizeQueue { 583 return &fakeTerminalSizeQueue{ 584 maxSizes: max, 585 terminalSizes: make([]remotecommand.TerminalSize, 0, max), 586 } 587 } 588 589 // Next returns a pointer to the next random TerminalSize, or nil if we have 590 // already returned "maxSizes" TerminalSizes already. Stores the randomly 591 // created TerminalSize in "terminalSizes" field for later validation. 592 func (f *fakeTerminalSizeQueue) Next() *remotecommand.TerminalSize { 593 if len(f.terminalSizes) >= f.maxSizes { 594 return nil 595 } 596 size := randomTerminalSize() 597 f.terminalSizes = append(f.terminalSizes, size) 598 return &size 599 } 600 601 // randomTerminalSize returns a TerminalSize with random values in the 602 // range (0-65535) for the fields Width and Height. 603 func randomTerminalSize() remotecommand.TerminalSize { 604 randWidth := uint16(mrand.Intn(int(math.Pow(2, 16)))) 605 randHeight := uint16(mrand.Intn(int(math.Pow(2, 16)))) 606 return remotecommand.TerminalSize{ 607 Width: randWidth, 608 Height: randHeight, 609 } 610 } 611 612 // TestStreamTranslator_MultipleWriteChannels 613 func TestStreamTranslator_TTYResizeChannel(t *testing.T) { 614 metrics.Register() 615 metrics.ResetForTest() 616 t.Cleanup(metrics.ResetForTest) 617 // Create the fake terminal size queue and the actualTerminalSizes which 618 // will be received at the opposite websocket endpoint. 619 numSizeQueue := 10000 620 sizeQueue := newTerminalSizeQueue(numSizeQueue) 621 actualTerminalSizes := make([]remotecommand.TerminalSize, 0, numSizeQueue) 622 // Create upstream fake SPDY server which copies STDIN back onto STDERR stream. 623 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 624 ctx, err := createSPDYServerStreams(w, req, Options{ 625 Tty: true, 626 }) 627 if err != nil { 628 t.Errorf("error on createHTTPStreams: %v", err) 629 return 630 } 631 defer ctx.conn.Close() 632 // Read the terminal resize requests, storing them in actualTerminalSizes 633 for i := 0; i < numSizeQueue; i++ { 634 actualTerminalSize := <-ctx.resizeChan 635 actualTerminalSizes = append(actualTerminalSizes, actualTerminalSize) 636 } 637 })) 638 defer spdyServer.Close() 639 // Create StreamTranslatorHandler, which points upstream to fake SPDY server with 640 // resize (TTY resize) stream. Create test server from StreamTranslatorHandler. 641 spdyLocation, err := url.Parse(spdyServer.URL) 642 if err != nil { 643 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 644 } 645 spdyTransport, err := fakeTransport() 646 if err != nil { 647 t.Fatalf("Unexpected error creating transport: %v", err) 648 } 649 streams := Options{Tty: true} 650 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 651 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 652 streamTranslator.ServeHTTP(w, req) 653 })) 654 defer streamTranslatorServer.Close() 655 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 656 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 657 if err != nil { 658 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 659 } 660 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 661 if err != nil { 662 t.Errorf("unexpected error creating websocket executor: %v", err) 663 } 664 options := &remotecommand.StreamOptions{ 665 Tty: true, 666 TerminalSizeQueue: sizeQueue, 667 } 668 errorChan := make(chan error) 669 go func() { 670 // Start the streaming on the WebSocket "exec" client. 671 errorChan <- exec.StreamWithContext(context.Background(), *options) 672 }() 673 674 select { 675 case <-time.After(wait.ForeverTestTimeout): 676 t.Fatalf("expect stream to be closed after connection is closed.") 677 case err := <-errorChan: 678 if err != nil { 679 t.Errorf("unexpected error: %v", err) 680 } 681 } 682 // Validate the random TerminalSizes sent on the resize stream are the same 683 // as the actual TerminalSizes received at the websocket server. 684 if len(actualTerminalSizes) != numSizeQueue { 685 t.Fatalf("expected to receive num terminal resizes (%d), got (%d)", 686 numSizeQueue, len(actualTerminalSizes)) 687 } 688 for i, actual := range actualTerminalSizes { 689 expected := sizeQueue.terminalSizes[i] 690 if !reflect.DeepEqual(expected, actual) { 691 t.Errorf("expected terminal resize window %v, got %v", expected, actual) 692 } 693 } 694 // Validate the streamtranslator metrics; should have one 200 success. 695 metricNames := []string{"apiserver_stream_translator_requests_total"} 696 expected := ` 697 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 698 # TYPE apiserver_stream_translator_requests_total counter 699 apiserver_stream_translator_requests_total{code="200"} 1 700 ` 701 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 702 t.Fatal(err) 703 } 704 } 705 706 // TestStreamTranslator_WebSocketServerErrors validates that when there is a problem creating 707 // the websocket server as the first step of the StreamTranslator an error is properly returned. 708 func TestStreamTranslator_WebSocketServerErrors(t *testing.T) { 709 metrics.Register() 710 metrics.ResetForTest() 711 t.Cleanup(metrics.ResetForTest) 712 spdyLocation, err := url.Parse("http://127.0.0.1") 713 if err != nil { 714 t.Fatalf("Unable to parse spdy server URL") 715 } 716 spdyTransport, err := fakeTransport() 717 if err != nil { 718 t.Fatalf("Unexpected error creating transport: %v", err) 719 } 720 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, Options{}) 721 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 722 streamTranslator.ServeHTTP(w, req) 723 })) 724 defer streamTranslatorServer.Close() 725 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 726 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 727 if err != nil { 728 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 729 } 730 exec, err := remotecommand.NewWebSocketExecutorForProtocols( 731 &rest.Config{Host: streamTranslatorLocation.Host}, 732 "GET", 733 streamTranslatorServer.URL, 734 rcconstants.StreamProtocolV4Name, // RemoteCommand V4 protocol is unsupported 735 ) 736 if err != nil { 737 t.Errorf("unexpected error creating websocket executor: %v", err) 738 } 739 errorChan := make(chan error) 740 go func() { 741 // Start the streaming on the WebSocket "exec" client. The WebSocket server within the 742 // StreamTranslator propagates an error here because the V4 protocol is not supported. 743 errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{}) 744 }() 745 746 select { 747 case <-time.After(wait.ForeverTestTimeout): 748 t.Fatalf("expect stream to be closed after connection is closed.") 749 case err := <-errorChan: 750 // Must return "websocket unable to upgrade" (bad handshake) error. 751 if err == nil { 752 t.Fatalf("expected error, but received none") 753 } 754 if !strings.Contains(err.Error(), "unable to upgrade streaming request") { 755 t.Errorf("expected websocket bad handshake error, got (%s)", err) 756 } 757 } 758 // Validate the streamtranslator metrics; should have one 500 failure. 759 metricNames := []string{"apiserver_stream_translator_requests_total"} 760 expected := ` 761 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 762 # TYPE apiserver_stream_translator_requests_total counter 763 apiserver_stream_translator_requests_total{code="400"} 1 764 ` 765 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 766 t.Fatal(err) 767 } 768 } 769 770 // TestStreamTranslator_BlockRedirects verifies that the StreamTranslator will *not* follow 771 // redirects; it will thrown an error instead. 772 func TestStreamTranslator_BlockRedirects(t *testing.T) { 773 metrics.Register() 774 metrics.ResetForTest() 775 t.Cleanup(metrics.ResetForTest) 776 for _, statusCode := range []int{ 777 http.StatusMovedPermanently, // 301 778 http.StatusFound, // 302 779 http.StatusSeeOther, // 303 780 http.StatusTemporaryRedirect, // 307 781 http.StatusPermanentRedirect, // 308 782 } { 783 // Create upstream fake SPDY server which returns a redirect. 784 spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 785 w.Header().Set("Location", "/") 786 w.WriteHeader(statusCode) 787 })) 788 defer spdyServer.Close() 789 spdyLocation, err := url.Parse(spdyServer.URL) 790 if err != nil { 791 t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL) 792 } 793 spdyTransport, err := fakeTransport() 794 if err != nil { 795 t.Fatalf("Unexpected error creating transport: %v", err) 796 } 797 streams := Options{Stdout: true} 798 streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams) 799 streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 800 streamTranslator.ServeHTTP(w, req) 801 })) 802 defer streamTranslatorServer.Close() 803 // Now create the websocket client (executor), and point it to the "streamTranslatorServer". 804 streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL) 805 if err != nil { 806 t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL) 807 } 808 exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL) 809 if err != nil { 810 t.Errorf("unexpected error creating websocket executor: %v", err) 811 } 812 errorChan := make(chan error) 813 go func() { 814 // Start the streaming on the WebSocket "exec" client. 815 // Should return "redirect not allowed" error. 816 errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{}) 817 }() 818 819 select { 820 case <-time.After(wait.ForeverTestTimeout): 821 t.Fatalf("expect stream to be closed after connection is closed.") 822 case err := <-errorChan: 823 // Must return "redirect now allowed" error. 824 if err == nil { 825 t.Fatalf("expected error, but received none") 826 } 827 if !strings.Contains(err.Error(), "redirect not allowed") { 828 t.Errorf("expected redirect not allowed error, got (%s)", err) 829 } 830 } 831 // Validate the streamtranslator metrics; should have one 500 failure each loop. 832 metricNames := []string{"apiserver_stream_translator_requests_total"} 833 expected := ` 834 # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5 835 # TYPE apiserver_stream_translator_requests_total counter 836 apiserver_stream_translator_requests_total{code="500"} 1 837 ` 838 if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil { 839 t.Fatal(err) 840 } 841 metrics.ResetForTest() // Clear metrics each loop 842 } 843 } 844 845 // streamContext encapsulates the structures necessary to communicate through 846 // a SPDY connection, including the Reader/Writer streams. 847 type streamContext struct { 848 conn io.Closer 849 stdinStream io.ReadCloser 850 stdoutStream io.WriteCloser 851 stderrStream io.WriteCloser 852 resizeStream io.ReadCloser 853 resizeChan chan remotecommand.TerminalSize 854 writeStatus func(status *apierrors.StatusError) error 855 } 856 857 type streamAndReply struct { 858 httpstream.Stream 859 replySent <-chan struct{} 860 } 861 862 // CreateSPDYServerStreams upgrades the passed HTTP request to a SPDY bi-directional streaming 863 // connection with remote command streams defined in passed options. Returns a streamContext 864 // structure containing the Reader/Writer streams to communicate through the SDPY connection. 865 // Returns an error if unable to upgrade the HTTP connection to a SPDY connection. 866 func createSPDYServerStreams(w http.ResponseWriter, req *http.Request, opts Options) (*streamContext, error) { 867 _, err := httpstream.Handshake(req, w, []string{rcconstants.StreamProtocolV4Name}) 868 if err != nil { 869 return nil, err 870 } 871 872 upgrader := spdy.NewResponseUpgrader() 873 streamCh := make(chan streamAndReply) 874 conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream, replySent <-chan struct{}) error { 875 streamCh <- streamAndReply{Stream: stream, replySent: replySent} 876 return nil 877 }) 878 ctx := &streamContext{ 879 conn: conn, 880 } 881 882 // wait for stream 883 replyChan := make(chan struct{}, 5) 884 defer close(replyChan) 885 receivedStreams := 0 886 expectedStreams := 1 // expect at least the error stream 887 if opts.Stdout { 888 expectedStreams++ 889 } 890 if opts.Stdin { 891 expectedStreams++ 892 } 893 if opts.Stderr { 894 expectedStreams++ 895 } 896 if opts.Tty { 897 expectedStreams++ 898 } 899 WaitForStreams: 900 for { 901 select { 902 case stream := <-streamCh: 903 streamType := stream.Headers().Get(v1.StreamType) 904 switch streamType { 905 case v1.StreamTypeError: 906 replyChan <- struct{}{} 907 ctx.writeStatus = v4WriteStatusFunc(stream) 908 case v1.StreamTypeStdout: 909 replyChan <- struct{}{} 910 ctx.stdoutStream = stream 911 case v1.StreamTypeStdin: 912 replyChan <- struct{}{} 913 ctx.stdinStream = stream 914 case v1.StreamTypeStderr: 915 replyChan <- struct{}{} 916 ctx.stderrStream = stream 917 case v1.StreamTypeResize: 918 replyChan <- struct{}{} 919 ctx.resizeStream = stream 920 default: 921 // add other stream ... 922 return nil, errors.New("unimplemented stream type") 923 } 924 case <-replyChan: 925 receivedStreams++ 926 if receivedStreams == expectedStreams { 927 break WaitForStreams 928 } 929 } 930 } 931 932 if ctx.resizeStream != nil { 933 ctx.resizeChan = make(chan remotecommand.TerminalSize) 934 go handleResizeEvents(req.Context(), ctx.resizeStream, ctx.resizeChan) 935 } 936 937 return ctx, nil 938 } 939 940 func v4WriteStatusFunc(stream io.Writer) func(status *apierrors.StatusError) error { 941 return func(status *apierrors.StatusError) error { 942 bs, err := json.Marshal(status.Status()) 943 if err != nil { 944 return err 945 } 946 _, err = stream.Write(bs) 947 return err 948 } 949 } 950 951 func fakeTransport() (*http.Transport, error) { 952 cfg := &transport.Config{ 953 TLS: transport.TLSConfig{ 954 Insecure: true, 955 CAFile: "", 956 }, 957 } 958 rt, err := transport.New(cfg) 959 if err != nil { 960 return nil, err 961 } 962 t, ok := rt.(*http.Transport) 963 if !ok { 964 return nil, fmt.Errorf("unknown transport type: %T", rt) 965 } 966 return t, nil 967 }