k8s.io/apiserver@v0.31.1/pkg/util/proxy/streamtranslator_test.go (about)

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package proxy
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/rand"
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"io"
    27  	"math"
    28  	mrand "math/rand"
    29  	"net/http"
    30  	"net/http/httptest"
    31  	"net/url"
    32  	"reflect"
    33  	"strings"
    34  	"testing"
    35  	"time"
    36  
    37  	v1 "k8s.io/api/core/v1"
    38  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    39  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    40  	"k8s.io/apimachinery/pkg/util/httpstream"
    41  	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
    42  	rcconstants "k8s.io/apimachinery/pkg/util/remotecommand"
    43  	"k8s.io/apimachinery/pkg/util/wait"
    44  	"k8s.io/apiserver/pkg/util/proxy/metrics"
    45  	"k8s.io/client-go/rest"
    46  	"k8s.io/client-go/tools/remotecommand"
    47  	"k8s.io/client-go/transport"
    48  	"k8s.io/component-base/metrics/legacyregistry"
    49  	"k8s.io/component-base/metrics/testutil"
    50  )
    51  
    52  // TestStreamTranslator_LoopbackStdinToStdout returns random data sent on the client's
    53  // STDIN channel back onto the client's STDOUT channel. There are two servers in this test: the
    54  // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the
    55  // data received from the websocket client upstream to the SPDY server (by translating the
    56  // websocket data into spdy). The returned data read on the websocket client STDOUT is then
    57  // compared the random data sent on STDIN to ensure they are the same.
    58  func TestStreamTranslator_LoopbackStdinToStdout(t *testing.T) {
    59  	metrics.Register()
    60  	metrics.ResetForTest()
    61  	t.Cleanup(metrics.ResetForTest)
    62  	// Create upstream fake SPDY server which copies STDIN back onto STDOUT stream.
    63  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
    64  		ctx, err := createSPDYServerStreams(w, req, Options{
    65  			Stdin:  true,
    66  			Stdout: true,
    67  		})
    68  		if err != nil {
    69  			t.Errorf("error on createHTTPStreams: %v", err)
    70  			return
    71  		}
    72  		defer ctx.conn.Close()
    73  		// Loopback STDIN data onto STDOUT stream.
    74  		_, err = io.Copy(ctx.stdoutStream, ctx.stdinStream)
    75  		if err != nil {
    76  			t.Fatalf("error copying STDIN to STDOUT: %v", err)
    77  		}
    78  
    79  	}))
    80  	defer spdyServer.Close()
    81  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server with
    82  	// streams STDIN and STDOUT. Create test server from StreamTranslatorHandler.
    83  	spdyLocation, err := url.Parse(spdyServer.URL)
    84  	if err != nil {
    85  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
    86  	}
    87  	spdyTransport, err := fakeTransport()
    88  	if err != nil {
    89  		t.Fatalf("Unexpected error creating transport: %v", err)
    90  	}
    91  	streams := Options{Stdin: true, Stdout: true}
    92  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
    93  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
    94  		streamTranslator.ServeHTTP(w, req)
    95  	}))
    96  	defer streamTranslatorServer.Close()
    97  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
    98  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
    99  	if err != nil {
   100  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   101  	}
   102  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   103  	if err != nil {
   104  		t.Errorf("unexpected error creating websocket executor: %v", err)
   105  	}
   106  	// Generate random data, and set it up to stream on STDIN. The data will be
   107  	// returned on the STDOUT buffer.
   108  	randomSize := 1024 * 1024
   109  	randomData := make([]byte, randomSize)
   110  	if _, err := rand.Read(randomData); err != nil {
   111  		t.Errorf("unexpected error reading random data: %v", err)
   112  	}
   113  	var stdout bytes.Buffer
   114  	options := &remotecommand.StreamOptions{
   115  		Stdin:  bytes.NewReader(randomData),
   116  		Stdout: &stdout,
   117  	}
   118  	errorChan := make(chan error)
   119  	go func() {
   120  		// Start the streaming on the WebSocket "exec" client.
   121  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   122  	}()
   123  
   124  	select {
   125  	case <-time.After(wait.ForeverTestTimeout):
   126  		t.Fatalf("expect stream to be closed after connection is closed.")
   127  	case err := <-errorChan:
   128  		if err != nil {
   129  			t.Errorf("unexpected error: %v", err)
   130  		}
   131  	}
   132  	data, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
   133  	if err != nil {
   134  		t.Errorf("error reading the stream: %v", err)
   135  		return
   136  	}
   137  	// Check the random data sent on STDIN was the same returned on STDOUT.
   138  	if !bytes.Equal(randomData, data) {
   139  		t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData))
   140  	}
   141  	// Validate the streamtranslator metrics; should be one 200 success.
   142  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   143  	expected := `
   144  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   145  # TYPE apiserver_stream_translator_requests_total counter
   146  apiserver_stream_translator_requests_total{code="200"} 1
   147  `
   148  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   149  		t.Fatal(err)
   150  	}
   151  }
   152  
   153  // TestStreamTranslator_LoopbackStdinToStderr returns random data sent on the client's
   154  // STDIN channel back onto the client's STDERR channel. There are two servers in this test: the
   155  // upstream fake SPDY server, and the StreamTranslator server. The StreamTranslator proxys the
   156  // data received from the websocket client upstream to the SPDY server (by translating the
   157  // websocket data into spdy). The returned data read on the websocket client STDERR is then
   158  // compared the random data sent on STDIN to ensure they are the same.
   159  func TestStreamTranslator_LoopbackStdinToStderr(t *testing.T) {
   160  	metrics.Register()
   161  	metrics.ResetForTest()
   162  	t.Cleanup(metrics.ResetForTest)
   163  	// Create upstream fake SPDY server which copies STDIN back onto STDERR stream.
   164  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   165  		ctx, err := createSPDYServerStreams(w, req, Options{
   166  			Stdin:  true,
   167  			Stderr: true,
   168  		})
   169  		if err != nil {
   170  			t.Errorf("error on createHTTPStreams: %v", err)
   171  			return
   172  		}
   173  		defer ctx.conn.Close()
   174  		// Loopback STDIN data onto STDERR stream.
   175  		_, err = io.Copy(ctx.stderrStream, ctx.stdinStream)
   176  		if err != nil {
   177  			t.Fatalf("error copying STDIN to STDERR: %v", err)
   178  		}
   179  	}))
   180  	defer spdyServer.Close()
   181  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server with
   182  	// streams STDIN and STDERR. Create test server from StreamTranslatorHandler.
   183  	spdyLocation, err := url.Parse(spdyServer.URL)
   184  	if err != nil {
   185  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   186  	}
   187  	spdyTransport, err := fakeTransport()
   188  	if err != nil {
   189  		t.Fatalf("Unexpected error creating transport: %v", err)
   190  	}
   191  	streams := Options{Stdin: true, Stderr: true}
   192  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
   193  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   194  		streamTranslator.ServeHTTP(w, req)
   195  	}))
   196  	defer streamTranslatorServer.Close()
   197  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   198  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   199  	if err != nil {
   200  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   201  	}
   202  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   203  	if err != nil {
   204  		t.Errorf("unexpected error creating websocket executor: %v", err)
   205  	}
   206  	// Generate random data, and set it up to stream on STDIN. The data will be
   207  	// returned on the STDERR buffer.
   208  	randomSize := 1024 * 1024
   209  	randomData := make([]byte, randomSize)
   210  	if _, err := rand.Read(randomData); err != nil {
   211  		t.Errorf("unexpected error reading random data: %v", err)
   212  	}
   213  	var stderr bytes.Buffer
   214  	options := &remotecommand.StreamOptions{
   215  		Stdin:  bytes.NewReader(randomData),
   216  		Stderr: &stderr,
   217  	}
   218  	errorChan := make(chan error)
   219  	go func() {
   220  		// Start the streaming on the WebSocket "exec" client.
   221  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   222  	}()
   223  
   224  	select {
   225  	case <-time.After(wait.ForeverTestTimeout):
   226  		t.Fatalf("expect stream to be closed after connection is closed.")
   227  	case err := <-errorChan:
   228  		if err != nil {
   229  			t.Errorf("unexpected error: %v", err)
   230  		}
   231  	}
   232  	data, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
   233  	if err != nil {
   234  		t.Errorf("error reading the stream: %v", err)
   235  		return
   236  	}
   237  	// Check the random data sent on STDIN was the same returned on STDERR.
   238  	if !bytes.Equal(randomData, data) {
   239  		t.Errorf("unexpected data received: %d sent: %d", len(data), len(randomData))
   240  	}
   241  	// Validate the streamtranslator metrics; should be one 200 success.
   242  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   243  	expected := `
   244  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   245  # TYPE apiserver_stream_translator_requests_total counter
   246  apiserver_stream_translator_requests_total{code="200"} 1
   247  `
   248  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   249  		t.Fatal(err)
   250  	}
   251  }
   252  
   253  // Returns a random exit code in the range(1-127).
   254  func randomExitCode() int {
   255  	errorCode := mrand.Intn(127) // Range: (0 - 126)
   256  	errorCode += 1               // Range: (1 - 127)
   257  	return errorCode
   258  }
   259  
   260  // TestStreamTranslator_ErrorStream tests the error stream by sending an error with a random
   261  // exit code, then validating the error arrives on the error stream.
   262  func TestStreamTranslator_ErrorStream(t *testing.T) {
   263  	metrics.Register()
   264  	metrics.ResetForTest()
   265  	t.Cleanup(metrics.ResetForTest)
   266  	expectedExitCode := randomExitCode()
   267  	// Create upstream fake SPDY server, returning a non-zero exit code
   268  	// on error stream within the structured error.
   269  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   270  		ctx, err := createSPDYServerStreams(w, req, Options{
   271  			Stdout: true,
   272  		})
   273  		if err != nil {
   274  			t.Errorf("error on createHTTPStreams: %v", err)
   275  			return
   276  		}
   277  		defer ctx.conn.Close()
   278  		// Read/discard STDIN data before returning error on error stream.
   279  		_, err = io.Copy(io.Discard, ctx.stdinStream)
   280  		if err != nil {
   281  			t.Fatalf("error copying STDIN to DISCARD: %v", err)
   282  		}
   283  		// Force an non-zero exit code error returned on the error stream.
   284  		err = ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
   285  			Status: metav1.StatusFailure,
   286  			Reason: rcconstants.NonZeroExitCodeReason,
   287  			Details: &metav1.StatusDetails{
   288  				Causes: []metav1.StatusCause{
   289  					{
   290  						Type:    rcconstants.ExitCodeCauseType,
   291  						Message: fmt.Sprintf("%d", expectedExitCode),
   292  					},
   293  				},
   294  			},
   295  		}})
   296  		if err != nil {
   297  			t.Fatalf("error writing status: %v", err)
   298  		}
   299  	}))
   300  	defer spdyServer.Close()
   301  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server, and
   302  	// create a test server using the  StreamTranslatorHandler.
   303  	spdyLocation, err := url.Parse(spdyServer.URL)
   304  	if err != nil {
   305  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   306  	}
   307  	spdyTransport, err := fakeTransport()
   308  	if err != nil {
   309  		t.Fatalf("Unexpected error creating transport: %v", err)
   310  	}
   311  	streams := Options{Stdin: true}
   312  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
   313  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   314  		streamTranslator.ServeHTTP(w, req)
   315  	}))
   316  	defer streamTranslatorServer.Close()
   317  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   318  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   319  	if err != nil {
   320  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   321  	}
   322  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   323  	if err != nil {
   324  		t.Errorf("unexpected error creating websocket executor: %v", err)
   325  	}
   326  	// Generate random data, and set it up to stream on STDIN. The data will be discarded at
   327  	// upstream SDPY server.
   328  	randomSize := 1024 * 1024
   329  	randomData := make([]byte, randomSize)
   330  	if _, err := rand.Read(randomData); err != nil {
   331  		t.Errorf("unexpected error reading random data: %v", err)
   332  	}
   333  	options := &remotecommand.StreamOptions{
   334  		Stdin: bytes.NewReader(randomData),
   335  	}
   336  	errorChan := make(chan error)
   337  	go func() {
   338  		// Start the streaming on the WebSocket "exec" client.
   339  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   340  	}()
   341  
   342  	select {
   343  	case <-time.After(wait.ForeverTestTimeout):
   344  		t.Fatalf("expect stream to be closed after connection is closed.")
   345  	case err := <-errorChan:
   346  		// Expect exit code error on error stream.
   347  		if err == nil {
   348  			t.Errorf("expected error, but received none")
   349  		}
   350  		expectedError := fmt.Sprintf("command terminated with exit code %d", expectedExitCode)
   351  		// Compare expected error with exit code to actual error.
   352  		if expectedError != err.Error() {
   353  			t.Errorf("expected error (%s), got (%s)", expectedError, err)
   354  		}
   355  	}
   356  	// Validate the streamtranslator metrics; an exit code error is considered 200 success.
   357  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   358  	expected := `
   359  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   360  # TYPE apiserver_stream_translator_requests_total counter
   361  apiserver_stream_translator_requests_total{code="200"} 1
   362  `
   363  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   364  		t.Fatal(err)
   365  	}
   366  }
   367  
   368  // TestStreamTranslator_MultipleReadChannels tests two streams (STDOUT, STDERR) reading from
   369  // the connections at the same time.
   370  func TestStreamTranslator_MultipleReadChannels(t *testing.T) {
   371  	metrics.Register()
   372  	metrics.ResetForTest()
   373  	t.Cleanup(metrics.ResetForTest)
   374  	// Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream.
   375  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   376  		ctx, err := createSPDYServerStreams(w, req, Options{
   377  			Stdin:  true,
   378  			Stdout: true,
   379  			Stderr: true,
   380  		})
   381  		if err != nil {
   382  			t.Errorf("error on createHTTPStreams: %v", err)
   383  			return
   384  		}
   385  		defer ctx.conn.Close()
   386  		// TeeReader copies data read on STDIN onto STDERR.
   387  		stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream)
   388  		// Also copy STDIN to STDOUT.
   389  		_, err = io.Copy(ctx.stdoutStream, stdinReader)
   390  		if err != nil {
   391  			t.Errorf("error copying STDIN to STDOUT: %v", err)
   392  		}
   393  	}))
   394  	defer spdyServer.Close()
   395  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server with
   396  	// streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler.
   397  	spdyLocation, err := url.Parse(spdyServer.URL)
   398  	if err != nil {
   399  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   400  	}
   401  	spdyTransport, err := fakeTransport()
   402  	if err != nil {
   403  		t.Fatalf("Unexpected error creating transport: %v", err)
   404  	}
   405  	streams := Options{Stdin: true, Stdout: true, Stderr: true}
   406  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
   407  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   408  		streamTranslator.ServeHTTP(w, req)
   409  	}))
   410  	defer streamTranslatorServer.Close()
   411  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   412  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   413  	if err != nil {
   414  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   415  	}
   416  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   417  	if err != nil {
   418  		t.Errorf("unexpected error creating websocket executor: %v", err)
   419  	}
   420  	// Generate random data, and set it up to stream on STDIN. The data will be
   421  	// returned on the STDOUT and STDERR buffer.
   422  	randomSize := 1024 * 1024
   423  	randomData := make([]byte, randomSize)
   424  	if _, err := rand.Read(randomData); err != nil {
   425  		t.Errorf("unexpected error reading random data: %v", err)
   426  	}
   427  	var stdout, stderr bytes.Buffer
   428  	options := &remotecommand.StreamOptions{
   429  		Stdin:  bytes.NewReader(randomData),
   430  		Stdout: &stdout,
   431  		Stderr: &stderr,
   432  	}
   433  	errorChan := make(chan error)
   434  	go func() {
   435  		// Start the streaming on the WebSocket "exec" client.
   436  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   437  	}()
   438  
   439  	select {
   440  	case <-time.After(wait.ForeverTestTimeout):
   441  		t.Fatalf("expect stream to be closed after connection is closed.")
   442  	case err := <-errorChan:
   443  		if err != nil {
   444  			t.Errorf("unexpected error: %v", err)
   445  		}
   446  	}
   447  	stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
   448  	if err != nil {
   449  		t.Errorf("error reading the stream: %v", err)
   450  		return
   451  	}
   452  	// Check the random data sent on STDIN was the same returned on STDOUT.
   453  	if !bytes.Equal(stdoutBytes, randomData) {
   454  		t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData))
   455  	}
   456  	stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
   457  	if err != nil {
   458  		t.Errorf("error reading the stream: %v", err)
   459  		return
   460  	}
   461  	// Check the random data sent on STDIN was the same returned on STDERR.
   462  	if !bytes.Equal(stderrBytes, randomData) {
   463  		t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData))
   464  	}
   465  	// Validate the streamtranslator metrics; should have one 200 success.
   466  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   467  	expected := `
   468  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   469  # TYPE apiserver_stream_translator_requests_total counter
   470  apiserver_stream_translator_requests_total{code="200"} 1
   471  `
   472  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   473  		t.Fatal(err)
   474  	}
   475  }
   476  
   477  // TestStreamTranslator_ThrottleReadChannels tests two streams (STDOUT, STDERR) using rate limited streams.
   478  func TestStreamTranslator_ThrottleReadChannels(t *testing.T) {
   479  	// Create upstream fake SPDY server which copies STDIN back onto STDOUT and STDERR stream.
   480  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   481  		ctx, err := createSPDYServerStreams(w, req, Options{
   482  			Stdin:  true,
   483  			Stdout: true,
   484  			Stderr: true,
   485  		})
   486  		if err != nil {
   487  			t.Errorf("error on createHTTPStreams: %v", err)
   488  			return
   489  		}
   490  		defer ctx.conn.Close()
   491  		// TeeReader copies data read on STDIN onto STDERR.
   492  		stdinReader := io.TeeReader(ctx.stdinStream, ctx.stderrStream)
   493  		// Also copy STDIN to STDOUT.
   494  		_, err = io.Copy(ctx.stdoutStream, stdinReader)
   495  		if err != nil {
   496  			t.Errorf("error copying STDIN to STDOUT: %v", err)
   497  		}
   498  	}))
   499  	defer spdyServer.Close()
   500  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server with
   501  	// streams STDIN, STDOUT, and STDERR. Create test server from StreamTranslatorHandler.
   502  	spdyLocation, err := url.Parse(spdyServer.URL)
   503  	if err != nil {
   504  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   505  	}
   506  	spdyTransport, err := fakeTransport()
   507  	if err != nil {
   508  		t.Fatalf("Unexpected error creating transport: %v", err)
   509  	}
   510  	streams := Options{Stdin: true, Stdout: true, Stderr: true}
   511  	maxBytesPerSec := 900 * 1024 // slightly less than the 1MB that is being transferred to exercise throttling.
   512  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, int64(maxBytesPerSec), streams)
   513  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   514  		streamTranslator.ServeHTTP(w, req)
   515  	}))
   516  	defer streamTranslatorServer.Close()
   517  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   518  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   519  	if err != nil {
   520  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   521  	}
   522  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   523  	if err != nil {
   524  		t.Errorf("unexpected error creating websocket executor: %v", err)
   525  	}
   526  	// Generate random data, and set it up to stream on STDIN. The data will be
   527  	// returned on the STDOUT and STDERR buffer.
   528  	randomSize := 1024 * 1024
   529  	randomData := make([]byte, randomSize)
   530  	if _, err := rand.Read(randomData); err != nil {
   531  		t.Errorf("unexpected error reading random data: %v", err)
   532  	}
   533  	var stdout, stderr bytes.Buffer
   534  	options := &remotecommand.StreamOptions{
   535  		Stdin:  bytes.NewReader(randomData),
   536  		Stdout: &stdout,
   537  		Stderr: &stderr,
   538  	}
   539  	errorChan := make(chan error)
   540  	go func() {
   541  		// Start the streaming on the WebSocket "exec" client.
   542  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   543  	}()
   544  
   545  	select {
   546  	case <-time.After(wait.ForeverTestTimeout):
   547  		t.Fatalf("expect stream to be closed after connection is closed.")
   548  	case err := <-errorChan:
   549  		if err != nil {
   550  			t.Errorf("unexpected error: %v", err)
   551  		}
   552  	}
   553  	stdoutBytes, err := io.ReadAll(bytes.NewReader(stdout.Bytes()))
   554  	if err != nil {
   555  		t.Errorf("error reading the stream: %v", err)
   556  		return
   557  	}
   558  	// Check the random data sent on STDIN was the same returned on STDOUT.
   559  	if !bytes.Equal(stdoutBytes, randomData) {
   560  		t.Errorf("unexpected data received: %d sent: %d", len(stdoutBytes), len(randomData))
   561  	}
   562  	stderrBytes, err := io.ReadAll(bytes.NewReader(stderr.Bytes()))
   563  	if err != nil {
   564  		t.Errorf("error reading the stream: %v", err)
   565  		return
   566  	}
   567  	// Check the random data sent on STDIN was the same returned on STDERR.
   568  	if !bytes.Equal(stderrBytes, randomData) {
   569  		t.Errorf("unexpected data received: %d sent: %d", len(stderrBytes), len(randomData))
   570  	}
   571  }
   572  
   573  // fakeTerminalSizeQueue implements TerminalSizeQueue, returning a random set of
   574  // "maxSizes" number of TerminalSizes, storing the TerminalSizes in "sizes" slice.
   575  type fakeTerminalSizeQueue struct {
   576  	maxSizes      int
   577  	terminalSizes []remotecommand.TerminalSize
   578  }
   579  
   580  // newTerminalSizeQueue returns a pointer to a fakeTerminalSizeQueue passing
   581  // "max" number of random TerminalSizes created.
   582  func newTerminalSizeQueue(max int) *fakeTerminalSizeQueue {
   583  	return &fakeTerminalSizeQueue{
   584  		maxSizes:      max,
   585  		terminalSizes: make([]remotecommand.TerminalSize, 0, max),
   586  	}
   587  }
   588  
   589  // Next returns a pointer to the next random TerminalSize, or nil if we have
   590  // already returned "maxSizes" TerminalSizes already. Stores the randomly
   591  // created TerminalSize in "terminalSizes" field for later validation.
   592  func (f *fakeTerminalSizeQueue) Next() *remotecommand.TerminalSize {
   593  	if len(f.terminalSizes) >= f.maxSizes {
   594  		return nil
   595  	}
   596  	size := randomTerminalSize()
   597  	f.terminalSizes = append(f.terminalSizes, size)
   598  	return &size
   599  }
   600  
   601  // randomTerminalSize returns a TerminalSize with random values in the
   602  // range (0-65535) for the fields Width and Height.
   603  func randomTerminalSize() remotecommand.TerminalSize {
   604  	randWidth := uint16(mrand.Intn(int(math.Pow(2, 16))))
   605  	randHeight := uint16(mrand.Intn(int(math.Pow(2, 16))))
   606  	return remotecommand.TerminalSize{
   607  		Width:  randWidth,
   608  		Height: randHeight,
   609  	}
   610  }
   611  
   612  // TestStreamTranslator_MultipleWriteChannels
   613  func TestStreamTranslator_TTYResizeChannel(t *testing.T) {
   614  	metrics.Register()
   615  	metrics.ResetForTest()
   616  	t.Cleanup(metrics.ResetForTest)
   617  	// Create the fake terminal size queue and the actualTerminalSizes which
   618  	// will be received at the opposite websocket endpoint.
   619  	numSizeQueue := 10000
   620  	sizeQueue := newTerminalSizeQueue(numSizeQueue)
   621  	actualTerminalSizes := make([]remotecommand.TerminalSize, 0, numSizeQueue)
   622  	// Create upstream fake SPDY server which copies STDIN back onto STDERR stream.
   623  	spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   624  		ctx, err := createSPDYServerStreams(w, req, Options{
   625  			Tty: true,
   626  		})
   627  		if err != nil {
   628  			t.Errorf("error on createHTTPStreams: %v", err)
   629  			return
   630  		}
   631  		defer ctx.conn.Close()
   632  		// Read the terminal resize requests, storing them in actualTerminalSizes
   633  		for i := 0; i < numSizeQueue; i++ {
   634  			actualTerminalSize := <-ctx.resizeChan
   635  			actualTerminalSizes = append(actualTerminalSizes, actualTerminalSize)
   636  		}
   637  	}))
   638  	defer spdyServer.Close()
   639  	// Create StreamTranslatorHandler, which points upstream to fake SPDY server with
   640  	// resize (TTY resize) stream. Create test server from StreamTranslatorHandler.
   641  	spdyLocation, err := url.Parse(spdyServer.URL)
   642  	if err != nil {
   643  		t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   644  	}
   645  	spdyTransport, err := fakeTransport()
   646  	if err != nil {
   647  		t.Fatalf("Unexpected error creating transport: %v", err)
   648  	}
   649  	streams := Options{Tty: true}
   650  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
   651  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   652  		streamTranslator.ServeHTTP(w, req)
   653  	}))
   654  	defer streamTranslatorServer.Close()
   655  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   656  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   657  	if err != nil {
   658  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   659  	}
   660  	exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   661  	if err != nil {
   662  		t.Errorf("unexpected error creating websocket executor: %v", err)
   663  	}
   664  	options := &remotecommand.StreamOptions{
   665  		Tty:               true,
   666  		TerminalSizeQueue: sizeQueue,
   667  	}
   668  	errorChan := make(chan error)
   669  	go func() {
   670  		// Start the streaming on the WebSocket "exec" client.
   671  		errorChan <- exec.StreamWithContext(context.Background(), *options)
   672  	}()
   673  
   674  	select {
   675  	case <-time.After(wait.ForeverTestTimeout):
   676  		t.Fatalf("expect stream to be closed after connection is closed.")
   677  	case err := <-errorChan:
   678  		if err != nil {
   679  			t.Errorf("unexpected error: %v", err)
   680  		}
   681  	}
   682  	// Validate the random TerminalSizes sent on the resize stream are the same
   683  	// as the actual TerminalSizes received at the websocket server.
   684  	if len(actualTerminalSizes) != numSizeQueue {
   685  		t.Fatalf("expected to receive num terminal resizes (%d), got (%d)",
   686  			numSizeQueue, len(actualTerminalSizes))
   687  	}
   688  	for i, actual := range actualTerminalSizes {
   689  		expected := sizeQueue.terminalSizes[i]
   690  		if !reflect.DeepEqual(expected, actual) {
   691  			t.Errorf("expected terminal resize window %v, got %v", expected, actual)
   692  		}
   693  	}
   694  	// Validate the streamtranslator metrics; should have one 200 success.
   695  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   696  	expected := `
   697  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   698  # TYPE apiserver_stream_translator_requests_total counter
   699  apiserver_stream_translator_requests_total{code="200"} 1
   700  `
   701  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   702  		t.Fatal(err)
   703  	}
   704  }
   705  
   706  // TestStreamTranslator_WebSocketServerErrors validates that when there is a problem creating
   707  // the websocket server as the first step of the StreamTranslator an error is properly returned.
   708  func TestStreamTranslator_WebSocketServerErrors(t *testing.T) {
   709  	metrics.Register()
   710  	metrics.ResetForTest()
   711  	t.Cleanup(metrics.ResetForTest)
   712  	spdyLocation, err := url.Parse("http://127.0.0.1")
   713  	if err != nil {
   714  		t.Fatalf("Unable to parse spdy server URL")
   715  	}
   716  	spdyTransport, err := fakeTransport()
   717  	if err != nil {
   718  		t.Fatalf("Unexpected error creating transport: %v", err)
   719  	}
   720  	streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, Options{})
   721  	streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   722  		streamTranslator.ServeHTTP(w, req)
   723  	}))
   724  	defer streamTranslatorServer.Close()
   725  	// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   726  	streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   727  	if err != nil {
   728  		t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   729  	}
   730  	exec, err := remotecommand.NewWebSocketExecutorForProtocols(
   731  		&rest.Config{Host: streamTranslatorLocation.Host},
   732  		"GET",
   733  		streamTranslatorServer.URL,
   734  		rcconstants.StreamProtocolV4Name, // RemoteCommand V4 protocol is unsupported
   735  	)
   736  	if err != nil {
   737  		t.Errorf("unexpected error creating websocket executor: %v", err)
   738  	}
   739  	errorChan := make(chan error)
   740  	go func() {
   741  		// Start the streaming on the WebSocket "exec" client. The WebSocket server within the
   742  		// StreamTranslator propagates an error here because the V4 protocol is not supported.
   743  		errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{})
   744  	}()
   745  
   746  	select {
   747  	case <-time.After(wait.ForeverTestTimeout):
   748  		t.Fatalf("expect stream to be closed after connection is closed.")
   749  	case err := <-errorChan:
   750  		// Must return "websocket unable to upgrade" (bad handshake) error.
   751  		if err == nil {
   752  			t.Fatalf("expected error, but received none")
   753  		}
   754  		if !strings.Contains(err.Error(), "unable to upgrade streaming request") {
   755  			t.Errorf("expected websocket bad handshake error, got (%s)", err)
   756  		}
   757  	}
   758  	// Validate the streamtranslator metrics; should have one 500 failure.
   759  	metricNames := []string{"apiserver_stream_translator_requests_total"}
   760  	expected := `
   761  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   762  # TYPE apiserver_stream_translator_requests_total counter
   763  apiserver_stream_translator_requests_total{code="400"} 1
   764  `
   765  	if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   766  		t.Fatal(err)
   767  	}
   768  }
   769  
   770  // TestStreamTranslator_BlockRedirects verifies that the StreamTranslator will *not* follow
   771  // redirects; it will thrown an error instead.
   772  func TestStreamTranslator_BlockRedirects(t *testing.T) {
   773  	metrics.Register()
   774  	metrics.ResetForTest()
   775  	t.Cleanup(metrics.ResetForTest)
   776  	for _, statusCode := range []int{
   777  		http.StatusMovedPermanently,  // 301
   778  		http.StatusFound,             // 302
   779  		http.StatusSeeOther,          // 303
   780  		http.StatusTemporaryRedirect, // 307
   781  		http.StatusPermanentRedirect, // 308
   782  	} {
   783  		// Create upstream fake SPDY server which returns a redirect.
   784  		spdyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   785  			w.Header().Set("Location", "/")
   786  			w.WriteHeader(statusCode)
   787  		}))
   788  		defer spdyServer.Close()
   789  		spdyLocation, err := url.Parse(spdyServer.URL)
   790  		if err != nil {
   791  			t.Fatalf("Unable to parse spdy server URL: %s", spdyServer.URL)
   792  		}
   793  		spdyTransport, err := fakeTransport()
   794  		if err != nil {
   795  			t.Fatalf("Unexpected error creating transport: %v", err)
   796  		}
   797  		streams := Options{Stdout: true}
   798  		streamTranslator := NewStreamTranslatorHandler(spdyLocation, spdyTransport, 0, streams)
   799  		streamTranslatorServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   800  			streamTranslator.ServeHTTP(w, req)
   801  		}))
   802  		defer streamTranslatorServer.Close()
   803  		// Now create the websocket client (executor), and point it to the "streamTranslatorServer".
   804  		streamTranslatorLocation, err := url.Parse(streamTranslatorServer.URL)
   805  		if err != nil {
   806  			t.Fatalf("Unable to parse StreamTranslator server URL: %s", streamTranslatorServer.URL)
   807  		}
   808  		exec, err := remotecommand.NewWebSocketExecutor(&rest.Config{Host: streamTranslatorLocation.Host}, "GET", streamTranslatorServer.URL)
   809  		if err != nil {
   810  			t.Errorf("unexpected error creating websocket executor: %v", err)
   811  		}
   812  		errorChan := make(chan error)
   813  		go func() {
   814  			// Start the streaming on the WebSocket "exec" client.
   815  			// Should return "redirect not allowed" error.
   816  			errorChan <- exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{})
   817  		}()
   818  
   819  		select {
   820  		case <-time.After(wait.ForeverTestTimeout):
   821  			t.Fatalf("expect stream to be closed after connection is closed.")
   822  		case err := <-errorChan:
   823  			// Must return "redirect now allowed" error.
   824  			if err == nil {
   825  				t.Fatalf("expected error, but received none")
   826  			}
   827  			if !strings.Contains(err.Error(), "redirect not allowed") {
   828  				t.Errorf("expected redirect not allowed error, got (%s)", err)
   829  			}
   830  		}
   831  		// Validate the streamtranslator metrics; should have one 500 failure each loop.
   832  		metricNames := []string{"apiserver_stream_translator_requests_total"}
   833  		expected := `
   834  # HELP apiserver_stream_translator_requests_total [ALPHA] Total number of requests that were handled by the StreamTranslatorProxy, which processes streaming RemoteCommand/V5
   835  # TYPE apiserver_stream_translator_requests_total counter
   836  apiserver_stream_translator_requests_total{code="500"} 1
   837  `
   838  		if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...); err != nil {
   839  			t.Fatal(err)
   840  		}
   841  		metrics.ResetForTest() // Clear metrics each loop
   842  	}
   843  }
   844  
   845  // streamContext encapsulates the structures necessary to communicate through
   846  // a SPDY connection, including the Reader/Writer streams.
   847  type streamContext struct {
   848  	conn         io.Closer
   849  	stdinStream  io.ReadCloser
   850  	stdoutStream io.WriteCloser
   851  	stderrStream io.WriteCloser
   852  	resizeStream io.ReadCloser
   853  	resizeChan   chan remotecommand.TerminalSize
   854  	writeStatus  func(status *apierrors.StatusError) error
   855  }
   856  
   857  type streamAndReply struct {
   858  	httpstream.Stream
   859  	replySent <-chan struct{}
   860  }
   861  
   862  // CreateSPDYServerStreams upgrades the passed HTTP request to a SPDY bi-directional streaming
   863  // connection with remote command streams defined in passed options. Returns a streamContext
   864  // structure containing the Reader/Writer streams to communicate through the SDPY connection.
   865  // Returns an error if unable to upgrade the HTTP connection to a SPDY connection.
   866  func createSPDYServerStreams(w http.ResponseWriter, req *http.Request, opts Options) (*streamContext, error) {
   867  	_, err := httpstream.Handshake(req, w, []string{rcconstants.StreamProtocolV4Name})
   868  	if err != nil {
   869  		return nil, err
   870  	}
   871  
   872  	upgrader := spdy.NewResponseUpgrader()
   873  	streamCh := make(chan streamAndReply)
   874  	conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream, replySent <-chan struct{}) error {
   875  		streamCh <- streamAndReply{Stream: stream, replySent: replySent}
   876  		return nil
   877  	})
   878  	ctx := &streamContext{
   879  		conn: conn,
   880  	}
   881  
   882  	// wait for stream
   883  	replyChan := make(chan struct{}, 5)
   884  	defer close(replyChan)
   885  	receivedStreams := 0
   886  	expectedStreams := 1 // expect at least the error stream
   887  	if opts.Stdout {
   888  		expectedStreams++
   889  	}
   890  	if opts.Stdin {
   891  		expectedStreams++
   892  	}
   893  	if opts.Stderr {
   894  		expectedStreams++
   895  	}
   896  	if opts.Tty {
   897  		expectedStreams++
   898  	}
   899  WaitForStreams:
   900  	for {
   901  		select {
   902  		case stream := <-streamCh:
   903  			streamType := stream.Headers().Get(v1.StreamType)
   904  			switch streamType {
   905  			case v1.StreamTypeError:
   906  				replyChan <- struct{}{}
   907  				ctx.writeStatus = v4WriteStatusFunc(stream)
   908  			case v1.StreamTypeStdout:
   909  				replyChan <- struct{}{}
   910  				ctx.stdoutStream = stream
   911  			case v1.StreamTypeStdin:
   912  				replyChan <- struct{}{}
   913  				ctx.stdinStream = stream
   914  			case v1.StreamTypeStderr:
   915  				replyChan <- struct{}{}
   916  				ctx.stderrStream = stream
   917  			case v1.StreamTypeResize:
   918  				replyChan <- struct{}{}
   919  				ctx.resizeStream = stream
   920  			default:
   921  				// add other stream ...
   922  				return nil, errors.New("unimplemented stream type")
   923  			}
   924  		case <-replyChan:
   925  			receivedStreams++
   926  			if receivedStreams == expectedStreams {
   927  				break WaitForStreams
   928  			}
   929  		}
   930  	}
   931  
   932  	if ctx.resizeStream != nil {
   933  		ctx.resizeChan = make(chan remotecommand.TerminalSize)
   934  		go handleResizeEvents(req.Context(), ctx.resizeStream, ctx.resizeChan)
   935  	}
   936  
   937  	return ctx, nil
   938  }
   939  
   940  func v4WriteStatusFunc(stream io.Writer) func(status *apierrors.StatusError) error {
   941  	return func(status *apierrors.StatusError) error {
   942  		bs, err := json.Marshal(status.Status())
   943  		if err != nil {
   944  			return err
   945  		}
   946  		_, err = stream.Write(bs)
   947  		return err
   948  	}
   949  }
   950  
   951  func fakeTransport() (*http.Transport, error) {
   952  	cfg := &transport.Config{
   953  		TLS: transport.TLSConfig{
   954  			Insecure: true,
   955  			CAFile:   "",
   956  		},
   957  	}
   958  	rt, err := transport.New(cfg)
   959  	if err != nil {
   960  		return nil, err
   961  	}
   962  	t, ok := rt.(*http.Transport)
   963  	if !ok {
   964  		return nil, fmt.Errorf("unknown transport type: %T", rt)
   965  	}
   966  	return t, nil
   967  }