github.com/uber/kraken@v0.1.4/origin/blobserver/cluster_client_test.go (about)

     1  // Copyright (c) 2016-2019 Uber Technologies, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  package blobserver
    15  
    16  import (
    17  	"bytes"
    18  	"io/ioutil"
    19  	"sort"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/uber/kraken/core"
    24  	"github.com/uber/kraken/lib/backend"
    25  	"github.com/uber/kraken/lib/hostlist"
    26  	"github.com/uber/kraken/lib/persistedretry/writeback"
    27  	"github.com/uber/kraken/mocks/origin/blobclient"
    28  	"github.com/uber/kraken/origin/blobclient"
    29  	"github.com/uber/kraken/utils/httputil"
    30  
    31  	"github.com/cenkalti/backoff"
    32  	"github.com/golang/mock/gomock"
    33  	"github.com/stretchr/testify/require"
    34  )
    35  
    36  func toAddrs(clients []blobclient.Client) []string {
    37  	var addrs []string
    38  	for _, c := range clients {
    39  		addrs = append(addrs, c.Addr())
    40  	}
    41  	sort.Strings(addrs)
    42  	return addrs
    43  }
    44  
    45  func TestClusterClientResilientToUnavailableMasters(t *testing.T) {
    46  	require := require.New(t)
    47  
    48  	cp := newTestClientProvider()
    49  
    50  	s := newTestServer(t, master1, hashRingMaxReplica(), cp)
    51  	defer s.cleanup()
    52  
    53  	// Register dummy master addresses so Provide can still create a Client for
    54  	// unavailable masters.
    55  	cp.register(master2, blobclient.New("localhost:0"))
    56  	cp.register(master3, blobclient.New("localhost:0"))
    57  
    58  	r := blobclient.NewClientResolver(cp, hostlist.Fixture(master1))
    59  	cc := blobclient.NewClusterClient(r)
    60  
    61  	// Run many times to make sure we eventually hit unavailable masters.
    62  	for i := 0; i < 100; i++ {
    63  		blob := core.SizedBlobFixture(256, 8)
    64  
    65  		s.writeBackManager.EXPECT().Add(
    66  			writeback.MatchTask(writeback.NewTask(
    67  				backend.NoopNamespace, blob.Digest.Hex(), 0))).Return(nil)
    68  		require.NoError(cc.UploadBlob(backend.NoopNamespace, blob.Digest, bytes.NewReader(blob.Content)))
    69  
    70  		bi, err := cc.Stat(backend.NoopNamespace, blob.Digest)
    71  		require.NoError(err)
    72  		require.NotNil(bi)
    73  		require.Equal(int64(256), bi.Size)
    74  
    75  		mi, err := cc.GetMetaInfo(backend.NoopNamespace, blob.Digest)
    76  		require.NoError(err)
    77  		require.NotNil(mi)
    78  
    79  		var buf bytes.Buffer
    80  		require.NoError(cc.DownloadBlob(backend.NoopNamespace, blob.Digest, &buf))
    81  		require.Equal(string(blob.Content), buf.String())
    82  
    83  		peers, err := cc.Owners(blob.Digest)
    84  		require.NoError(err)
    85  		require.Len(peers, 1)
    86  		require.Equal(s.pctx, peers[0])
    87  	}
    88  }
    89  
    90  func TestClusterClientReturnsErrorOnNoAvailability(t *testing.T) {
    91  	require := require.New(t)
    92  
    93  	cp := newTestClientProvider()
    94  	cp.register(master1, blobclient.New("localhost:0"))
    95  	cp.register(master2, blobclient.New("localhost:0"))
    96  	cp.register(master3, blobclient.New("localhost:0"))
    97  
    98  	r := blobclient.NewClientResolver(cp, hostlist.Fixture(master1))
    99  	cc := blobclient.NewClusterClient(r)
   100  
   101  	blob := core.NewBlobFixture()
   102  
   103  	require.Error(cc.UploadBlob(backend.NoopNamespace, blob.Digest, bytes.NewReader(blob.Content)))
   104  
   105  	_, err := cc.Stat(backend.NoopNamespace, blob.Digest)
   106  	require.Error(err)
   107  
   108  	_, err = cc.GetMetaInfo(backend.NoopNamespace, blob.Digest)
   109  	require.Error(err)
   110  
   111  	require.Error(cc.DownloadBlob(backend.NoopNamespace, blob.Digest, ioutil.Discard))
   112  
   113  	_, err = cc.Owners(blob.Digest)
   114  	require.Error(err)
   115  }
   116  
   117  func TestPollSkipsOriginOnTimeout(t *testing.T) {
   118  	require := require.New(t)
   119  
   120  	ctrl := gomock.NewController(t)
   121  	defer ctrl.Finish()
   122  
   123  	blob := core.NewBlobFixture()
   124  	namespace := core.TagFixture()
   125  
   126  	mockResolver := mockblobclient.NewMockClientResolver(ctrl)
   127  
   128  	mockClient1 := mockblobclient.NewMockClient(ctrl)
   129  	mockClient2 := mockblobclient.NewMockClient(ctrl)
   130  
   131  	mockResolver.EXPECT().Resolve(blob.Digest).Return(
   132  		[]blobclient.Client{mockClient1, mockClient2}, nil)
   133  
   134  	mockClient1.EXPECT().DownloadBlob(
   135  		namespace, blob.Digest, nil).Return(httputil.StatusError{Status: 202}).MinTimes(1)
   136  	mockClient1.EXPECT().Addr().Return("client1")
   137  	mockClient2.EXPECT().DownloadBlob(namespace, blob.Digest, nil).Return(nil)
   138  
   139  	b := backoff.WithMaxRetries(backoff.NewConstantBackOff(100*time.Millisecond), 5)
   140  
   141  	require.NoError(blobclient.Poll(mockResolver, b, blob.Digest, func(c blobclient.Client) error {
   142  		return c.DownloadBlob(namespace, blob.Digest, nil)
   143  	}))
   144  }
   145  
   146  func TestPollSkipsOriginOnNetworkErrors(t *testing.T) {
   147  	require := require.New(t)
   148  
   149  	ctrl := gomock.NewController(t)
   150  	defer ctrl.Finish()
   151  
   152  	blob := core.NewBlobFixture()
   153  	namespace := core.TagFixture()
   154  
   155  	mockResolver := mockblobclient.NewMockClientResolver(ctrl)
   156  
   157  	mockClient1 := mockblobclient.NewMockClient(ctrl)
   158  	mockClient2 := mockblobclient.NewMockClient(ctrl)
   159  
   160  	mockResolver.EXPECT().Resolve(blob.Digest).Return([]blobclient.Client{mockClient1, mockClient2}, nil)
   161  
   162  	mockClient1.EXPECT().DownloadBlob(namespace, blob.Digest, nil).Return(httputil.NetworkError{})
   163  	mockClient1.EXPECT().Addr().Return("client1")
   164  	mockClient2.EXPECT().DownloadBlob(namespace, blob.Digest, nil).Return(nil)
   165  
   166  	b := backoff.WithMaxRetries(backoff.NewConstantBackOff(100*time.Millisecond), 5)
   167  
   168  	require.NoError(blobclient.Poll(mockResolver, b, blob.Digest, func(c blobclient.Client) error {
   169  		return c.DownloadBlob(namespace, blob.Digest, nil)
   170  	}))
   171  }
   172  
   173  func TestClusterClientReturnsErrorOnNoAvailableOrigins(t *testing.T) {
   174  	require := require.New(t)
   175  
   176  	ctrl := gomock.NewController(t)
   177  	defer ctrl.Finish()
   178  
   179  	mockResolver := mockblobclient.NewMockClientResolver(ctrl)
   180  
   181  	cc := blobclient.NewClusterClient(mockResolver)
   182  
   183  	blob := core.NewBlobFixture()
   184  	namespace := core.TagFixture()
   185  
   186  	mockClient1 := mockblobclient.NewMockClient(ctrl)
   187  	mockClient2 := mockblobclient.NewMockClient(ctrl)
   188  	mockResolver.EXPECT().Resolve(blob.Digest).Return([]blobclient.Client{mockClient1, mockClient2}, nil)
   189  
   190  	mockClient1.EXPECT().GetMetaInfo(namespace, blob.Digest).Return(nil, httputil.NetworkError{})
   191  	mockClient2.EXPECT().GetMetaInfo(namespace, blob.Digest).Return(nil, httputil.NetworkError{})
   192  
   193  	_, err := cc.GetMetaInfo(namespace, blob.Digest)
   194  	require.Error(err)
   195  }
   196  
   197  func TestClusterClientOverwriteMetainfo(t *testing.T) {
   198  	require := require.New(t)
   199  
   200  	ctrl := gomock.NewController(t)
   201  	defer ctrl.Finish()
   202  
   203  	mockResolver := mockblobclient.NewMockClientResolver(ctrl)
   204  
   205  	cc := blobclient.NewClusterClient(mockResolver)
   206  
   207  	d := core.DigestFixture()
   208  
   209  	mockClient1 := mockblobclient.NewMockClient(ctrl)
   210  	mockClient2 := mockblobclient.NewMockClient(ctrl)
   211  	mockResolver.EXPECT().Resolve(d).Return([]blobclient.Client{mockClient1, mockClient2}, nil)
   212  
   213  	mockClient1.EXPECT().OverwriteMetaInfo(d, int64(16)).Return(nil)
   214  	mockClient2.EXPECT().OverwriteMetaInfo(d, int64(16)).Return(nil)
   215  
   216  	err := cc.OverwriteMetaInfo(d, 16)
   217  	require.NoError(err)
   218  }
   219  
   220  func TestClusterClientStatContinueWhenNotFound(t *testing.T) {
   221  	require := require.New(t)
   222  
   223  	ctrl := gomock.NewController(t)
   224  	defer ctrl.Finish()
   225  
   226  	mockResolver := mockblobclient.NewMockClientResolver(ctrl)
   227  
   228  	cc := blobclient.NewClusterClient(mockResolver)
   229  
   230  	blob := core.SizedBlobFixture(256, 8)
   231  	namespace := core.TagFixture()
   232  
   233  	mockClient := mockblobclient.NewMockClient(ctrl)
   234  	// Reuse the same mockClient for two origins because origins are shuffled.
   235  	mockResolver.EXPECT().Resolve(blob.Digest).Return([]blobclient.Client{mockClient, mockClient}, nil)
   236  
   237  	gomock.InOrder(
   238  		mockClient.EXPECT().Stat(namespace, blob.Digest).Return(nil, blobclient.ErrBlobNotFound),
   239  		mockClient.EXPECT().Stat(namespace, blob.Digest).Return(core.NewBlobInfo(256), nil),
   240  	)
   241  
   242  	bi, err := cc.Stat(namespace, blob.Digest)
   243  	require.NoError(err)
   244  	require.NotNil(bi)
   245  	require.Equal(int64(256), bi.Size)
   246  }