github.com/uber/kraken@v0.1.4/lib/backend/hdfsbackend/config.go (about)

     1  // Copyright (c) 2016-2019 Uber Technologies, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  package hdfsbackend
    15  
    16  import "github.com/uber/kraken/lib/backend/hdfsbackend/webhdfs"
    17  
    18  // Config defines configuration for all HDFS clients.
    19  type Config struct {
    20  	NameNodes     []string `yaml:"namenodes"`
    21  	UserName      string   `yaml:"username"`
    22  	RootDirectory string   `yaml:"root_directory"`
    23  
    24  	// ListConcurrency is the number of threads used for listing.
    25  	ListConcurrency int `yaml:"list_concurrency"`
    26  
    27  	// NamePath identifies which namepath.Pather to use.
    28  	NamePath string `yaml:"name_path"`
    29  
    30  	// UploadDirectory is scratch space, relative to RootDirectory, used for
    31  	// uploading files before moving them to content-addressable storage. Avoids
    32  	// partial uploads corrupting the content-addressable storage space.
    33  	UploadDirectory string `yaml:"upload_directory"`
    34  
    35  	WebHDFS webhdfs.Config `yaml:"webhdfs"`
    36  
    37  	// Enables test-only behavior.
    38  	testing bool
    39  }
    40  
    41  func (c *Config) applyDefaults() {
    42  	if c.RootDirectory == "" {
    43  		c.RootDirectory = "/infra/dockerRegistry/"
    44  	}
    45  	if c.ListConcurrency == 0 {
    46  		c.ListConcurrency = 16
    47  	}
    48  	if c.UploadDirectory == "" {
    49  		c.UploadDirectory = "_uploads"
    50  	}
    51  }