github.com/zntrio/harp/v2@v2.0.9/pkg/bundle/vault/internal/operation/importer.go (about) 1 // Licensed to Elasticsearch B.V. under one or more contributor 2 // license agreements. See the NOTICE file distributed with 3 // this work for additional information regarding copyright 4 // ownership. Elasticsearch B.V. licenses this file to you under 5 // the Apache License, Version 2.0 (the "License"); you may 6 // not use this file except in compliance with the License. 7 // You may obtain a copy of the License at 8 // 9 // http://www.apache.org/licenses/LICENSE-2.0 10 // 11 // Unless required by applicable law or agreed to in writing, 12 // software distributed under the License is distributed on an 13 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 // KIND, either express or implied. See the License for the 15 // specific language governing permissions and limitations 16 // under the License. 17 18 package operation 19 20 import ( 21 "context" 22 "fmt" 23 "path" 24 "strings" 25 "sync" 26 27 "github.com/hashicorp/vault/api" 28 "go.uber.org/zap" 29 30 bundlev1 "github.com/zntrio/harp/v2/api/gen/go/harp/bundle/v1" 31 "github.com/zntrio/harp/v2/pkg/bundle/secret" 32 "github.com/zntrio/harp/v2/pkg/sdk/log" 33 "github.com/zntrio/harp/v2/pkg/vault/kv" 34 vpath "github.com/zntrio/harp/v2/pkg/vault/path" 35 36 "golang.org/x/sync/errgroup" 37 "golang.org/x/sync/semaphore" 38 ) 39 40 // Importer initialize a secret importer operation. 41 func Importer(client *api.Client, bundleFile *bundlev1.Bundle, prefix string, withMetadata, withVaultMetadata bool, maxWorkerCount int64) Operation { 42 return &importer{ 43 client: client, 44 bundle: bundleFile, 45 prefix: prefix, 46 withMetadata: withMetadata || withVaultMetadata, 47 withVaultMetadata: withVaultMetadata, 48 backends: map[string]kv.Service{}, 49 maxWorkerCount: maxWorkerCount, 50 } 51 } 52 53 // ----------------------------------------------------------------------------- 54 55 type importer struct { 56 client *api.Client 57 bundle *bundlev1.Bundle 58 prefix string 59 withMetadata bool 60 withVaultMetadata bool 61 backends map[string]kv.Service 62 backendsMutex sync.RWMutex 63 maxWorkerCount int64 64 } 65 66 // Run the implemented operation 67 // 68 //nolint:gocognit,funlen,gocyclo // To refactor 69 func (op *importer) Run(ctx context.Context) error { 70 // Initialize sub context 71 g, gctx := errgroup.WithContext(ctx) 72 73 // Prepare channels 74 packageChan := make(chan *bundlev1.Package) 75 76 // Validate worker count 77 if op.maxWorkerCount < 1 { 78 op.maxWorkerCount = 1 79 } 80 81 // consumers --------------------------------------------------------------- 82 83 // Secret writer 84 g.Go(func() error { 85 // Initialize a semaphore with maxReaderWorker tokens 86 sem := semaphore.NewWeighted(op.maxWorkerCount) 87 88 // Writer errGroup 89 gWriter, gWriterCtx := errgroup.WithContext(gctx) 90 91 // Listen for message 92 for secretPackage := range packageChan { 93 // Assign local reference 94 secretPackage := secretPackage 95 96 if err := gWriterCtx.Err(); err != nil { 97 // Stop processing 98 break 99 } 100 101 // Acquire a token 102 if err := sem.Acquire(gWriterCtx, 1); err != nil { 103 return fmt.Errorf("unable to acquire a semaphore token: %w", err) 104 } 105 106 log.For(gWriterCtx).Debug("Writing secret ...", zap.String("prefix", op.prefix), zap.String("path", secretPackage.Name)) 107 108 // Build function reader 109 gWriter.Go(func() error { 110 defer sem.Release(1) 111 112 if err := gWriterCtx.Err(); err != nil { 113 //nolint:nilerr // Context has already an error 114 return nil 115 } 116 117 // No data to insert 118 if secretPackage.Secrets == nil { 119 return nil 120 } 121 122 data := map[string]interface{}{} 123 // Wrap secret k/v as a map 124 for _, s := range secretPackage.Secrets.Data { 125 // Unpack secret to original value 126 var value interface{} 127 if err := secret.Unpack(s.Value, &value); err != nil { 128 return fmt.Errorf("unable to unpack secret value for path %q with key %q: %w", secretPackage.Name, s.Key, err) 129 } 130 131 // Assign to map for vault storage 132 data[s.Key] = value 133 } 134 135 // Export metadata 136 metadata := map[string]interface{}{} 137 if op.withMetadata { 138 // Has annotations 139 if len(secretPackage.Annotations) > 0 { 140 for k, v := range secretPackage.Annotations { 141 metadata[k] = v 142 } 143 } 144 145 // Has labels 146 if len(secretPackage.Labels) > 0 { 147 for k, v := range secretPackage.Labels { 148 metadata[fmt.Sprintf("label#%s", k)] = v 149 } 150 } 151 } 152 153 // Assemble secret path 154 secretPath := secretPackage.Name 155 if op.prefix != "" { 156 secretPath = path.Join(op.prefix, secretPath) 157 } 158 159 // Extract root backend path 160 rootPath := strings.Split(vpath.SanitizePath(secretPath), "/")[0] 161 162 // Check backend initialization 163 if _, ok := op.backends[rootPath]; !ok { 164 // Initialize new service for backend 165 service, err := kv.New(op.client, rootPath, kv.WithVaultMetatadata(op.withVaultMetadata), kv.WithContext(gWriterCtx)) 166 if err != nil { 167 return fmt.Errorf("unable to initialize Vault service for %q KV backend: %w", op.prefix, err) 168 } 169 170 // All queries will be handled by same backend service 171 op.backendsMutex.Lock() 172 op.backends[rootPath] = service 173 op.backendsMutex.Unlock() 174 } 175 176 // Write secret to Vault 177 if err := op.backends[rootPath].WriteWithMeta(gWriterCtx, secretPath, data, metadata); err != nil { 178 return fmt.Errorf("unable to write secret data for path %q: %w", secretPath, err) 179 } 180 181 // No error 182 return nil 183 }) 184 } 185 186 // No error 187 return gWriter.Wait() 188 }) 189 190 // producers --------------------------------------------------------------- 191 192 // Bundle package publisher 193 g.Go(func() error { 194 defer close(packageChan) 195 196 for _, p := range op.bundle.Packages { 197 select { 198 case <-gctx.Done(): 199 return gctx.Err() 200 case packageChan <- p: 201 } 202 } 203 204 // No error 205 return nil 206 }) 207 208 // Wait for all goroutime to complete 209 if err := g.Wait(); err != nil { 210 return fmt.Errorf("vault operation error: %w", err) 211 } 212 213 // No error 214 return nil 215 }