github.com/microsoft/fabrikate@v1.0.0-alpha.1.0.20210115014322-dc09194d0885/internal/generators/helm.go (about)

     1  package generators
     2  
     3  import (
     4  	"fmt"
     5  	"io/ioutil"
     6  	"os"
     7  	"os/exec"
     8  	"path"
     9  	"path/filepath"
    10  	"reflect"
    11  	"sort"
    12  	"strings"
    13  	"sync"
    14  
    15  	"github.com/google/uuid"
    16  	"github.com/kyokomi/emoji"
    17  	"github.com/microsoft/fabrikate/internal/core"
    18  	"github.com/microsoft/fabrikate/internal/git"
    19  	"github.com/microsoft/fabrikate/internal/helm"
    20  	"github.com/microsoft/fabrikate/internal/logger"
    21  	"github.com/timfpark/yaml"
    22  )
    23  
    24  // HelmGenerator provides 'helm generate' generator functionality to Fabrikate
    25  type HelmGenerator struct{}
    26  
    27  type namespaceInjectionResponse struct {
    28  	index              int
    29  	namespacedManifest *[]byte
    30  	err                error
    31  	warn               *string
    32  }
    33  
    34  // func addNamespaceToManifests(manifests, namespace string) (namespacedManifests string, err error) {
    35  func addNamespaceToManifests(manifests, namespace string) chan namespaceInjectionResponse {
    36  	respChan := make(chan namespaceInjectionResponse)
    37  	syncGroup := sync.WaitGroup{}
    38  	splitManifest := strings.Split(manifests, "\n---")
    39  
    40  	// Wait for all manifests to be iterated over then close the channel
    41  	syncGroup.Add(len(splitManifest))
    42  	go func() {
    43  		syncGroup.Wait()
    44  		close(respChan)
    45  	}()
    46  
    47  	// Iterate over all manifests, decrementing the wait group for every channel put
    48  	for index, manifest := range splitManifest {
    49  		go func(index int, manifest string) {
    50  			parsedManifest := make(map[interface{}]interface{})
    51  
    52  			// Push a warning if unable to unmarshal
    53  			if err := yaml.Unmarshal([]byte(manifest), &parsedManifest); err != nil {
    54  				warning := emoji.Sprintf(":question: Unable to unmarshal manifest into type '%s', this is most likely a warning message outputted from `helm template`. Skipping namespace injection of '%s' into manifest: '%s'", reflect.TypeOf(parsedManifest), namespace, manifest)
    55  				respChan <- namespaceInjectionResponse{warn: &warning}
    56  				syncGroup.Done()
    57  				return
    58  			}
    59  
    60  			// strip any empty entries
    61  			if len(parsedManifest) == 0 {
    62  				syncGroup.Done()
    63  				return
    64  			}
    65  
    66  			// Inject the namespace
    67  			if parsedManifest["metadata"] != nil {
    68  				metadataMap := parsedManifest["metadata"].(map[interface{}]interface{})
    69  				if metadataMap["namespace"] == nil {
    70  					metadataMap["namespace"] = namespace
    71  				}
    72  			}
    73  
    74  			// Marshal updated manifest and put the response on channel
    75  			updatedManifest, err := yaml.Marshal(&parsedManifest)
    76  			if err != nil {
    77  				respChan <- namespaceInjectionResponse{err: err}
    78  				syncGroup.Done()
    79  				return
    80  			}
    81  			respChan <- namespaceInjectionResponse{index: index, namespacedManifest: &updatedManifest}
    82  			syncGroup.Done()
    83  		}(index, manifest)
    84  	}
    85  
    86  	return respChan
    87  }
    88  
    89  // cleanK8sManifest attempts to remove any invalid entries in k8s yaml.
    90  // If any entries after being split by "---" are not a map or are empty, they are removed
    91  func cleanK8sManifest(manifests string) (cleanedManifests string, err error) {
    92  	splitManifest := strings.Split(manifests, "\n---")
    93  
    94  	for _, manifest := range splitManifest {
    95  		parsedManifest := make(map[interface{}]interface{})
    96  
    97  		// Log a warning if unable to unmarshal; skip the entry
    98  		if err := yaml.Unmarshal([]byte(manifest), &parsedManifest); err != nil {
    99  			warning := emoji.Sprintf(":question: Unable to unmarshal manifest into type '%s', this is most likely a warning message outputted from `helm template`.\nRemoving manifest entry: '%s'\nUnmarshal error encountered: '%s'", reflect.TypeOf(parsedManifest), manifest, err)
   100  			logger.Warn(warning)
   101  			continue
   102  		}
   103  
   104  		// Remove empty entries
   105  		if len(parsedManifest) == 0 {
   106  			continue
   107  		}
   108  
   109  		cleanedManifests += fmt.Sprintf("---\n%s\n", manifest)
   110  	}
   111  
   112  	return cleanedManifests, err
   113  }
   114  
   115  // makeHelmRepoPath returns the path where the components helm charts are
   116  // located -- will be an entire helm repo if `method: git` or just the target
   117  // chart if `method: helm`
   118  func (hg *HelmGenerator) makeHelmRepoPath(c *core.Component) string {
   119  	// `method: git` will clone the entire helm repo; uses path to point to chart dir
   120  	if c.Method == "git" || c.Method == "helm" {
   121  		return path.Join(c.PhysicalPath, "helm_repos", c.Name)
   122  	}
   123  
   124  	return c.PhysicalPath
   125  }
   126  
   127  // getChartPath returns the absolute path to the directory containing the
   128  // Chart.yaml
   129  func (hg *HelmGenerator) getChartPath(c *core.Component) (string, error) {
   130  	installer, err := c.ToInstallable()
   131  	if err != nil {
   132  		return "", err
   133  	}
   134  	installPath, err := installer.GetInstallPath()
   135  	if err != nil {
   136  		return "", err
   137  	}
   138  	return installPath, nil
   139  	// if c.Method == "helm" || c.Method == "git" {
   140  	// 	absHelmPath, err := filepath.Abs(hg.makeHelmRepoPath(c))
   141  	// 	if err != nil {
   142  	// 		return "", err
   143  	// 	}
   144  	// 	switch c.Method {
   145  	// 	case "git":
   146  	// 		// method: git downloads the entire repo into _helm_chart and the dir containing Chart.yaml specified by Path
   147  	// 		return path.Join(absHelmPath, c.Path), nil
   148  	// 	case "helm":
   149  	// 		// method: helm only downloads target chart into _helm_chart
   150  	// 		return absHelmPath, nil
   151  	// 	}
   152  	// }
   153  
   154  	// // Default to `method: local` and use the Path provided as location of the chart
   155  	// return filepath.Abs(path.Join(c.PhysicalPath, c.Path))
   156  }
   157  
   158  // Generate returns the helm templated manifests specified by this component.
   159  func (hg *HelmGenerator) Generate(component *core.Component) (manifest string, err error) {
   160  	logger.Info(emoji.Sprintf(":truck: Generating component '%s' with helm with repo %s", component.Name, component.Source))
   161  
   162  	configYaml, err := yaml.Marshal(&component.Config.Config)
   163  	if err != nil {
   164  		logger.Error(fmt.Sprintf("Marshalling config yaml for helm generated component '%s' failed with: %s\n", component.Name, err.Error()))
   165  		return "", err
   166  	}
   167  
   168  	// Write helm config to temporary file in tmp folder
   169  	randomString, err := uuid.NewRandom()
   170  	if err != nil {
   171  		return "", err
   172  	}
   173  	overriddenValuesFileName := fmt.Sprintf("%s.yaml", randomString.String())
   174  	absOverriddenPath := path.Join(os.TempDir(), overriddenValuesFileName)
   175  	defer os.Remove(absOverriddenPath)
   176  
   177  	logger.Debug(emoji.Sprintf(":pencil: Writing config %s to %s\n", configYaml, absOverriddenPath))
   178  	if err = ioutil.WriteFile(absOverriddenPath, configYaml, 0777); err != nil {
   179  		return "", err
   180  	}
   181  
   182  	// Default to `default` namespace unless provided
   183  	namespace := "default"
   184  	if component.Config.Namespace != "" {
   185  		namespace = component.Config.Namespace
   186  	}
   187  
   188  	// Run `helm template` on the chart using the config stored in temp dir
   189  	chartPath, err := hg.getChartPath(component)
   190  	if err != nil {
   191  		return "", err
   192  	}
   193  	logger.Info(emoji.Sprintf(":memo: Running `helm template` on template '%s'", chartPath))
   194  	output, err := exec.Command("helm", "template", component.Name, chartPath, "--values", absOverriddenPath, "--namespace", namespace).CombinedOutput()
   195  	if err != nil {
   196  		logger.Error(fmt.Sprintf("helm template failed with:\n%s: %s", err, output))
   197  		return "", err
   198  	}
   199  	// Remove any empty/non-map entries in manifests
   200  	logger.Info(emoji.Sprintf(":scissors: Removing empty entries from generated manifests from chart '%s'", chartPath))
   201  	stringManifests, err := cleanK8sManifest(string(output))
   202  	if err != nil {
   203  		return "", err
   204  	}
   205  
   206  	// helm template does not inject namespace unless chart directly provides support for it: https://github.com/helm/helm/issues/3553
   207  	// some helm templates expect Tiller to inject namespace, so enable Fabrikate component designer to
   208  	// opt into injecting these namespaces manually.  We should reassess if this is necessary after Helm 3 is released and client side
   209  	// templating really becomes a first class function in Helm.
   210  	if component.Config.InjectNamespace && component.Config.Namespace != "" {
   211  		logger.Info(emoji.Sprintf(":syringe: Injecting namespace '%s' into manifests for component '%s'", component.Config.Namespace, component.Name))
   212  		var successes []namespaceInjectionResponse
   213  		for resp := range addNamespaceToManifests(stringManifests, component.Config.Namespace) {
   214  			// If error; return the error immediately
   215  			if resp.err != nil {
   216  				logger.Error(emoji.Sprintf(":exclamation: Encountered error while injecting namespace '%s' into manifests for component '%s':\n%s", component.Config.Namespace, component.Name, resp.err))
   217  				return stringManifests, resp.err
   218  			}
   219  
   220  			// If warning; just log the warning
   221  			if resp.warn != nil {
   222  				logger.Warn(emoji.Sprintf(":question: Encountered warning while injecting namespace '%s' into manifests for component '%s':\n%s", component.Config.Namespace, component.Name, *resp.warn))
   223  			}
   224  
   225  			// Add the manifest if one was returned
   226  			if resp.namespacedManifest != nil {
   227  				successes = append(successes, resp)
   228  			}
   229  		}
   230  
   231  		sort.Slice(successes, func(i, j int) bool {
   232  			return successes[i].index < successes[j].index
   233  		})
   234  
   235  		namespacedManifests := ""
   236  		for _, resp := range successes {
   237  			namespacedManifests += fmt.Sprintf("---\n%s\n", *resp.namespacedManifest)
   238  		}
   239  
   240  		stringManifests = namespacedManifests
   241  	}
   242  
   243  	return stringManifests, err
   244  }
   245  
   246  // Install installs the helm chart specified by the passed component and performs any
   247  // helm lifecycle events needed.
   248  func (hg *HelmGenerator) Install(c *core.Component) (err error) {
   249  	// Install the chart
   250  	if (c.Method == "helm" || c.Method == "git") && c.Source != "" && c.Path != "" {
   251  		// Download the helm chart
   252  		helmRepoPath := hg.makeHelmRepoPath(c)
   253  		switch c.Method {
   254  		case "helm":
   255  			logger.Info(emoji.Sprintf(":helicopter: Component '%s' requesting helm chart '%s' from helm repository '%s'", c.Name, c.Path, c.Source))
   256  			// Pull to a temporary directory
   257  			tmpHelmDir, err := ioutil.TempDir("", "fabrikate")
   258  			defer os.RemoveAll(tmpHelmDir)
   259  			if err != nil {
   260  				return err
   261  			}
   262  			if err = helm.Pull(c.Source, c.Path, c.Version, tmpHelmDir); err != nil {
   263  				return err
   264  			}
   265  
   266  			// Create the component directory -- deleting if it already exists
   267  			if err != nil {
   268  				return err
   269  			}
   270  			if err := os.RemoveAll(helmRepoPath); err != nil {
   271  				return err
   272  			}
   273  
   274  			// ensure the parent directory exists
   275  			if err := os.MkdirAll(filepath.Dir(helmRepoPath), 0755); err != nil {
   276  				return err
   277  			}
   278  
   279  			// Move the extracted chart from tmp to the helm_repos
   280  			extractedChartPath := path.Join(tmpHelmDir, c.Path)
   281  			if err := os.Rename(extractedChartPath, helmRepoPath); err != nil {
   282  				return err
   283  			}
   284  		case "git":
   285  			// Clone whole repo into helm repo path
   286  			logger.Info(emoji.Sprintf(":helicopter: Component '%s' requesting helm chart in path '%s' from git repository '%s'", c.Name, c.Source, c.PhysicalPath))
   287  			cloneOpts := &git.CloneOpts{
   288  				URL:    c.Source,
   289  				SHA:    c.Version,
   290  				Branch: c.Branch,
   291  				Into:   helmRepoPath,
   292  			}
   293  			if err = git.Clone(cloneOpts); err != nil {
   294  				return err
   295  			}
   296  			// Update chart dependencies in chart path -- this is manually done here but automatically done in downloadChart in the case of `method: helm`
   297  			chartPath, err := hg.getChartPath(c)
   298  			if err != nil {
   299  				return err
   300  			}
   301  			if err = helm.DependencyUpdate(chartPath); err != nil {
   302  				return err
   303  			}
   304  		}
   305  	}
   306  
   307  	return err
   308  }