github.com/databricks/cli@v0.203.0/bundle/libraries/libraries.go (about)

     1  package libraries
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"path/filepath"
     7  	"strings"
     8  
     9  	"github.com/databricks/cli/bundle"
    10  	"github.com/databricks/cli/bundle/config"
    11  	"github.com/databricks/cli/libs/cmdio"
    12  	"github.com/databricks/databricks-sdk-go/service/compute"
    13  	"github.com/databricks/databricks-sdk-go/service/jobs"
    14  )
    15  
    16  type match struct {
    17  }
    18  
    19  func MatchWithArtifacts() bundle.Mutator {
    20  	return &match{}
    21  }
    22  
    23  func (a *match) Name() string {
    24  	return "libraries.MatchWithArtifacts"
    25  }
    26  
    27  func (a *match) Apply(ctx context.Context, b *bundle.Bundle) error {
    28  	r := b.Config.Resources
    29  	for k := range b.Config.Resources.Jobs {
    30  		tasks := r.Jobs[k].JobSettings.Tasks
    31  		for i := range tasks {
    32  			task := &tasks[i]
    33  			if isMissingRequiredLibraries(task) {
    34  				return fmt.Errorf("task '%s' is missing required libraries. Please include your package code in task libraries block", task.TaskKey)
    35  			}
    36  			for j := range task.Libraries {
    37  				lib := &task.Libraries[j]
    38  				err := findArtifactsAndMarkForUpload(ctx, lib, b)
    39  				if err != nil {
    40  					return err
    41  				}
    42  			}
    43  		}
    44  	}
    45  	return nil
    46  }
    47  
    48  func isMissingRequiredLibraries(task *jobs.Task) bool {
    49  	if task.Libraries != nil {
    50  		return false
    51  	}
    52  
    53  	return task.PythonWheelTask != nil || task.SparkJarTask != nil
    54  }
    55  
    56  func findLibraryMatches(lib *compute.Library, b *bundle.Bundle) ([]string, error) {
    57  	path := libPath(lib)
    58  	if path == "" {
    59  		return nil, nil
    60  	}
    61  
    62  	fullPath := filepath.Join(b.Config.Path, path)
    63  	return filepath.Glob(fullPath)
    64  }
    65  
    66  func findArtifactsAndMarkForUpload(ctx context.Context, lib *compute.Library, b *bundle.Bundle) error {
    67  	matches, err := findLibraryMatches(lib, b)
    68  	if err != nil {
    69  		return err
    70  	}
    71  
    72  	if len(matches) == 0 && isLocalLibrary(lib) {
    73  		return fmt.Errorf("no library found for %s", libPath(lib))
    74  	}
    75  
    76  	for _, match := range matches {
    77  		af, err := findArtifactFileByLocalPath(match, b)
    78  		if err != nil {
    79  			cmdio.LogString(ctx, fmt.Sprintf("%s. Skipping %s. In order to use the library upload it manually", err.Error(), match))
    80  		} else {
    81  			af.Libraries = append(af.Libraries, lib)
    82  		}
    83  	}
    84  
    85  	return nil
    86  }
    87  
    88  func findArtifactFileByLocalPath(path string, b *bundle.Bundle) (*config.ArtifactFile, error) {
    89  	for _, a := range b.Config.Artifacts {
    90  		for k := range a.Files {
    91  			if a.Files[k].Source == path {
    92  				return &a.Files[k], nil
    93  			}
    94  		}
    95  	}
    96  
    97  	return nil, fmt.Errorf("artifact file is not found for path %s", path)
    98  }
    99  
   100  func libPath(library *compute.Library) string {
   101  	if library.Whl != "" {
   102  		return library.Whl
   103  	}
   104  	if library.Jar != "" {
   105  		return library.Jar
   106  	}
   107  	if library.Egg != "" {
   108  		return library.Egg
   109  	}
   110  
   111  	return ""
   112  }
   113  
   114  func isLocalLibrary(library *compute.Library) bool {
   115  	path := libPath(library)
   116  	if path == "" {
   117  		return false
   118  	}
   119  
   120  	return !isDbfsPath(path) && !isWorkspacePath(path)
   121  }
   122  
   123  func isDbfsPath(path string) bool {
   124  	return strings.HasPrefix(path, "dbfs:/")
   125  }
   126  
   127  func isWorkspacePath(path string) bool {
   128  	return strings.HasPrefix(path, "/Workspace/") ||
   129  		strings.HasPrefix(path, "/Users/") ||
   130  		strings.HasPrefix(path, "/Shared/")
   131  }