github.com/databricks/cli@v0.203.0/bundle/libraries/libraries.go (about) 1 package libraries 2 3 import ( 4 "context" 5 "fmt" 6 "path/filepath" 7 "strings" 8 9 "github.com/databricks/cli/bundle" 10 "github.com/databricks/cli/bundle/config" 11 "github.com/databricks/cli/libs/cmdio" 12 "github.com/databricks/databricks-sdk-go/service/compute" 13 "github.com/databricks/databricks-sdk-go/service/jobs" 14 ) 15 16 type match struct { 17 } 18 19 func MatchWithArtifacts() bundle.Mutator { 20 return &match{} 21 } 22 23 func (a *match) Name() string { 24 return "libraries.MatchWithArtifacts" 25 } 26 27 func (a *match) Apply(ctx context.Context, b *bundle.Bundle) error { 28 r := b.Config.Resources 29 for k := range b.Config.Resources.Jobs { 30 tasks := r.Jobs[k].JobSettings.Tasks 31 for i := range tasks { 32 task := &tasks[i] 33 if isMissingRequiredLibraries(task) { 34 return fmt.Errorf("task '%s' is missing required libraries. Please include your package code in task libraries block", task.TaskKey) 35 } 36 for j := range task.Libraries { 37 lib := &task.Libraries[j] 38 err := findArtifactsAndMarkForUpload(ctx, lib, b) 39 if err != nil { 40 return err 41 } 42 } 43 } 44 } 45 return nil 46 } 47 48 func isMissingRequiredLibraries(task *jobs.Task) bool { 49 if task.Libraries != nil { 50 return false 51 } 52 53 return task.PythonWheelTask != nil || task.SparkJarTask != nil 54 } 55 56 func findLibraryMatches(lib *compute.Library, b *bundle.Bundle) ([]string, error) { 57 path := libPath(lib) 58 if path == "" { 59 return nil, nil 60 } 61 62 fullPath := filepath.Join(b.Config.Path, path) 63 return filepath.Glob(fullPath) 64 } 65 66 func findArtifactsAndMarkForUpload(ctx context.Context, lib *compute.Library, b *bundle.Bundle) error { 67 matches, err := findLibraryMatches(lib, b) 68 if err != nil { 69 return err 70 } 71 72 if len(matches) == 0 && isLocalLibrary(lib) { 73 return fmt.Errorf("no library found for %s", libPath(lib)) 74 } 75 76 for _, match := range matches { 77 af, err := findArtifactFileByLocalPath(match, b) 78 if err != nil { 79 cmdio.LogString(ctx, fmt.Sprintf("%s. Skipping %s. In order to use the library upload it manually", err.Error(), match)) 80 } else { 81 af.Libraries = append(af.Libraries, lib) 82 } 83 } 84 85 return nil 86 } 87 88 func findArtifactFileByLocalPath(path string, b *bundle.Bundle) (*config.ArtifactFile, error) { 89 for _, a := range b.Config.Artifacts { 90 for k := range a.Files { 91 if a.Files[k].Source == path { 92 return &a.Files[k], nil 93 } 94 } 95 } 96 97 return nil, fmt.Errorf("artifact file is not found for path %s", path) 98 } 99 100 func libPath(library *compute.Library) string { 101 if library.Whl != "" { 102 return library.Whl 103 } 104 if library.Jar != "" { 105 return library.Jar 106 } 107 if library.Egg != "" { 108 return library.Egg 109 } 110 111 return "" 112 } 113 114 func isLocalLibrary(library *compute.Library) bool { 115 path := libPath(library) 116 if path == "" { 117 return false 118 } 119 120 return !isDbfsPath(path) && !isWorkspacePath(path) 121 } 122 123 func isDbfsPath(path string) bool { 124 return strings.HasPrefix(path, "dbfs:/") 125 } 126 127 func isWorkspacePath(path string) bool { 128 return strings.HasPrefix(path, "/Workspace/") || 129 strings.HasPrefix(path, "/Users/") || 130 strings.HasPrefix(path, "/Shared/") 131 }