kythe.io@v0.0.68-0.20240422202219-7225dbc01741/kythe/go/extractors/config/runextractor/compdb/compdb.go (about) 1 /* 2 * Copyright 2018 The Kythe Authors. All rights reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 // Package compdb contains functionality necessary for extracting from a 18 // compile_commands.json file. 19 package compdb // import "kythe.io/kythe/go/extractors/config/runextractor/compdb" 20 21 import ( 22 "context" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io/ioutil" 27 "os" 28 "os/exec" 29 "path/filepath" 30 "strings" 31 "sync" 32 "sync/atomic" 33 34 "kythe.io/kythe/go/util/log" 35 36 "bitbucket.org/creachadair/shell" 37 "golang.org/x/sync/semaphore" 38 ) 39 40 // A compileCommand holds the decoded arguments of a LLVM compilation database 41 // JSON command spec. 42 type compileCommand struct { 43 Arguments []string 44 Command string 45 Directory string 46 } 47 48 func (cc *compileCommand) asCommand() string { 49 if len(cc.Arguments) > 0 { 50 return shell.Join(cc.Arguments) 51 } 52 return cc.Command 53 } 54 55 func (cc *compileCommand) asArguments() ([]string, bool) { 56 if len(cc.Arguments) > 0 { 57 return cc.Arguments, true 58 } 59 return shell.Split(cc.Command) 60 } 61 62 // ExtractOptions holds additional options related to compilation DB extraction. 63 type ExtractOptions struct { 64 ExtraArguments []string // additional arguments to pass to the extractor 65 } 66 67 // ExtractCompilations runs the specified extractor over each compilation record 68 // found in the compile_commands.json file at path. 69 func ExtractCompilations(ctx context.Context, extractor, path string, opts *ExtractOptions) error { 70 commands, err := readCommands(path) 71 if err != nil { 72 return err 73 } 74 env, err := extractorEnv() 75 if err != nil { 76 return err 77 } 78 79 var failCount uint64 80 sem := semaphore.NewWeighted(128) // Limit concurrency. 81 var wg sync.WaitGroup 82 wg.Add(len(commands)) 83 for _, entry := range commands { 84 go func(entry compileCommand) { 85 defer wg.Done() 86 if err := sem.Acquire(ctx, 1); err != nil { 87 atomic.AddUint64(&failCount, 1) 88 log.ErrorContext(ctx, err) 89 return 90 } 91 defer sem.Release(1) 92 93 if err := extractOne(ctx, extractor, entry, env, opts); err != nil { 94 // Log error, but continue processing other compilations. 95 atomic.AddUint64(&failCount, 1) 96 log.ErrorContextf(ctx, "extracting compilation with command '%s': %v", entry.asCommand(), err) 97 } 98 }(entry) 99 } 100 wg.Wait() 101 102 if failCount != 0 { 103 return fmt.Errorf("Failed to extract %d compilations", failCount) 104 } 105 106 return nil 107 } 108 109 // extractOne invokes the extractor for the given compileCommand. 110 func extractOne(ctx context.Context, extractor string, cc compileCommand, env []string, opts *ExtractOptions) error { 111 cmd := exec.CommandContext(ctx, extractor, "--with_executable") 112 args, ok := cc.asArguments() 113 if !ok { 114 return fmt.Errorf("unable to split command line") 115 } 116 // Wire through any additional arguments from the command line. 117 args = append(args, opts.extraArguments()...) 118 cmd.Args = append(cmd.Args, args...) 119 var err error 120 cmd.Dir, err = filepath.Abs(cc.Directory) 121 if err != nil { 122 return fmt.Errorf("unable to resolve cmake directory: %v", err) 123 } 124 cmd.Env = env 125 if _, err := cmd.Output(); err != nil { 126 if exit, ok := err.(*exec.ExitError); ok { 127 return fmt.Errorf("error running extractor: %v (%s)", exit, exit.Stderr) 128 } 129 return fmt.Errorf("error running extractor: %v", err) 130 } 131 return nil 132 } 133 134 // readCommands reads the JSON file at path into a slice of compileCommands. 135 func readCommands(path string) ([]compileCommand, error) { 136 data, err := ioutil.ReadFile(path) 137 if err != nil { 138 return nil, err 139 } 140 var commands []compileCommand 141 if err := json.Unmarshal(data, &commands); err != nil { 142 return nil, err 143 } 144 return commands, nil 145 } 146 147 // extractorEnv copies the existing environment and modifies it to be suitable for an extractor invocation. 148 func extractorEnv() ([]string, error) { 149 var env []string 150 outputFound := false 151 for _, value := range os.Environ() { 152 parts := strings.SplitN(value, "=", 2) 153 // Until kzip support comes along, we only support writing to a single directory so strip these options. 154 if parts[0] == "KYTHE_INDEX_PACK" || parts[0] == "KYTHE_OUTPUT_FILE" { 155 continue 156 } else if parts[0] == "KYTHE_OUTPUT_DIRECTORY" { 157 // Remap KYTHE_OUTPUT_DIRECTORY to be an absolute path. 158 output, err := filepath.Abs(parts[1]) 159 if err != nil { 160 return nil, err 161 } 162 outputFound = true 163 env = append(env, "KYTHE_OUTPUT_DIRECTORY="+output) 164 } else { 165 // Otherwise, preserve the environment unchanged. 166 env = append(env, value) 167 } 168 169 } 170 if !outputFound { 171 return nil, errors.New("missing mandatory environment variable: KYTHE_OUTPUT_DIRECTORY") 172 } 173 return env, nil 174 } 175 176 // extraArguments returns a slice of additional arguments to provide to the extractor. 177 func (o *ExtractOptions) extraArguments() []string { 178 if o != nil && len(o.ExtraArguments) > 0 { 179 return o.ExtraArguments 180 } 181 return nil 182 }