github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/client/driver/java.go (about) 1 package driver 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/json" 7 "fmt" 8 "log" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "runtime" 13 "strings" 14 "syscall" 15 "time" 16 17 "github.com/hashicorp/go-multierror" 18 "github.com/hashicorp/go-plugin" 19 "github.com/mitchellh/mapstructure" 20 21 "github.com/hashicorp/nomad/client/config" 22 "github.com/hashicorp/nomad/client/driver/env" 23 "github.com/hashicorp/nomad/client/driver/executor" 24 dstructs "github.com/hashicorp/nomad/client/driver/structs" 25 "github.com/hashicorp/nomad/client/fingerprint" 26 cstructs "github.com/hashicorp/nomad/client/structs" 27 "github.com/hashicorp/nomad/helper" 28 "github.com/hashicorp/nomad/helper/fields" 29 "github.com/hashicorp/nomad/nomad/structs" 30 ) 31 32 const ( 33 // The key populated in Node Attributes to indicate presence of the Java 34 // driver 35 javaDriverAttr = "driver.java" 36 ) 37 38 // JavaDriver is a simple driver to execute applications packaged in Jars. 39 // It literally just fork/execs tasks with the java command. 40 type JavaDriver struct { 41 DriverContext 42 fingerprint.StaticFingerprinter 43 44 // A tri-state boolean to know if the fingerprinting has happened and 45 // whether it has been successful 46 fingerprintSuccess *bool 47 } 48 49 type JavaDriverConfig struct { 50 Class string `mapstructure:"class"` 51 ClassPath string `mapstructure:"class_path"` 52 JarPath string `mapstructure:"jar_path"` 53 JvmOpts []string `mapstructure:"jvm_options"` 54 Args []string `mapstructure:"args"` 55 } 56 57 // javaHandle is returned from Start/Open as a handle to the PID 58 type javaHandle struct { 59 pluginClient *plugin.Client 60 userPid int 61 executor executor.Executor 62 isolationConfig *dstructs.IsolationConfig 63 taskDir string 64 65 killTimeout time.Duration 66 maxKillTimeout time.Duration 67 version string 68 logger *log.Logger 69 waitCh chan *dstructs.WaitResult 70 doneCh chan struct{} 71 } 72 73 // NewJavaDriver is used to create a new exec driver 74 func NewJavaDriver(ctx *DriverContext) Driver { 75 return &JavaDriver{DriverContext: *ctx} 76 } 77 78 // Validate is used to validate the driver configuration 79 func (d *JavaDriver) Validate(config map[string]interface{}) error { 80 fd := &fields.FieldData{ 81 Raw: config, 82 Schema: map[string]*fields.FieldSchema{ 83 "class": { 84 Type: fields.TypeString, 85 }, 86 "class_path": { 87 Type: fields.TypeString, 88 }, 89 "jar_path": { 90 Type: fields.TypeString, 91 }, 92 "jvm_options": { 93 Type: fields.TypeArray, 94 }, 95 "args": { 96 Type: fields.TypeArray, 97 }, 98 }, 99 } 100 101 if err := fd.Validate(); err != nil { 102 return err 103 } 104 105 return nil 106 } 107 108 func (d *JavaDriver) Abilities() DriverAbilities { 109 return DriverAbilities{ 110 SendSignals: true, 111 Exec: true, 112 } 113 } 114 115 func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { 116 // Only enable if we are root and cgroups are mounted when running on linux systems. 117 if runtime.GOOS == "linux" && (syscall.Geteuid() != 0 || !cgroupsMounted(node)) { 118 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 119 d.logger.Printf("[DEBUG] driver.java: root priviledges and mounted cgroups required on linux, disabling") 120 } 121 delete(node.Attributes, "driver.java") 122 d.fingerprintSuccess = helper.BoolToPtr(false) 123 return false, nil 124 } 125 126 // Find java version 127 var out bytes.Buffer 128 var erOut bytes.Buffer 129 cmd := exec.Command("java", "-version") 130 cmd.Stdout = &out 131 cmd.Stderr = &erOut 132 err := cmd.Run() 133 if err != nil { 134 // assume Java wasn't found 135 delete(node.Attributes, javaDriverAttr) 136 d.fingerprintSuccess = helper.BoolToPtr(false) 137 return false, nil 138 } 139 140 // 'java -version' returns output on Stderr typically. 141 // Check stdout, but it's probably empty 142 var infoString string 143 if out.String() != "" { 144 infoString = out.String() 145 } 146 147 if erOut.String() != "" { 148 infoString = erOut.String() 149 } 150 151 if infoString == "" { 152 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 153 d.logger.Println("[WARN] driver.java: error parsing Java version information, aborting") 154 } 155 delete(node.Attributes, javaDriverAttr) 156 d.fingerprintSuccess = helper.BoolToPtr(false) 157 return false, nil 158 } 159 160 // Assume 'java -version' returns 3 lines: 161 // java version "1.6.0_36" 162 // OpenJDK Runtime Environment (IcedTea6 1.13.8) (6b36-1.13.8-0ubuntu1~12.04) 163 // OpenJDK 64-Bit Server VM (build 23.25-b01, mixed mode) 164 // Each line is terminated by \n 165 info := strings.Split(infoString, "\n") 166 versionString := info[0] 167 versionString = strings.TrimPrefix(versionString, "java version ") 168 versionString = strings.Trim(versionString, "\"") 169 node.Attributes[javaDriverAttr] = "1" 170 node.Attributes["driver.java.version"] = versionString 171 node.Attributes["driver.java.runtime"] = info[1] 172 node.Attributes["driver.java.vm"] = info[2] 173 d.fingerprintSuccess = helper.BoolToPtr(true) 174 175 return true, nil 176 } 177 178 func (d *JavaDriver) Prestart(*ExecContext, *structs.Task) (*PrestartResponse, error) { 179 return nil, nil 180 } 181 182 func NewJavaDriverConfig(task *structs.Task, env *env.TaskEnv) (*JavaDriverConfig, error) { 183 var driverConfig JavaDriverConfig 184 if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { 185 return nil, err 186 } 187 188 // Interpolate everything 189 driverConfig.Class = env.ReplaceEnv(driverConfig.Class) 190 driverConfig.ClassPath = env.ReplaceEnv(driverConfig.ClassPath) 191 driverConfig.JarPath = env.ReplaceEnv(driverConfig.JarPath) 192 driverConfig.JvmOpts = env.ParseAndReplace(driverConfig.JvmOpts) 193 driverConfig.Args = env.ParseAndReplace(driverConfig.Args) 194 195 // Validate 196 jarSpecified := driverConfig.JarPath != "" 197 classSpecified := driverConfig.Class != "" 198 if !jarSpecified && !classSpecified { 199 return nil, fmt.Errorf("jar_path or class must be specified") 200 } 201 202 return &driverConfig, nil 203 } 204 205 func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) { 206 driverConfig, err := NewJavaDriverConfig(task, ctx.TaskEnv) 207 if err != nil { 208 return nil, err 209 } 210 211 args := []string{} 212 213 // Look for jvm options 214 if len(driverConfig.JvmOpts) != 0 { 215 d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", driverConfig.JvmOpts) 216 args = append(args, driverConfig.JvmOpts...) 217 } 218 219 // Add the classpath 220 if driverConfig.ClassPath != "" { 221 args = append(args, "-cp", driverConfig.ClassPath) 222 } 223 224 // Add the jar 225 if driverConfig.JarPath != "" { 226 args = append(args, "-jar", driverConfig.JarPath) 227 } 228 229 // Add the class 230 if driverConfig.Class != "" { 231 args = append(args, driverConfig.Class) 232 } 233 234 // Add any args 235 if len(driverConfig.Args) != 0 { 236 args = append(args, driverConfig.Args...) 237 } 238 239 pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") 240 executorConfig := &dstructs.ExecutorConfig{ 241 LogFile: pluginLogFile, 242 LogLevel: d.config.LogLevel, 243 } 244 245 execIntf, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) 246 if err != nil { 247 return nil, err 248 } 249 250 // Set the context 251 executorCtx := &executor.ExecutorContext{ 252 TaskEnv: ctx.TaskEnv, 253 Driver: "java", 254 Task: task, 255 TaskDir: ctx.TaskDir.Dir, 256 LogDir: ctx.TaskDir.LogDir, 257 } 258 if err := execIntf.SetContext(executorCtx); err != nil { 259 pluginClient.Kill() 260 return nil, fmt.Errorf("failed to set executor context: %v", err) 261 } 262 263 absPath, err := GetAbsolutePath("java") 264 if err != nil { 265 return nil, err 266 } 267 268 taskKillSignal, err := getTaskKillSignal(task.KillSignal) 269 if err != nil { 270 return nil, err 271 } 272 273 execCmd := &executor.ExecCommand{ 274 Cmd: absPath, 275 Args: args, 276 FSIsolation: true, 277 ResourceLimits: true, 278 User: getExecutorUser(task), 279 TaskKillSignal: taskKillSignal, 280 } 281 ps, err := execIntf.LaunchCmd(execCmd) 282 if err != nil { 283 pluginClient.Kill() 284 return nil, err 285 } 286 d.logger.Printf("[DEBUG] driver.java: started process with pid: %v", ps.Pid) 287 288 // Return a driver handle 289 maxKill := d.DriverContext.config.MaxKillTimeout 290 h := &javaHandle{ 291 pluginClient: pluginClient, 292 executor: execIntf, 293 userPid: ps.Pid, 294 isolationConfig: ps.IsolationConfig, 295 taskDir: ctx.TaskDir.Dir, 296 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 297 maxKillTimeout: maxKill, 298 version: d.config.Version.VersionNumber(), 299 logger: d.logger, 300 doneCh: make(chan struct{}), 301 waitCh: make(chan *dstructs.WaitResult, 1), 302 } 303 go h.run() 304 return &StartResponse{Handle: h}, nil 305 } 306 307 func (d *JavaDriver) Cleanup(*ExecContext, *CreatedResources) error { return nil } 308 309 type javaId struct { 310 Version string 311 KillTimeout time.Duration 312 MaxKillTimeout time.Duration 313 PluginConfig *PluginReattachConfig 314 IsolationConfig *dstructs.IsolationConfig 315 TaskDir string 316 UserPid int 317 } 318 319 func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 320 id := &javaId{} 321 if err := json.Unmarshal([]byte(handleID), id); err != nil { 322 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 323 } 324 325 pluginConfig := &plugin.ClientConfig{ 326 Reattach: id.PluginConfig.PluginConfig(), 327 } 328 exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) 329 if err != nil { 330 merrs := new(multierror.Error) 331 merrs.Errors = append(merrs.Errors, err) 332 d.logger.Println("[ERR] driver.java: error connecting to plugin so destroying plugin pid and user pid") 333 if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { 334 merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e)) 335 } 336 if id.IsolationConfig != nil { 337 ePid := pluginConfig.Reattach.Pid 338 if e := executor.ClientCleanup(id.IsolationConfig, ePid); e != nil { 339 merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying resource container failed: %v", e)) 340 } 341 } 342 343 return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil()) 344 } 345 346 ver, _ := exec.Version() 347 d.logger.Printf("[DEBUG] driver.java: version of executor: %v", ver.Version) 348 349 // Return a driver handle 350 h := &javaHandle{ 351 pluginClient: pluginClient, 352 executor: exec, 353 userPid: id.UserPid, 354 isolationConfig: id.IsolationConfig, 355 logger: d.logger, 356 version: id.Version, 357 killTimeout: id.KillTimeout, 358 maxKillTimeout: id.MaxKillTimeout, 359 doneCh: make(chan struct{}), 360 waitCh: make(chan *dstructs.WaitResult, 1), 361 } 362 go h.run() 363 return h, nil 364 } 365 366 func (h *javaHandle) ID() string { 367 id := javaId{ 368 Version: h.version, 369 KillTimeout: h.killTimeout, 370 MaxKillTimeout: h.maxKillTimeout, 371 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 372 UserPid: h.userPid, 373 IsolationConfig: h.isolationConfig, 374 TaskDir: h.taskDir, 375 } 376 377 data, err := json.Marshal(id) 378 if err != nil { 379 h.logger.Printf("[ERR] driver.java: failed to marshal ID to JSON: %s", err) 380 } 381 return string(data) 382 } 383 384 func (h *javaHandle) WaitCh() chan *dstructs.WaitResult { 385 return h.waitCh 386 } 387 388 func (h *javaHandle) Update(task *structs.Task) error { 389 // Store the updated kill timeout. 390 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 391 h.executor.UpdateTask(task) 392 393 // Update is not possible 394 return nil 395 } 396 397 func (h *javaHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) { 398 deadline, ok := ctx.Deadline() 399 if !ok { 400 // No deadline set on context; default to 1 minute 401 deadline = time.Now().Add(time.Minute) 402 } 403 return h.executor.Exec(deadline, cmd, args) 404 } 405 406 func (h *javaHandle) Signal(s os.Signal) error { 407 return h.executor.Signal(s) 408 } 409 410 func (h *javaHandle) Kill() error { 411 if err := h.executor.ShutDown(); err != nil { 412 if h.pluginClient.Exited() { 413 return nil 414 } 415 return fmt.Errorf("executor Shutdown failed: %v", err) 416 } 417 418 select { 419 case <-h.doneCh: 420 case <-time.After(h.killTimeout): 421 if h.pluginClient.Exited() { 422 break 423 } 424 if err := h.executor.Exit(); err != nil { 425 return fmt.Errorf("executor Exit failed: %v", err) 426 } 427 428 } 429 return nil 430 } 431 432 func (h *javaHandle) Stats() (*cstructs.TaskResourceUsage, error) { 433 return h.executor.Stats() 434 } 435 436 func (h *javaHandle) run() { 437 ps, werr := h.executor.Wait() 438 close(h.doneCh) 439 if ps.ExitCode == 0 && werr != nil { 440 if h.isolationConfig != nil { 441 ePid := h.pluginClient.ReattachConfig().Pid 442 if e := executor.ClientCleanup(h.isolationConfig, ePid); e != nil { 443 h.logger.Printf("[ERR] driver.java: destroying resource container failed: %v", e) 444 } 445 } else { 446 if e := killProcess(h.userPid); e != nil { 447 h.logger.Printf("[ERR] driver.java: error killing user process: %v", e) 448 } 449 } 450 } 451 452 // Exit the executor 453 h.executor.Exit() 454 h.pluginClient.Kill() 455 456 // Send the results 457 h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr} 458 close(h.waitCh) 459 }