github.com/informationsea/shellflow@v0.1.3/viewlog_test.go (about)

     1  package main
     2  
     3  import (
     4  	"encoding/json"
     5  	"os"
     6  	"os/exec"
     7  	"path"
     8  	"reflect"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/informationsea/shellflow/flowscript"
    13  )
    14  
    15  func TestCollectLog(t *testing.T) {
    16  	ClearCache()
    17  	tmp, err := NewTempDir("viewlog")
    18  	if err != nil {
    19  		t.Fatalf("error: %s", err.Error())
    20  	}
    21  	shellflow := path.Join(tmp.originalCwd, "shellflow")
    22  	os.Args[0] = shellflow
    23  
    24  	defer tmp.Close()
    25  
    26  	err = os.Chdir("examples")
    27  	if err != nil {
    28  		t.Fatalf("error: %s", err.Error())
    29  	}
    30  
    31  	env := NewEnvironment()
    32  	param := make(map[string]interface{})
    33  	builder, err := parse(env, "build.sf", param)
    34  	if err != nil {
    35  		t.Fatalf("%s", err.Error())
    36  	}
    37  
    38  	gen, err := GenerateTaskScripts("build.sf", "", env, builder)
    39  	if err != nil {
    40  		t.Fatalf("%s", err.Error())
    41  	}
    42  
    43  	// -- before run
    44  	log, err := CollectLogsForOneWork(gen.workflowRoot)
    45  	if err != nil {
    46  		t.Fatalf("%s", err.Error())
    47  	}
    48  	//fmt.Printf("%s\n", log)
    49  
    50  	workflowLogRoot := Abs(path.Join(WorkflowLogDir, path.Base(gen.workflowRoot)))
    51  	expectedLogs := &WorkflowLog{
    52  		WorkflowScript:  Abs("build.sf"),
    53  		WorkflowLogRoot: workflowLogRoot,
    54  		ParameterFile:   "",
    55  		ChangedInput:    []string{},
    56  		StartDate:       log.StartDate,
    57  		JobLogs: []*JobLog{
    58  			&JobLog{
    59  				JobLogRoot:         path.Join(workflowLogRoot, "job001"),
    60  				IsStarted:          false,
    61  				IsAnyInputChanged:  false,
    62  				IsDone:             false,
    63  				IsAnyOutputChanged: false,
    64  				ExitCode:           -1,
    65  				ScriptExitCode:     -1,
    66  				ShellTask:          builder.Tasks[0],
    67  			},
    68  			&JobLog{
    69  				JobLogRoot:         path.Join(workflowLogRoot, "job002"),
    70  				IsStarted:          false,
    71  				IsAnyInputChanged:  false,
    72  				IsDone:             false,
    73  				IsAnyOutputChanged: false,
    74  				ExitCode:           -1,
    75  				ScriptExitCode:     -1,
    76  				ShellTask:          builder.Tasks[1],
    77  			},
    78  			&JobLog{
    79  				JobLogRoot:         path.Join(workflowLogRoot, "job003"),
    80  				IsStarted:          false,
    81  				IsAnyInputChanged:  false,
    82  				IsDone:             false,
    83  				IsAnyOutputChanged: false,
    84  				ExitCode:           -1,
    85  				ScriptExitCode:     -1,
    86  				ShellTask:          builder.Tasks[2],
    87  			},
    88  		},
    89  	}
    90  
    91  	for i, v := range log.JobLogs {
    92  		if !reflect.DeepEqual(v.JobLogRoot, expectedLogs.JobLogs[i].JobLogRoot) {
    93  			t.Fatalf("bad log data[%d] JobLogRoot", i)
    94  		}
    95  
    96  		if !reflect.DeepEqual(v.InputFiles, expectedLogs.JobLogs[i].InputFiles) {
    97  			t.Fatalf("bad log data[%d] InputFiles", i)
    98  		}
    99  
   100  		if !reflect.DeepEqual(v.OutputFiles, expectedLogs.JobLogs[i].OutputFiles) {
   101  			t.Fatalf("bad log data[%d] output files", i)
   102  		}
   103  
   104  		if !reflect.DeepEqual(v.ShellTask, expectedLogs.JobLogs[i].ShellTask) {
   105  			t.Fatalf("bad log data[%d] shell task", i)
   106  		}
   107  
   108  		if !reflect.DeepEqual(v, expectedLogs.JobLogs[i]) {
   109  
   110  			logsJson, e := json.MarshalIndent(v, "", "  ")
   111  			if e != nil {
   112  				t.Fatalf("error: %s", e.Error())
   113  			}
   114  			expectedLogsJson, e := json.MarshalIndent(expectedLogs.JobLogs[i], "", "  ")
   115  			if e != nil {
   116  				t.Fatalf("error: %s", e.Error())
   117  			}
   118  
   119  			t.Fatalf("bad log data[%d]: %s / expected: %s / %v / %v", i, logsJson, expectedLogsJson, reflect.DeepEqual(v.ShellTask, expectedLogs.JobLogs[i].ShellTask), reflect.DeepEqual(logsJson, expectedLogsJson))
   120  		}
   121  	}
   122  
   123  	if !reflect.DeepEqual(log, expectedLogs) {
   124  		logsJson, e := json.MarshalIndent(log, "", "  ")
   125  		if e != nil {
   126  			t.Fatalf("error: %s", e.Error())
   127  		}
   128  		expectedLogsJson, e := json.MarshalIndent(expectedLogs, "", "  ")
   129  		if e != nil {
   130  			t.Fatalf("error: %s", e.Error())
   131  		}
   132  
   133  		t.Fatalf("Bad log data: %s / expected: %s / %v", logsJson, expectedLogsJson, reflect.DeepEqual(log, &expectedLogs))
   134  	}
   135  
   136  	// -------- partial run ----------
   137  	cmd := exec.Command("/bin/bash", "-xe", gen.scripts[1].RunScriptPath)
   138  	if err != nil {
   139  		t.Fatalf("%s", err.Error())
   140  	}
   141  
   142  	err = cmd.Run()
   143  	if err != nil {
   144  		t.Fatalf("%s", err.Error())
   145  	}
   146  	// -- after one job run
   147  	log, err = CollectLogsForOneWork(gen.workflowRoot)
   148  	if err != nil {
   149  		t.Fatalf("%s", err.Error())
   150  	}
   151  	//fmt.Printf("After Partial Run: %s\n", logs)
   152  
   153  	expectedLogs = &WorkflowLog{
   154  		WorkflowScript:  Abs("build.sf"),
   155  		WorkflowLogRoot: workflowLogRoot,
   156  		ChangedInput:    []string{},
   157  		StartDate:       log.StartDate,
   158  		JobLogs: []*JobLog{
   159  			&JobLog{
   160  				JobLogRoot:         path.Join(workflowLogRoot, "job001"),
   161  				IsStarted:          true,
   162  				IsAnyInputChanged:  false,
   163  				IsDone:             true,
   164  				IsAnyOutputChanged: false,
   165  				ExitCode:           0,
   166  				ScriptExitCode:     0,
   167  				ShellTask:          builder.Tasks[0],
   168  			},
   169  			&JobLog{
   170  				JobLogRoot:         path.Join(workflowLogRoot, "job002"),
   171  				IsStarted:          false,
   172  				IsAnyInputChanged:  false,
   173  				IsDone:             false,
   174  				IsAnyOutputChanged: false,
   175  				ExitCode:           -1,
   176  				ScriptExitCode:     -1,
   177  				ShellTask:          builder.Tasks[1],
   178  			},
   179  			&JobLog{
   180  				JobLogRoot:         path.Join(workflowLogRoot, "job003"),
   181  				IsStarted:          false,
   182  				IsAnyInputChanged:  false,
   183  				IsDone:             false,
   184  				IsAnyOutputChanged: false,
   185  				ExitCode:           -1,
   186  				ScriptExitCode:     -1,
   187  				ShellTask:          builder.Tasks[2],
   188  			},
   189  		},
   190  	}
   191  
   192  	if len(log.JobLogs[0].OutputFiles) != 1 {
   193  		t.Fatalf("Invalid number of output files: %s", log)
   194  	}
   195  	expectedLogs.JobLogs[0].OutputFiles = log.JobLogs[0].OutputFiles
   196  
   197  	if len(log.JobLogs[0].InputFiles) != 2 {
   198  		t.Fatalf("Invalid number of input files: %s", log)
   199  	}
   200  	expectedLogs.JobLogs[0].InputFiles = log.JobLogs[0].InputFiles
   201  
   202  	for j, y := range log.JobLogs {
   203  		if !reflect.DeepEqual(y, expectedLogs.JobLogs[j]) {
   204  			t.Fatalf("Bad log data[%d]: %s / expected: %s", j, y, expectedLogs.JobLogs[j])
   205  		}
   206  	}
   207  
   208  	if !reflect.DeepEqual(log, expectedLogs) {
   209  		t.Fatalf("Bad log data: %s / expected: %s", log, expectedLogs)
   210  	}
   211  
   212  	// ---------- run all ------------
   213  	time.Sleep(100 * time.Millisecond)
   214  	ClearCache()
   215  	builder, err = parse(env, "build.sf", param)
   216  	if err != nil {
   217  		t.Fatalf("%s", err.Error())
   218  	}
   219  
   220  	gen, err = GenerateTaskScripts("build.sf", "", env, builder)
   221  	if err != nil {
   222  		t.Fatalf("%s", err.Error())
   223  	}
   224  
   225  	//for _, x := range gen.scripts {
   226  	//	fmt.Printf("%s %v\n", x.ScriptPath, x.Skip)
   227  	//}
   228  
   229  	workflowLogRoot = path.Join(WorkflowLogDir, path.Base(gen.workflowRoot))
   230  
   231  	//for _, x := range builder.Tasks {
   232  	//	fmt.Printf("%s\n", x)
   233  	//}
   234  
   235  	err = ExecuteLocalSingle(gen)
   236  	if err != nil {
   237  		t.Fatalf("%s", err.Error())
   238  	}
   239  
   240  	// -- after run
   241  	log, err = CollectLogsForOneWork(workflowLogRoot)
   242  	if err != nil {
   243  		t.Fatalf("%s", err.Error())
   244  	}
   245  	// fmt.Printf("After Run: %s\n", log)
   246  
   247  	expectedLogs = &WorkflowLog{
   248  		WorkflowScript:  Abs("build.sf"),
   249  		WorkflowLogRoot: workflowLogRoot,
   250  		StartDate:       log.StartDate,
   251  		ChangedInput:    []string{},
   252  		JobLogs: []*JobLog{
   253  			&JobLog{
   254  				JobLogRoot:         path.Join(workflowLogRoot, "job001"),
   255  				IsStarted:          true,
   256  				IsAnyInputChanged:  false,
   257  				IsDone:             true,
   258  				IsAnyOutputChanged: false,
   259  				ExitCode:           0,
   260  				ScriptExitCode:     0,
   261  				ShellTask:          builder.Tasks[0],
   262  			},
   263  			&JobLog{
   264  				JobLogRoot:         path.Join(workflowLogRoot, "job002"),
   265  				IsStarted:          true,
   266  				IsAnyInputChanged:  false,
   267  				IsDone:             true,
   268  				IsAnyOutputChanged: false,
   269  				ExitCode:           0,
   270  				ScriptExitCode:     0,
   271  				ShellTask:          builder.Tasks[1],
   272  			},
   273  			&JobLog{
   274  				JobLogRoot:         path.Join(workflowLogRoot, "job003"),
   275  				IsStarted:          true,
   276  				IsAnyInputChanged:  false,
   277  				IsDone:             true,
   278  				IsAnyOutputChanged: false,
   279  				ExitCode:           0,
   280  				ScriptExitCode:     0,
   281  				ShellTask:          builder.Tasks[2],
   282  			},
   283  		},
   284  	}
   285  
   286  	//for _, v := range expectedLog.JobLogs {
   287  	//	fmt.Printf("%s\n", v.ShellTask)
   288  	//}
   289  
   290  	for i, v := range log.JobLogs {
   291  		//fmt.Printf("Actual: %s\nExpected: %s\n\n", v.ShellTask, expectedLogs[0].JobLogs[i])
   292  
   293  		expectedOutputs := flowscript.NewStringSet()
   294  		for _, u := range v.OutputFiles {
   295  			expectedOutputs.Add(u.Relpath)
   296  		}
   297  		if !reflect.DeepEqual(expectedOutputs.Array(), v.ShellTask.CreatingFiles.Array()) {
   298  			t.Fatalf("bad log: %d : %s", i, v)
   299  		}
   300  
   301  		expectedInputs := flowscript.NewStringSet()
   302  		for _, u := range v.InputFiles {
   303  			expectedInputs.Add(u.Relpath)
   304  		}
   305  		if !reflect.DeepEqual(expectedInputs.Array(), v.ShellTask.DependentFiles.Array()) {
   306  			t.Fatalf("bad log: %d : %s", i, v)
   307  		}
   308  
   309  		expectedLogs.JobLogs[i].OutputFiles = v.OutputFiles
   310  		expectedLogs.JobLogs[i].InputFiles = v.InputFiles
   311  
   312  		if !reflect.DeepEqual(v, expectedLogs.JobLogs[i]) {
   313  			u := expectedLogs.JobLogs[i]
   314  
   315  			t.Fatalf(`bad log data: %d: %s / expected: %s`, i, v, u)
   316  		}
   317  	}
   318  
   319  	if !reflect.DeepEqual(log, expectedLogs) {
   320  		t.Fatalf("Bad log data: %s / expected: %s", log, expectedLogs)
   321  	}
   322  
   323  	// --------- change some file ---------------
   324  	{
   325  		file, err := os.OpenFile("hello.c", os.O_APPEND|os.O_WRONLY, 0644)
   326  		if err != nil {
   327  			t.Fatalf("%s", err.Error())
   328  		}
   329  		defer file.Close()
   330  		_, err = file.WriteString("\n")
   331  		if err != nil {
   332  			t.Fatalf("%s", err.Error())
   333  		}
   334  	}
   335  
   336  	// clear cache
   337  	ClearCache()
   338  
   339  	// -- after changed
   340  
   341  	log, err = CollectLogsForOneWork(gen.workflowRoot)
   342  	if err != nil {
   343  		t.Fatalf("%s", err.Error())
   344  	}
   345  	//fmt.Printf("After Changed: %s\n", logs)
   346  
   347  	expectedLogs = &WorkflowLog{
   348  		WorkflowScript:  Abs("build.sf"),
   349  		WorkflowLogRoot: workflowLogRoot,
   350  		ChangedInput:    []string{"hello.c"},
   351  		StartDate:       log.StartDate,
   352  		JobLogs: []*JobLog{
   353  			&JobLog{
   354  				JobLogRoot:         path.Join(workflowLogRoot, "job001"),
   355  				IsStarted:          true,
   356  				IsAnyInputChanged:  true,
   357  				IsDone:             true,
   358  				IsAnyOutputChanged: false,
   359  				ExitCode:           0,
   360  				ScriptExitCode:     0,
   361  				ShellTask:          builder.Tasks[0],
   362  			},
   363  			&JobLog{
   364  				JobLogRoot:         path.Join(workflowLogRoot, "job002"),
   365  				IsStarted:          true,
   366  				IsAnyInputChanged:  false,
   367  				IsDone:             true,
   368  				IsAnyOutputChanged: false,
   369  				ExitCode:           0,
   370  				ScriptExitCode:     0,
   371  				ShellTask:          builder.Tasks[1],
   372  			},
   373  			&JobLog{
   374  				JobLogRoot:         path.Join(workflowLogRoot, "job003"),
   375  				IsStarted:          true,
   376  				IsAnyInputChanged:  false,
   377  				IsDone:             true,
   378  				IsAnyOutputChanged: false,
   379  				ExitCode:           0,
   380  				ScriptExitCode:     0,
   381  				ShellTask:          builder.Tasks[2],
   382  			},
   383  		},
   384  	}
   385  
   386  	for i, v := range log.JobLogs {
   387  		expectedOutputs := flowscript.NewStringSet()
   388  		for _, u := range v.OutputFiles {
   389  			expectedOutputs.Add(u.Relpath)
   390  		}
   391  		if !reflect.DeepEqual(expectedOutputs.Array(), v.ShellTask.CreatingFiles.Array()) {
   392  			t.Fatalf("bad log: %d : %s", i, v)
   393  		}
   394  
   395  		expectedInputs := flowscript.NewStringSet()
   396  		for _, u := range v.InputFiles {
   397  			expectedInputs.Add(u.Relpath)
   398  		}
   399  		if !reflect.DeepEqual(expectedInputs.Array(), v.ShellTask.DependentFiles.Array()) {
   400  			t.Fatalf("bad log: %d : %s", i, v)
   401  		}
   402  
   403  		expectedLogs.JobLogs[i].OutputFiles = v.OutputFiles
   404  		expectedLogs.JobLogs[i].InputFiles = v.InputFiles
   405  	}
   406  
   407  	for j, y := range log.JobLogs {
   408  		if !reflect.DeepEqual(y, expectedLogs.JobLogs[j]) {
   409  			t.Fatalf("Bad log data[%d]: %s / expected: %s", j, y, expectedLogs.JobLogs[j])
   410  		}
   411  	}
   412  
   413  	if !reflect.DeepEqual(log, expectedLogs) {
   414  		t.Fatalf("Bad log data: %s / expected: %s", log, expectedLogs)
   415  	}
   416  
   417  	ViewLog(false, false)
   418  }