github.com/dshekhar95/sub_dgraph@v0.0.0-20230424164411-6be28e40bbf1/dgraph/cmd/live/load-uids/load_test.go (about) 1 /* 2 * Copyright 2017-2022 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "context" 21 "fmt" 22 "log" 23 "os" 24 "path/filepath" 25 "regexp" 26 "runtime" 27 "strings" 28 "testing" 29 "time" 30 31 "github.com/stretchr/testify/assert" 32 "github.com/stretchr/testify/require" 33 34 "github.com/dgraph-io/dgo/v210" 35 "github.com/dgraph-io/dgo/v210/protos/api" 36 "github.com/dgraph-io/dgraph/ee" 37 "github.com/dgraph-io/dgraph/testutil" 38 "github.com/dgraph-io/dgraph/x" 39 ) 40 41 var ( 42 testDataDir string 43 dg *dgo.Dgraph 44 ) 45 46 var ( 47 alphaService string 48 zeroService string 49 alphaName string 50 alphaExportPath string 51 localExportPath = "./export_copy" 52 ) 53 54 func checkDifferentUid(t *testing.T, wantMap, gotMap map[string]interface{}) { 55 require.NotEqual(t, gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"], 56 wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"], 57 "new uid was assigned") 58 59 gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1 60 wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1 61 testutil.CompareJSONMaps(t, wantMap, gotMap) 62 } 63 64 func checkUpsertLoadedData(t *testing.T) { 65 resp, err := dg.NewTxn().Query(context.Background(), ` 66 { 67 q(func: eq(xid, "m.1234")) { 68 xid 69 name 70 value 71 } 72 } 73 `) 74 require.NoError(t, err) 75 76 gotMap := testutil.UnmarshalJSON(t, string(resp.GetJson())) 77 wantMap := testutil.UnmarshalJSON(t, ` 78 { 79 "q": [ 80 { 81 "xid": "m.1234", 82 "name": "name 1234", 83 "value": "value 1234" 84 } 85 ] 86 } 87 `) 88 89 testutil.CompareJSONMaps(t, wantMap, gotMap) 90 } 91 92 func TestLiveLoadUpsertAtOnce(t *testing.T) { 93 testutil.DropAll(t, dg) 94 95 file := testDataDir + "/xid_a.rdf, " + testDataDir + "/xid_b.rdf" 96 97 pipeline := [][]string{ 98 {testutil.DgraphBinaryPath(), "live", 99 "--schema", testDataDir + "/xid.schema", "--files", file, "--alpha", 100 alphaService, "--zero", zeroService, 101 "--creds", "user=groot;password=password;", "-U", "xid"}, 102 } 103 _, err := testutil.Pipeline(pipeline) 104 require.NoError(t, err, "live loading JSON file exited with error") 105 106 checkUpsertLoadedData(t) 107 } 108 109 func TestLiveLoadUpsert(t *testing.T) { 110 testutil.DropAll(t, dg) 111 112 pipeline := [][]string{ 113 {testutil.DgraphBinaryPath(), "live", 114 "--schema", testDataDir + "/xid.schema", "--files", testDataDir + "/xid_a.rdf", 115 "--alpha", alphaService, "--zero", zeroService, 116 "--creds", "user=groot;password=password;", "-U", "xid"}, 117 } 118 _, err := testutil.Pipeline(pipeline) 119 require.NoError(t, err, "live loading JSON file exited with error") 120 121 pipeline = [][]string{ 122 {testutil.DgraphBinaryPath(), "live", 123 "--schema", testDataDir + "/xid.schema", "--files", testDataDir + "/xid_b.rdf", 124 "--alpha", alphaService, "--zero", zeroService, 125 "--creds", "user=groot;password=password;", "-U", "xid"}, 126 } 127 _, err = testutil.Pipeline(pipeline) 128 require.NoError(t, err, "live loading JSON file exited with error") 129 130 checkUpsertLoadedData(t) 131 } 132 133 func checkLoadedData(t *testing.T, newUids bool) { 134 resp, err := dg.NewTxn().Query(context.Background(), ` 135 { 136 q(func: anyofterms(name, "Homer")) { 137 uid 138 name 139 age 140 role 141 } 142 } 143 `) 144 require.NoError(t, err) 145 146 gotMap := testutil.UnmarshalJSON(t, string(resp.GetJson())) 147 wantMap := testutil.UnmarshalJSON(t, ` 148 { 149 "q": [ 150 { 151 "uid": "0x2001", 152 "name": "Homer", 153 "age": 38, 154 "role": "father" 155 } 156 ] 157 } 158 `) 159 if newUids { 160 checkDifferentUid(t, wantMap, gotMap) 161 } else { 162 testutil.CompareJSONMaps(t, wantMap, gotMap) 163 } 164 165 resp, err = dg.NewTxn().Query(context.Background(), ` 166 { 167 q(func: anyofterms(name, "Maggie")) { 168 uid 169 name 170 role 171 carries 172 } 173 } 174 `) 175 require.NoError(t, err) 176 177 gotMap = testutil.UnmarshalJSON(t, string(resp.GetJson())) 178 wantMap = testutil.UnmarshalJSON(t, ` 179 { 180 "q": [ 181 { 182 "uid": "0x3003", 183 "name": "Maggie", 184 "role": "daughter", 185 "carries": "pacifier" 186 } 187 ] 188 } 189 `) 190 if newUids { 191 checkDifferentUid(t, wantMap, gotMap) 192 } else { 193 testutil.CompareJSONMaps(t, wantMap, gotMap) 194 } 195 } 196 197 func TestLiveLoadJsonUidKeep(t *testing.T) { 198 testutil.DropAll(t, dg) 199 200 pipeline := [][]string{ 201 {testutil.DgraphBinaryPath(), "live", 202 "--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json", 203 "--alpha", alphaService, "--zero", zeroService, 204 "--creds", "user=groot;password=password;"}, 205 } 206 _, err := testutil.Pipeline(pipeline) 207 require.NoError(t, err, "live loading JSON file exited with error") 208 209 checkLoadedData(t, false) 210 } 211 212 func TestLiveLoadJsonUidDiscard(t *testing.T) { 213 testutil.DropAll(t, dg) 214 215 pipeline := [][]string{ 216 {testutil.DgraphBinaryPath(), "live", "--new_uids", 217 "--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json", 218 "--alpha", alphaService, "--zero", zeroService, 219 "--creds", "user=groot;password=password;"}, 220 } 221 _, err := testutil.Pipeline(pipeline) 222 require.NoError(t, err, "live loading JSON file exited with error") 223 224 checkLoadedData(t, true) 225 } 226 227 func TestLiveLoadRdfUidKeep(t *testing.T) { 228 testutil.DropAll(t, dg) 229 230 pipeline := [][]string{ 231 {testutil.DgraphBinaryPath(), "live", 232 "--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf", 233 "--alpha", alphaService, "--zero", zeroService, 234 "--creds", "user=groot;password=password;"}, 235 } 236 _, err := testutil.Pipeline(pipeline) 237 require.NoError(t, err, "live loading JSON file exited with error") 238 239 checkLoadedData(t, false) 240 } 241 242 func TestLiveLoadRdfUidDiscard(t *testing.T) { 243 testutil.DropAll(t, dg) 244 245 pipeline := [][]string{ 246 {testutil.DgraphBinaryPath(), "live", "--new_uids", 247 "--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf", 248 "--alpha", alphaService, "--zero", zeroService, 249 "--creds", "user=groot;password=password;"}, 250 } 251 _, err := testutil.Pipeline(pipeline) 252 require.NoError(t, err, "live loading JSON file exited with error") 253 254 checkLoadedData(t, true) 255 } 256 257 func TestLiveLoadExportedSchema(t *testing.T) { 258 testutil.DropAll(t, dg) 259 260 // initiate export 261 params := &testutil.GraphQLParams{ 262 Query: ` 263 mutation { 264 export(input: {format: "rdf"}) { 265 response { 266 code 267 message 268 } 269 } 270 }`, 271 } 272 token := testutil.GrootHttpLogin("http://" + testutil.SockAddrHttp + "/admin") 273 resp := testutil.MakeGQLRequestWithAccessJwt(t, params, token.AccessJwt) 274 require.Nilf(t, resp.Errors, resp.Errors.Error()) 275 276 // wait a bit to be sure export is complete 277 time.Sleep(8 * time.Second) 278 279 // copy the export files from docker 280 exportId, groupId := copyExportToLocalFs(t) 281 282 // then loading the exported files should work 283 pipeline := [][]string{ 284 {testutil.DgraphBinaryPath(), "live", 285 "--schema", localExportPath + "/" + exportId + "/" + groupId + ".schema.gz", 286 "--files", localExportPath + "/" + exportId + "/" + groupId + ".rdf.gz", 287 "--encryption", 288 ee.BuildEncFlag(testDataDir + "/../../../../ee/enc/test-fixtures/enc-key"), 289 "--alpha", alphaService, "--zero", zeroService, 290 "--creds", "user=groot;password=password;"}, 291 } 292 _, err := testutil.Pipeline(pipeline) 293 require.NoError(t, err, "live loading exported schema exited with error") 294 295 // cleanup copied export files 296 require.NoError(t, os.RemoveAll(localExportPath), "Error removing export copy directory") 297 } 298 299 func copyExportToLocalFs(t *testing.T) (string, string) { 300 require.NoError(t, os.RemoveAll(localExportPath), "Error removing directory") 301 require.NoError(t, testutil.DockerCp(alphaExportPath, localExportPath), 302 "Error copying files from docker container") 303 304 childDirs, err := os.ReadDir(localExportPath) 305 require.NoError(t, err, "Couldn't read local export copy directory") 306 require.True(t, len(childDirs) > 0, "Local export copy directory is empty!!!") 307 308 exportFiles, err := os.ReadDir(localExportPath + "/" + childDirs[0].Name()) 309 require.NoError(t, err, "Couldn't read child of local export copy directory") 310 require.True(t, len(exportFiles) > 0, "no exported files found!!!") 311 312 groupId := strings.Split(exportFiles[0].Name(), ".")[0] 313 314 return childDirs[0].Name(), groupId 315 } 316 317 func extractErrLine(output string) string { 318 m := regexp.MustCompile(`Error while processing(.)*(rdf|json):`) 319 errLine := m.FindString(output) 320 return errLine 321 } 322 323 func TestLiveLoadFileName(t *testing.T) { 324 testutil.DropAll(t, dg) 325 326 pipeline := [][]string{ 327 {testutil.DgraphBinaryPath(), "live", 328 "--files", testDataDir + "/correct1.rdf," + testDataDir + "/errored1.rdf", 329 "--alpha", alphaService, "--zero", zeroService, 330 "--creds", "user=groot;password=password;"}, 331 } 332 333 out, err := testutil.Pipeline(pipeline) 334 require.Error(t, err, "error expected: live loader exited with no error") 335 errLine := extractErrLine(out) 336 errLineExp := fmt.Sprintf(`Error while processing data file %s/errored1.rdf:`, testDataDir) 337 require.Equal(t, errLineExp, errLine, "incorrect name for errored file") 338 } 339 340 func TestLiveLoadFileNameMultipleErrored(t *testing.T) { 341 testutil.DropAll(t, dg) 342 343 pipeline := [][]string{ 344 {testutil.DgraphBinaryPath(), "live", 345 "--files", testDataDir + "/correct1.rdf," + testDataDir + "/errored1.rdf," + 346 testDataDir + "/errored2.rdf", "--alpha", alphaService, "--zero", zeroService, 347 "--creds", "user=groot;password=password;"}, 348 } 349 350 out, err := testutil.Pipeline(pipeline) 351 require.Error(t, err, "error expected: live loader exited with no error") 352 errLine := extractErrLine(out) 353 errLineExp1 := fmt.Sprintf(`Error while processing data file %s/errored1.rdf:`, testDataDir) 354 errLineExp2 := fmt.Sprintf(`Error while processing data file %s/errored2.rdf:`, testDataDir) 355 assert.Contains(t, []string{errLineExp1, errLineExp2}, errLine, "incorrect name for errored file") 356 } 357 358 func TestLiveLoadFileNameMultipleCorrect(t *testing.T) { 359 testutil.DropAll(t, dg) 360 361 pipeline := [][]string{ 362 {testutil.DgraphBinaryPath(), "live", 363 "--files", testDataDir + "/correct1.rdf," + testDataDir + "/correct2.rdf," + 364 testDataDir + "/errored1.rdf", "--alpha", alphaService, "--zero", zeroService, 365 "--creds", "user=groot;password=password;"}, 366 } 367 368 out, err := testutil.Pipeline(pipeline) 369 require.Error(t, err, "error expected: live loader exited with no error") 370 errLine := extractErrLine(out) 371 errLineExp := fmt.Sprintf(`Error while processing data file %s/errored1.rdf:`, testDataDir) 372 require.Equal(t, errLineExp, errLine, "incorrect name for errored file") 373 } 374 375 func TestMain(m *testing.M) { 376 alphaName = testutil.Instance 377 alphaService = testutil.SockAddr 378 zeroService = testutil.SockAddrZero 379 380 x.AssertTrue(strings.Count(alphaName, "_") == 2) 381 left := strings.Index(alphaName, "_") 382 right := strings.LastIndex(alphaName, "_") 383 alphaExportPath = alphaName + ":/data/" + alphaName[left+1:right] + "/export" 384 fmt.Printf("alphaExportPath: %s\n", alphaExportPath) 385 386 _, thisFile, _, _ := runtime.Caller(0) 387 testDataDir = filepath.Dir(thisFile) 388 fmt.Printf("Using test data dir: %s\n", testDataDir) 389 390 var err error 391 dg, err = testutil.DgraphClientWithGroot(testutil.SockAddr) 392 if err != nil { 393 log.Fatalf("Error while getting a dgraph client: %v", err) 394 } 395 x.Check(dg.Alter( 396 context.Background(), &api.Operation{DropAll: true})) 397 398 // Try to create any files in a dedicated temp directory that gets cleaned up 399 // instead of all over /tmp or the working directory. 400 tmpDir, err := os.MkdirTemp("", "test.tmp-") 401 x.Check(err) 402 os.Chdir(tmpDir) 403 defer os.RemoveAll(tmpDir) 404 405 os.Exit(m.Run()) 406 }