github.com/tendermint/tmlibs@v0.9.0/autofile/group_test.go (about) 1 package autofile 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "strconv" 10 "strings" 11 "testing" 12 13 "github.com/stretchr/testify/assert" 14 "github.com/stretchr/testify/require" 15 16 cmn "github.com/tendermint/tmlibs/common" 17 ) 18 19 // NOTE: Returned group has ticker stopped 20 func createTestGroup(t *testing.T, headSizeLimit int64) *Group { 21 testID := cmn.RandStr(12) 22 testDir := "_test_" + testID 23 err := cmn.EnsureDir(testDir, 0700) 24 require.NoError(t, err, "Error creating dir") 25 headPath := testDir + "/myfile" 26 g, err := OpenGroup(headPath) 27 require.NoError(t, err, "Error opening Group") 28 g.SetHeadSizeLimit(headSizeLimit) 29 g.stopTicker() 30 require.NotEqual(t, nil, g, "Failed to create Group") 31 return g 32 } 33 34 func destroyTestGroup(t *testing.T, g *Group) { 35 g.Close() 36 err := os.RemoveAll(g.Dir) 37 require.NoError(t, err, "Error removing test Group directory") 38 } 39 40 func assertGroupInfo(t *testing.T, gInfo GroupInfo, minIndex, maxIndex int, totalSize, headSize int64) { 41 assert.Equal(t, minIndex, gInfo.MinIndex) 42 assert.Equal(t, maxIndex, gInfo.MaxIndex) 43 assert.Equal(t, totalSize, gInfo.TotalSize) 44 assert.Equal(t, headSize, gInfo.HeadSize) 45 } 46 47 func TestCheckHeadSizeLimit(t *testing.T) { 48 g := createTestGroup(t, 1000*1000) 49 50 // At first, there are no files. 51 assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 0, 0) 52 53 // Write 1000 bytes 999 times. 54 for i := 0; i < 999; i++ { 55 err := g.WriteLine(cmn.RandStr(999)) 56 require.NoError(t, err, "Error appending to head") 57 } 58 g.Flush() 59 assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) 60 61 // Even calling checkHeadSizeLimit manually won't rotate it. 62 g.checkHeadSizeLimit() 63 assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) 64 65 // Write 1000 more bytes. 66 err := g.WriteLine(cmn.RandStr(999)) 67 require.NoError(t, err, "Error appending to head") 68 g.Flush() 69 70 // Calling checkHeadSizeLimit this time rolls it. 71 g.checkHeadSizeLimit() 72 assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0) 73 74 // Write 1000 more bytes. 75 err = g.WriteLine(cmn.RandStr(999)) 76 require.NoError(t, err, "Error appending to head") 77 g.Flush() 78 79 // Calling checkHeadSizeLimit does nothing. 80 g.checkHeadSizeLimit() 81 assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1001000, 1000) 82 83 // Write 1000 bytes 999 times. 84 for i := 0; i < 999; i++ { 85 err = g.WriteLine(cmn.RandStr(999)) 86 require.NoError(t, err, "Error appending to head") 87 } 88 g.Flush() 89 assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 2000000, 1000000) 90 91 // Calling checkHeadSizeLimit rolls it again. 92 g.checkHeadSizeLimit() 93 assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2000000, 0) 94 95 // Write 1000 more bytes. 96 _, err = g.Head.Write([]byte(cmn.RandStr(999) + "\n")) 97 require.NoError(t, err, "Error appending to head") 98 g.Flush() 99 assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000) 100 101 // Calling checkHeadSizeLimit does nothing. 102 g.checkHeadSizeLimit() 103 assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000) 104 105 // Cleanup 106 destroyTestGroup(t, g) 107 } 108 109 func TestSearch(t *testing.T) { 110 g := createTestGroup(t, 10*1000) 111 112 // Create some files in the group that have several INFO lines in them. 113 // Try to put the INFO lines in various spots. 114 for i := 0; i < 100; i++ { 115 // The random junk at the end ensures that this INFO linen 116 // is equally likely to show up at the end. 117 _, err := g.Head.Write([]byte(fmt.Sprintf("INFO %v %v\n", i, cmn.RandStr(123)))) 118 require.NoError(t, err, "Failed to write to head") 119 g.checkHeadSizeLimit() 120 for j := 0; j < 10; j++ { 121 _, err1 := g.Head.Write([]byte(cmn.RandStr(123) + "\n")) 122 require.NoError(t, err1, "Failed to write to head") 123 g.checkHeadSizeLimit() 124 } 125 } 126 127 // Create a search func that searches for line 128 makeSearchFunc := func(target int) SearchFunc { 129 return func(line string) (int, error) { 130 parts := strings.Split(line, " ") 131 if len(parts) != 3 { 132 return -1, errors.New("Line did not have 3 parts") 133 } 134 i, err := strconv.Atoi(parts[1]) 135 if err != nil { 136 return -1, errors.New("Failed to parse INFO: " + err.Error()) 137 } 138 if target < i { 139 return 1, nil 140 } else if target == i { 141 return 0, nil 142 } else { 143 return -1, nil 144 } 145 } 146 } 147 148 // Now search for each number 149 for i := 0; i < 100; i++ { 150 t.Log("Testing for i", i) 151 gr, match, err := g.Search("INFO", makeSearchFunc(i)) 152 require.NoError(t, err, "Failed to search for line") 153 assert.True(t, match, "Expected Search to return exact match") 154 line, err := gr.ReadLine() 155 require.NoError(t, err, "Failed to read line after search") 156 if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", i)) { 157 t.Fatal("Failed to get correct line") 158 } 159 // Make sure we can continue to read from there. 160 cur := i + 1 161 for { 162 line, err := gr.ReadLine() 163 if err == io.EOF { 164 if cur == 99+1 { 165 // OK! 166 break 167 } else { 168 t.Fatal("Got EOF after the wrong INFO #") 169 } 170 } else if err != nil { 171 t.Fatal("Error reading line", err) 172 } 173 if !strings.HasPrefix(line, "INFO ") { 174 continue 175 } 176 if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) { 177 t.Fatalf("Unexpected INFO #. Expected %v got:\n%v", cur, line) 178 } 179 cur++ 180 } 181 gr.Close() 182 } 183 184 // Now search for something that is too small. 185 // We should get the first available line. 186 { 187 gr, match, err := g.Search("INFO", makeSearchFunc(-999)) 188 require.NoError(t, err, "Failed to search for line") 189 assert.False(t, match, "Expected Search to not return exact match") 190 line, err := gr.ReadLine() 191 require.NoError(t, err, "Failed to read line after search") 192 if !strings.HasPrefix(line, "INFO 0 ") { 193 t.Error("Failed to fetch correct line, which is the earliest INFO") 194 } 195 err = gr.Close() 196 require.NoError(t, err, "Failed to close GroupReader") 197 } 198 199 // Now search for something that is too large. 200 // We should get an EOF error. 201 { 202 gr, _, err := g.Search("INFO", makeSearchFunc(999)) 203 assert.Equal(t, io.EOF, err) 204 assert.Nil(t, gr) 205 } 206 207 // Cleanup 208 destroyTestGroup(t, g) 209 } 210 211 func TestRotateFile(t *testing.T) { 212 g := createTestGroup(t, 0) 213 g.WriteLine("Line 1") 214 g.WriteLine("Line 2") 215 g.WriteLine("Line 3") 216 g.Flush() 217 g.RotateFile() 218 g.WriteLine("Line 4") 219 g.WriteLine("Line 5") 220 g.WriteLine("Line 6") 221 g.Flush() 222 223 // Read g.Head.Path+"000" 224 body1, err := ioutil.ReadFile(g.Head.Path + ".000") 225 assert.NoError(t, err, "Failed to read first rolled file") 226 if string(body1) != "Line 1\nLine 2\nLine 3\n" { 227 t.Errorf("Got unexpected contents: [%v]", string(body1)) 228 } 229 230 // Read g.Head.Path 231 body2, err := ioutil.ReadFile(g.Head.Path) 232 assert.NoError(t, err, "Failed to read first rolled file") 233 if string(body2) != "Line 4\nLine 5\nLine 6\n" { 234 t.Errorf("Got unexpected contents: [%v]", string(body2)) 235 } 236 237 // Cleanup 238 destroyTestGroup(t, g) 239 } 240 241 func TestFindLast1(t *testing.T) { 242 g := createTestGroup(t, 0) 243 244 g.WriteLine("Line 1") 245 g.WriteLine("Line 2") 246 g.WriteLine("# a") 247 g.WriteLine("Line 3") 248 g.Flush() 249 g.RotateFile() 250 g.WriteLine("Line 4") 251 g.WriteLine("Line 5") 252 g.WriteLine("Line 6") 253 g.WriteLine("# b") 254 g.Flush() 255 256 match, found, err := g.FindLast("#") 257 assert.NoError(t, err) 258 assert.True(t, found) 259 assert.Equal(t, "# b", match) 260 261 // Cleanup 262 destroyTestGroup(t, g) 263 } 264 265 func TestFindLast2(t *testing.T) { 266 g := createTestGroup(t, 0) 267 268 g.WriteLine("Line 1") 269 g.WriteLine("Line 2") 270 g.WriteLine("Line 3") 271 g.Flush() 272 g.RotateFile() 273 g.WriteLine("# a") 274 g.WriteLine("Line 4") 275 g.WriteLine("Line 5") 276 g.WriteLine("# b") 277 g.WriteLine("Line 6") 278 g.Flush() 279 280 match, found, err := g.FindLast("#") 281 assert.NoError(t, err) 282 assert.True(t, found) 283 assert.Equal(t, "# b", match) 284 285 // Cleanup 286 destroyTestGroup(t, g) 287 } 288 289 func TestFindLast3(t *testing.T) { 290 g := createTestGroup(t, 0) 291 292 g.WriteLine("Line 1") 293 g.WriteLine("# a") 294 g.WriteLine("Line 2") 295 g.WriteLine("# b") 296 g.WriteLine("Line 3") 297 g.Flush() 298 g.RotateFile() 299 g.WriteLine("Line 4") 300 g.WriteLine("Line 5") 301 g.WriteLine("Line 6") 302 g.Flush() 303 304 match, found, err := g.FindLast("#") 305 assert.NoError(t, err) 306 assert.True(t, found) 307 assert.Equal(t, "# b", match) 308 309 // Cleanup 310 destroyTestGroup(t, g) 311 } 312 313 func TestFindLast4(t *testing.T) { 314 g := createTestGroup(t, 0) 315 316 g.WriteLine("Line 1") 317 g.WriteLine("Line 2") 318 g.WriteLine("Line 3") 319 g.Flush() 320 g.RotateFile() 321 g.WriteLine("Line 4") 322 g.WriteLine("Line 5") 323 g.WriteLine("Line 6") 324 g.Flush() 325 326 match, found, err := g.FindLast("#") 327 assert.NoError(t, err) 328 assert.False(t, found) 329 assert.Empty(t, match) 330 331 // Cleanup 332 destroyTestGroup(t, g) 333 } 334 335 func TestWrite(t *testing.T) { 336 g := createTestGroup(t, 0) 337 338 written := []byte("Medusa") 339 g.Write(written) 340 g.Flush() 341 342 read := make([]byte, len(written)) 343 gr, err := g.NewReader(0) 344 require.NoError(t, err, "failed to create reader") 345 346 _, err = gr.Read(read) 347 assert.NoError(t, err, "failed to read data") 348 assert.Equal(t, written, read) 349 350 // Cleanup 351 destroyTestGroup(t, g) 352 } 353 354 // test that Read reads the required amount of bytes from all the files in the 355 // group and returns no error if n == size of the given slice. 356 func TestGroupReaderRead(t *testing.T) { 357 g := createTestGroup(t, 0) 358 359 professor := []byte("Professor Monster") 360 g.Write(professor) 361 g.Flush() 362 g.RotateFile() 363 frankenstein := []byte("Frankenstein's Monster") 364 g.Write(frankenstein) 365 g.Flush() 366 367 totalWrittenLength := len(professor) + len(frankenstein) 368 read := make([]byte, totalWrittenLength) 369 gr, err := g.NewReader(0) 370 require.NoError(t, err, "failed to create reader") 371 372 n, err := gr.Read(read) 373 assert.NoError(t, err, "failed to read data") 374 assert.Equal(t, totalWrittenLength, n, "not enough bytes read") 375 professorPlusFrankenstein := professor 376 professorPlusFrankenstein = append(professorPlusFrankenstein, frankenstein...) 377 assert.Equal(t, professorPlusFrankenstein, read) 378 379 // Cleanup 380 destroyTestGroup(t, g) 381 } 382 383 // test that Read returns an error if number of bytes read < size of 384 // the given slice. Subsequent call should return 0, io.EOF. 385 func TestGroupReaderRead2(t *testing.T) { 386 g := createTestGroup(t, 0) 387 388 professor := []byte("Professor Monster") 389 g.Write(professor) 390 g.Flush() 391 g.RotateFile() 392 frankenstein := []byte("Frankenstein's Monster") 393 frankensteinPart := []byte("Frankenstein") 394 g.Write(frankensteinPart) // note writing only a part 395 g.Flush() 396 397 totalLength := len(professor) + len(frankenstein) 398 read := make([]byte, totalLength) 399 gr, err := g.NewReader(0) 400 require.NoError(t, err, "failed to create reader") 401 402 // 1) n < (size of the given slice), io.EOF 403 n, err := gr.Read(read) 404 assert.Equal(t, io.EOF, err) 405 assert.Equal(t, len(professor)+len(frankensteinPart), n, "Read more/less bytes than it is in the group") 406 407 // 2) 0, io.EOF 408 n, err = gr.Read([]byte("0")) 409 assert.Equal(t, io.EOF, err) 410 assert.Equal(t, 0, n) 411 412 // Cleanup 413 destroyTestGroup(t, g) 414 } 415 416 func TestMinIndex(t *testing.T) { 417 g := createTestGroup(t, 0) 418 419 assert.Zero(t, g.MinIndex(), "MinIndex should be zero at the beginning") 420 421 // Cleanup 422 destroyTestGroup(t, g) 423 } 424 425 func TestMaxIndex(t *testing.T) { 426 g := createTestGroup(t, 0) 427 428 assert.Zero(t, g.MaxIndex(), "MaxIndex should be zero at the beginning") 429 430 g.WriteLine("Line 1") 431 g.Flush() 432 g.RotateFile() 433 434 assert.Equal(t, 1, g.MaxIndex(), "MaxIndex should point to the last file") 435 436 // Cleanup 437 destroyTestGroup(t, g) 438 }