github.com/anthropics/anthropic-sdk-go@v1.70.1/lib/environments/memories_test.go (about) 1 package environments 2 3 // The memory-store sync contract. 4 // 5 // An agent keeps a folder of notes on disk. The same folder lives on the 6 // server. Each sync makes the two agree: each side gets the other's 7 // changes, and when both changed the same file the server wins. 8 // 9 // Each test tells one story with two actors — `local` is the agent's folder 10 // on disk, `server` is the remote copy. Paths are the same string on both 11 // sides. server.Received holds only what the sync sent; arranging remote 12 // state never touches it. 13 14 import ( 15 "bytes" 16 "context" 17 "crypto/sha256" 18 "encoding/hex" 19 "encoding/json" 20 "fmt" 21 "io" 22 "log/slog" 23 "net/http" 24 "net/http/httptest" 25 "net/url" 26 "os" 27 "path/filepath" 28 "slices" 29 "strings" 30 "sync" 31 "sync/atomic" 32 "testing" 33 "time" 34 35 anthropic "github.com/anthropics/anthropic-sdk-go" 36 "github.com/anthropics/anthropic-sdk-go/internal/filestore" 37 "github.com/anthropics/anthropic-sdk-go/option" 38 "github.com/stretchr/testify/require" 39 ) 40 41 func shaHex(content string) string { 42 sum := sha256.Sum256([]byte(content)) 43 return hex.EncodeToString(sum[:]) 44 } 45 46 // Requests recorded on memoryServer.Received, in domain terms: paths and 47 // content. `was` is the pre-image content the request's precondition guards 48 // on — the builder computes the sha. 49 50 func created(path, content string) string { return "create " + path + " " + content } 51 52 func updated(path, content, was string) string { 53 return "update " + path + " " + content + " if=" + shaHex(was) 54 } 55 56 func deletedReq(path, was string) string { return "delete " + path + " if=" + shaHex(was) } 57 58 // memoryServer is the test's view of one remote memory store named "notes". 59 // 60 // Files is its current state (path → content); Received is every write the 61 // code under test sent it. Arranging via Write / Delete changes Files without 62 // touching Received — only the sync's own requests are recorded. It serves 63 // both the sessions lookup (recording Retrieves) and the memory_stores 64 // endpoints, so one server backs the pure SessionMemoryStores tests and the 65 // worker e2e tests alike. 66 type memoryServer struct { 67 t *testing.T 68 69 mu sync.Mutex 70 Files map[string]string 71 Received []string 72 Retrieves []string 73 // ContentFetches is every single-memory content fetch (the sync's 74 // content pass), by path, in arrival order — a test asserting "no 75 // content moved" checks this stays empty. 76 ContentFetches []string 77 78 access string // "" (read_write) or "read_only" 79 OnUpdate func() // race hook: runs before an update is processed 80 onList func() // race hook: runs after a listing is served; set via SetOnList 81 // onFetch is a race hook that runs as each content fetch arrives, with 82 // the server mutex held (Files is safe to touch); a non-zero return 83 // fails the fetch with that status. Set via SetOnFetch. 84 onFetch func(path string) int 85 // uploadHook runs as each create or update arrives, with the memory's 86 // path, before the request is recorded and without the server mutex — 87 // block in it to hold an upload open. A request whose client gave up 88 // while held is dropped unrecorded. Set via SetUploadHook. 89 uploadHook func(ctx context.Context, path string) 90 // FailUpdates answers every update with a 500 — recorded, so the attempt 91 // is visible, but never applied. 92 FailUpdates bool 93 // FailUploads answers every create or update with this status, before it 94 // is recorded or applied. Zero means none. 95 FailUploads int 96 // MaxContentBytes answers any create/update whose content exceeds it 97 // with a 400 — recorded, never applied. Zero means no cap. 98 MaxContentBytes int 99 // Broken attaches a second store, "memstore_broken", listed before the 100 // healthy one, whose listing yields one memory and then explodes on the 101 // next page — so a half-download is observable. 102 Broken bool 103 // MountPath is emitted on the "notes" store resource (null when empty). 104 MountPath string 105 // Noise adds a non-store resource to the session and a memory_prefix 106 // rollup to every listing. 107 Noise bool 108 // NoStores drops the "notes" store from the session, leaving one with no 109 // memory attached at all. 110 NoStores bool 111 } 112 113 func newMemoryServer(t *testing.T, initial map[string]string, access string) *memoryServer { 114 files := map[string]string{} 115 for k, v := range initial { 116 files[k] = v 117 } 118 return &memoryServer{t: t, Files: files, access: access} 119 } 120 121 func (m *memoryServer) Write(path, content string) { 122 m.mu.Lock() 123 defer m.mu.Unlock() 124 m.Files[path] = content 125 } 126 127 func (m *memoryServer) Delete(path string) { 128 m.mu.Lock() 129 defer m.mu.Unlock() 130 delete(m.Files, path) 131 } 132 133 func (m *memoryServer) SetOnList(fn func()) { 134 m.mu.Lock() 135 defer m.mu.Unlock() 136 m.onList = fn 137 } 138 139 func (m *memoryServer) SetOnFetch(fn func(path string) int) { 140 m.mu.Lock() 141 defer m.mu.Unlock() 142 m.onFetch = fn 143 } 144 145 func (m *memoryServer) SetUploadHook(fn func(ctx context.Context, path string)) { 146 m.mu.Lock() 147 defer m.mu.Unlock() 148 m.uploadHook = fn 149 } 150 151 func (m *memoryServer) ReceivedSnapshot() []string { 152 m.mu.Lock() 153 defer m.mu.Unlock() 154 return append([]string(nil), m.Received...) 155 } 156 157 // ContentFetchesSnapshot returns the content fetches so far, sorted — they 158 // fan out, so arrival order is not deterministic. 159 func (m *memoryServer) ContentFetchesSnapshot() []string { 160 m.mu.Lock() 161 defer m.mu.Unlock() 162 return slices.Sorted(slices.Values(m.ContentFetches)) 163 } 164 165 func (m *memoryServer) ClearContentFetches() { 166 m.mu.Lock() 167 defer m.mu.Unlock() 168 m.ContentFetches = nil 169 } 170 171 // memID round-trips a path through the URL-safe id the fake hands out. 172 func memID(path string) string { return "mem_" + hex.EncodeToString([]byte(path)) } 173 174 func memIDPath(id string) string { 175 raw, _ := hex.DecodeString(strings.TrimPrefix(id, "mem_")) 176 return string(raw) 177 } 178 179 // itemJSON renders one memory the way the API does: the basic view carries 180 // the sha but no content, the full view carries both. 181 func (m *memoryServer) itemJSON(path string, full bool) map[string]any { 182 content := m.Files[path] 183 item := map[string]any{ 184 "type": "memory", "id": memID(path), "path": "/" + path, 185 "content_sha256": shaHex(content), 186 "content_size_bytes": len(content), 187 "created_at": "2026-05-11T12:00:00Z", "updated_at": "2026-05-11T12:00:00Z", 188 "memory_store_id": "memstore_notes", "memory_version_id": "memver_1", 189 } 190 if full { 191 item["content"] = content 192 } 193 return item 194 } 195 196 func isFullView(r *http.Request) bool { return r.URL.Query().Get("view") == "full" } 197 198 func (m *memoryServer) sessionJSON() string { 199 var access any 200 if m.access != "" { 201 access = m.access 202 } 203 resources := []map[string]any{} 204 if m.Broken { 205 resources = append(resources, map[string]any{ 206 "type": "memory_store", "memory_store_id": "memstore_broken", 207 "mount_path": nil, "name": "broken", "access": nil, 208 }) 209 } 210 if m.Noise { 211 resources = append(resources, map[string]any{"type": "file", "file_id": "file_1"}) 212 } 213 var mount any 214 if m.MountPath != "" { 215 mount = m.MountPath 216 } 217 if !m.NoStores { 218 resources = append(resources, map[string]any{ 219 "type": "memory_store", "memory_store_id": "memstore_notes", 220 "mount_path": mount, "name": "notes", "access": access, 221 }) 222 } 223 body, _ := json.Marshal(map[string]any{ 224 "id": "s1", 225 "agent": map[string]any{"skills": []any{}}, 226 "resources": resources, 227 }) 228 return string(body) 229 } 230 231 func statusJSON(w http.ResponseWriter, code int) { 232 w.Header().Set("Content-Type", "application/json") 233 w.WriteHeader(code) 234 fmt.Fprintf(w, `{"type":"error","error":{"type":"api_error","message":"status %d"}}`, code) 235 } 236 237 // uploadTarget reports the memory path a create or update request is for; 238 // false for every other request. The body is drained and rewound first: a 239 // create names its path there, and net/http only notices a client that 240 // gave up once the body has been read to the end. 241 func uploadTarget(r *http.Request) (string, bool) { 242 if r.Method != http.MethodPost { 243 return "", false 244 } 245 body, _ := io.ReadAll(r.Body) 246 r.Body = io.NopCloser(bytes.NewReader(body)) 247 rest := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/v1/memory_stores/"), "/", 3) 248 switch { 249 case len(rest) == 2 && rest[1] == "memories": 250 var in struct{ Path string } 251 _ = json.Unmarshal(body, &in) 252 return strings.TrimLeft(in.Path, "/"), true 253 case len(rest) == 3 && rest[1] == "memories": 254 id, _ := url.PathUnescape(rest[2]) 255 return memIDPath(strings.SplitN(id, "?", 2)[0]), true 256 default: 257 return "", false 258 } 259 } 260 261 // handleMemories serves the /v1/memory_stores/... endpoints against the 262 // in-memory store. 263 func (m *memoryServer) handleMemories(w http.ResponseWriter, r *http.Request) { 264 if rel, ok := uploadTarget(r); ok { 265 m.mu.Lock() 266 hook := m.uploadHook 267 m.mu.Unlock() 268 if hook != nil { 269 hook(r.Context(), rel) 270 if r.Context().Err() != nil { 271 return 272 } 273 } 274 if m.FailUploads != 0 { 275 statusJSON(w, m.FailUploads) 276 return 277 } 278 } 279 m.mu.Lock() 280 defer m.mu.Unlock() 281 w.Header().Set("Content-Type", "application/json") 282 rest := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/v1/memory_stores/"), "/", 3) 283 switch { 284 case rest[0] == "memstore_broken": 285 // One memory is listed, then the pager explodes on the next page. 286 if r.URL.Query().Get("page") == "" { 287 half := map[string]any{ 288 "type": "memory", "id": "mem_half", "path": "/half.md", 289 "content_sha256": shaHex("partial"), 290 "content_size_bytes": len("partial"), 291 "created_at": "2026-05-11T12:00:00Z", "updated_at": "2026-05-11T12:00:00Z", 292 "memory_store_id": "memstore_broken", "memory_version_id": "memver_1", 293 } 294 if isFullView(r) { 295 half["content"] = "partial" 296 } 297 data := []any{} 298 if m.Noise { 299 data = append(data, map[string]any{"type": "memory_prefix", "path": "/projects/"}) 300 } 301 data = append(data, half) 302 _ = json.NewEncoder(w).Encode(map[string]any{"data": data, "next_page": "boom"}) 303 return 304 } 305 statusJSON(w, 500) 306 case len(rest) == 2 && rest[1] == "memories" && r.Method == http.MethodGet: 307 items := make([]map[string]any, 0, len(m.Files)+1) 308 if m.Noise { 309 // Depth-limited listings roll directories up as prefixes; only 310 // "memory" items carry content. 311 items = append(items, map[string]any{"type": "memory_prefix", "path": "/projects/"}) 312 } 313 full := isFullView(r) 314 for path := range m.Files { 315 items = append(items, m.itemJSON(path, full)) 316 } 317 _ = json.NewEncoder(w).Encode(map[string]any{"data": items, "next_page": nil}) 318 if m.onList != nil { 319 m.onList() 320 } 321 case len(rest) == 2 && rest[1] == "memories" && r.Method == http.MethodPost: 322 var in struct{ Path, Content string } 323 body, _ := io.ReadAll(r.Body) 324 require.NoError(m.t, json.Unmarshal(body, &in)) 325 bare := strings.TrimLeft(in.Path, "/") 326 m.Received = append(m.Received, created(bare, in.Content)) 327 if m.MaxContentBytes > 0 && len(in.Content) > m.MaxContentBytes { 328 statusJSON(w, 400) 329 return 330 } 331 m.Files[bare] = in.Content 332 _ = json.NewEncoder(w).Encode(m.itemJSON(bare, true)) 333 case len(rest) == 3 && rest[1] == "memories": 334 id, _ := url.PathUnescape(rest[2]) 335 id = strings.SplitN(id, "?", 2)[0] 336 path := memIDPath(id) 337 switch r.Method { 338 case http.MethodGet: // single-memory content fetch 339 m.ContentFetches = append(m.ContentFetches, path) 340 if m.onFetch != nil { 341 if code := m.onFetch(path); code != 0 { 342 statusJSON(w, code) 343 return 344 } 345 } 346 if _, ok := m.Files[path]; !ok { 347 statusJSON(w, 404) 348 return 349 } 350 _ = json.NewEncoder(w).Encode(m.itemJSON(path, isFullView(r))) 351 case http.MethodPost: // update 352 if m.OnUpdate != nil { 353 m.OnUpdate() 354 } 355 var in struct { 356 Content string 357 Precondition struct { 358 ContentSha256 string `json:"content_sha256"` 359 } 360 } 361 body, _ := io.ReadAll(r.Body) 362 require.NoError(m.t, json.Unmarshal(body, &in)) 363 m.Received = append(m.Received, "update "+path+" "+in.Content+" if="+in.Precondition.ContentSha256) 364 if m.MaxContentBytes > 0 && len(in.Content) > m.MaxContentBytes { 365 statusJSON(w, 400) 366 return 367 } 368 if m.FailUpdates { 369 statusJSON(w, 500) 370 return 371 } 372 current, ok := m.Files[path] 373 if !ok { 374 statusJSON(w, 404) 375 return 376 } 377 if shaHex(current) != in.Precondition.ContentSha256 { 378 statusJSON(w, 409) 379 return 380 } 381 m.Files[path] = in.Content 382 _ = json.NewEncoder(w).Encode(m.itemJSON(path, true)) 383 case http.MethodDelete: 384 expected := r.URL.Query().Get("expected_content_sha256") 385 m.Received = append(m.Received, "delete "+path+" if="+expected) 386 current, ok := m.Files[path] 387 if !ok { 388 statusJSON(w, 404) 389 return 390 } 391 if expected != "" && shaHex(current) != expected { 392 statusJSON(w, 409) 393 return 394 } 395 delete(m.Files, path) 396 _ = json.NewEncoder(w).Encode(map[string]any{"id": id, "type": "memory_deleted"}) 397 default: 398 statusJSON(w, 405) 399 } 400 default: 401 m.t.Errorf("unexpected memory_stores request: %s %s", r.Method, r.URL.Path) 402 statusJSON(w, 404) 403 } 404 } 405 406 func (m *memoryServer) serveHTTP(w http.ResponseWriter, r *http.Request) { 407 switch { 408 case strings.HasPrefix(r.URL.Path, "/v1/memory_stores/"): 409 m.handleMemories(w, r) 410 case strings.HasPrefix(r.URL.Path, "/v1/sessions/"): 411 m.mu.Lock() 412 m.Retrieves = append(m.Retrieves, strings.TrimPrefix(r.URL.Path, "/v1/sessions/")) 413 m.mu.Unlock() 414 w.Header().Set("Content-Type", "application/json") 415 _, _ = w.Write([]byte(m.sessionJSON())) 416 default: 417 m.t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) 418 statusJSON(w, 404) 419 } 420 } 421 422 // fetchSession retrieves the fake server's session. Download takes that 423 // snapshot and never looks the session up itself, so every test fetches it the 424 // way the worker does — once, up front. 425 func fetchSession(t *testing.T, client anthropic.Client) *anthropic.BetaManagedAgentsSession { 426 t.Helper() 427 session, err := client.Beta.Sessions.Get(context.Background(), "s1", anthropic.BetaSessionGetParams{}) 428 require.NoError(t, err) 429 return session 430 } 431 432 // mustStores is NewSessionMemoryStores for options a test knows are valid. 433 func mustStores(t *testing.T, client anthropic.Client, opts SessionMemoryStoresOptions) *SessionMemoryStores { 434 t.Helper() 435 stores, err := NewSessionMemoryStores(client, opts) 436 require.NoError(t, err) 437 return stores 438 } 439 440 // faults selects which disk verbs fail, and for which paths, and can act on 441 // the disk just before a write. The zero value does nothing; a test sets a 442 // field to break the verb it is exercising and clears it again to watch the 443 // retry. The disk events the baseline rules are written against — a write 444 // that never lands, a removal that cannot happen, a folder wiped between two 445 // writes — have no reliable fixture, so they are injected here. 446 type faults struct { 447 put func(rel string) bool 448 remove func(rel string) bool 449 hashFile func(rel string) bool 450 // onPut runs as each Put arrives, before it reaches the disk — a race 451 // hook to land a change on disk between two of a pass's writes. 452 onPut func(rel string) 453 } 454 455 // always breaks a verb for every path. 456 func always(string) bool { return true } 457 458 // open is the [SessionMemoryStores.openStore] seam: every store opens through 459 // a faultyStore reading these faults live. 460 func (f *faults) open(root string, opts ...filestore.Option) (storeFiles, error) { 461 files, err := filestore.Open(root, opts...) 462 if err != nil { 463 return nil, err 464 } 465 return &faultyStore{storeFiles: files, faults: f}, nil 466 } 467 468 type faultyStore struct { 469 storeFiles 470 faults *faults 471 } 472 473 func (f *faultyStore) Put(rel string, data []byte, opts ...filestore.PutOption) error { 474 if f.faults.onPut != nil { 475 f.faults.onPut(rel) 476 } 477 if f.faults.put != nil && f.faults.put(rel) { 478 return fmt.Errorf("injected write failure: %s", rel) 479 } 480 return f.storeFiles.Put(rel, data, opts...) 481 } 482 483 func (f *faultyStore) Remove(rel string) error { 484 if f.faults.remove != nil && f.faults.remove(rel) { 485 return fmt.Errorf("injected removal failure: %s", rel) 486 } 487 return f.storeFiles.Remove(rel) 488 } 489 490 func (f *faultyStore) HashFile(rel string) (string, error) { 491 if f.faults.hashFile != nil && f.faults.hashFile(rel) { 492 return "", fmt.Errorf("injected hash failure: %s", rel) 493 } 494 return f.storeFiles.HashFile(rel) 495 } 496 497 // captureLog is a debug-level text logger writing into buf, for tests that 498 // assert on log output. 499 func captureLog(buf *bytes.Buffer) *slog.Logger { 500 return slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug})) 501 } 502 503 // downloadedStores returns a SessionMemoryStores with the fake server's one 504 // store already downloaded to disk, plus the local dir and the server handle. 505 // disk, when non-nil, opens every store through a faultyStore. 506 func downloadedStores(t *testing.T, initial map[string]string, access string, logBuf *bytes.Buffer, disk *faults) (string, *memoryServer, *SessionMemoryStores) { 507 t.Helper() 508 server := newMemoryServer(t, initial, access) 509 httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP)) 510 t.Cleanup(httpSrv.Close) 511 client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0)) 512 513 logger := silentLogger 514 if logBuf != nil { 515 logger = captureLog(logBuf) 516 } 517 workdir := t.TempDir() 518 stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: workdir, Logger: logger}) 519 if disk != nil { 520 stores.openStore = disk.open 521 } 522 require.NoError(t, stores.Download(context.Background(), fetchSession(t, client))) 523 return filepath.Join(workdir, "memory", "notes"), server, stores 524 } 525 526 // testClock is a settable stand-in for the store's time source. 527 type testClock struct{ now time.Time } 528 529 func (c *testClock) Now() time.Time { return c.now } 530 531 // withClock swaps in a fake clock at time zero and returns a handle to it. 532 func withClock(stores *SessionMemoryStores) *testClock { 533 c := &testClock{now: time.Unix(0, 0)} 534 stores.setClock(c.Now) 535 return c 536 } 537 538 // runSync drives one reconcile pass without clock bookkeeping. 539 // 540 // The public cadence path (SyncIfDue plus the clock) is exercised by the 541 // worker tests; unit tests call the pass directly. 542 func runSync(ctx context.Context, s *SessionMemoryStores) { 543 s.syncAll(ctx, false) 544 } 545 546 func writeLocal(t *testing.T, dir, name, content string) { 547 t.Helper() 548 require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600)) 549 } 550 551 func readLocal(t *testing.T, dir, name string) string { 552 t.Helper() 553 got, err := os.ReadFile(filepath.Join(dir, name)) 554 require.NoError(t, err) 555 return string(got) 556 } 557 558 func TestMemorySync_EditsFlowBothWaysAndTheServerWinsConflicts(t *testing.T) { 559 var logBuf bytes.Buffer 560 local, server, stores := downloadedStores(t, 561 map[string]string{"push.md": "v1", "pull.md": "v1", "both.md": "v1", "gone.md": "v1"}, "", &logBuf, nil) 562 ctx := context.Background() 563 564 writeLocal(t, local, "push.md", "v2 from agent") 565 writeLocal(t, local, "new.md", "new from agent") 566 server.Write("pull.md", "v2 from server") 567 writeLocal(t, local, "both.md", "v2 from agent") 568 server.Write("both.md", "v2 from server") 569 server.Delete("gone.md") 570 571 runSync(ctx, stores) 572 573 // Each side has the other's change; the server won the conflict; the 574 // server's deletion took the local copy with it. 575 require.Equal(t, "v2 from agent", server.Files["push.md"]) 576 require.Equal(t, "new from agent", server.Files["new.md"]) 577 require.Equal(t, "v2 from server", readLocal(t, local, "pull.md")) 578 require.Equal(t, "v2 from server", readLocal(t, local, "both.md")) 579 require.NoFileExists(t, filepath.Join(local, "gone.md")) 580 require.NotContains(t, server.Files, "gone.md") 581 require.Contains(t, logBuf.String(), "changed both locally and remotely") 582 583 // The push was guarded: overwrite only if the server still had v1. 584 sent := []string{created("new.md", "new from agent"), updated("push.md", "v2 from agent", "v1")} 585 require.ElementsMatch(t, sent, server.ReceivedSnapshot()) 586 587 // Settled: a second sync sends nothing more. 588 runSync(ctx, stores) 589 require.Len(t, server.ReceivedSnapshot(), len(sent)) 590 } 591 592 func TestMemorySync_LocalDeletionsReachTheServerUnlessItChanged(t *testing.T) { 593 // One memory's download write fails, so it is never on disk to begin with 594 // — that absence is not a deletion. 595 disk := &faults{put: func(rel string) bool { return rel == "never.md" }} 596 // keep.md survives on disk so the pass is not an all-files-gone wipe. 597 local, server, stores := downloadedStores(t, 598 map[string]string{"drop.md": "v1", "raced.md": "v1", "never.md": "v1", "keep.md": "v1"}, "", nil, disk) 599 disk.put = nil 600 clock := withClock(stores) 601 ctx := context.Background() 602 require.NoFileExists(t, filepath.Join(local, "never.md")) 603 604 require.NoError(t, os.Remove(filepath.Join(local, "drop.md"))) 605 require.NoError(t, os.Remove(filepath.Join(local, "raced.md"))) 606 server.Write("raced.md", "v2 from server") 607 608 runSync(ctx, stores) 609 // The first pass only records the deletion; a corroborating pass 610 // after the window sends it. 611 require.Contains(t, server.Files, "drop.md") 612 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 613 runSync(ctx, stores) 614 615 // Only the un-raced deletion went out, guarded by the last-synced content. 616 sent := []string{deletedReq("drop.md", "v1")} 617 require.Equal(t, sent, server.ReceivedSnapshot()) 618 require.NotContains(t, server.Files, "drop.md") 619 // The raced deletion lost — the server's edit came back to disk instead. 620 require.Equal(t, "v2 from server", readLocal(t, local, "raced.md")) 621 // The never-downloaded file was pulled, not deleted. 622 require.Equal(t, "v1", readLocal(t, local, "never.md")) 623 require.Equal(t, "v1", server.Files["never.md"]) 624 625 runSync(ctx, stores) 626 require.Equal(t, sent, server.ReceivedSnapshot()) 627 } 628 629 func TestMemorySync_ReadOnlyStoresAreNeverWritten(t *testing.T) { 630 local, server, stores := downloadedStores(t, 631 map[string]string{"edit.md": "v1", "drop.md": "v1", "pull.md": "v1", "vanish.md": "v1"}, "read_only", nil, nil) 632 ctx := context.Background() 633 634 writeLocal(t, local, "edit.md", "v2 from agent") 635 require.NoError(t, os.Remove(filepath.Join(local, "drop.md"))) 636 writeLocal(t, local, "new.md", "new from agent") 637 server.Write("pull.md", "v2 from server") 638 server.Delete("vanish.md") 639 640 runSync(ctx, stores) 641 642 require.Empty(t, server.ReceivedSnapshot()) 643 require.Equal(t, map[string]string{"edit.md": "v1", "drop.md": "v1", "pull.md": "v2 from server"}, server.Files) 644 // Pulls still happen... 645 require.Equal(t, "v2 from server", readLocal(t, local, "pull.md")) 646 // ...including a removal, which is a pull and not a push: the server no 647 // longer holds vanish.md, so neither does the folder. 648 require.NoFileExists(t, filepath.Join(local, "vanish.md")) 649 // And the store's root is reported for the write/edit tool refusal, as 650 // well as for the file tools' reach. 651 require.Equal(t, []string{local}, stores.ReadOnlyRoots()) 652 require.Equal(t, stores.ReadOnlyRoots(), stores.Roots()) 653 } 654 655 func TestMemorySync_PathsThatEscapeTheStoreAreRefused(t *testing.T) { 656 var logBuf bytes.Buffer 657 local, server, stores := downloadedStores(t, 658 map[string]string{"../evil.md": "boom", "ok.md": "fine"}, "", &logBuf, nil) 659 660 runSync(context.Background(), stores) 661 662 require.Equal(t, "fine", readLocal(t, local, "ok.md")) 663 // Nothing named evil.md landed anywhere under the workdir. 664 workdir := filepath.Dir(filepath.Dir(local)) 665 require.NoError(t, filepath.Walk(workdir, func(path string, _ os.FileInfo, err error) error { 666 require.NotEqual(t, "evil.md", filepath.Base(path)) 667 return err 668 })) 669 require.Empty(t, server.ReceivedSnapshot()) 670 require.Contains(t, logBuf.String(), "escapes") 671 } 672 673 func TestMemorySync_LifecycleCadenceAndDispose(t *testing.T) { 674 // The worker's call sequence: download → SyncIfDue per call → Finish → 675 // Dispose. A fake clock makes the cadence observable. 676 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 677 httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP)) 678 t.Cleanup(httpSrv.Close) 679 client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0)) 680 681 now := time.Unix(0, 0) 682 workdir := t.TempDir() 683 stores := mustStores(t, client, SessionMemoryStoresOptions{ 684 Workdir: workdir, 685 SyncInterval: 300 * time.Second, 686 Logger: silentLogger, 687 }) 688 stores.setClock(func() time.Time { return now }) 689 ctx := context.Background() 690 require.NoError(t, stores.Download(ctx, fetchSession(t, client))) 691 local := filepath.Join(workdir, "memory", "notes") 692 693 writeLocal(t, local, "note.md", "edit 1") 694 stores.SyncIfDue(ctx) 695 require.Empty(t, server.ReceivedSnapshot()) 696 697 // One long gap fires exactly once, and not again until another interval. 698 now = time.Unix(2000, 0) 699 stores.SyncIfDue(ctx) 700 stores.SyncIfDue(ctx) 701 require.Equal(t, []string{updated("note.md", "edit 1", "v1")}, server.ReceivedSnapshot()) 702 now = time.Unix(2299, 0) 703 stores.SyncIfDue(ctx) 704 require.Len(t, server.ReceivedSnapshot(), 1) 705 706 // The last sync is Finish: unconditional, once. 707 writeLocal(t, local, "note.md", "edit 2") 708 stores.Finish(ctx) 709 received := server.ReceivedSnapshot() 710 require.Equal(t, updated("note.md", "edit 2", "edit 1"), received[len(received)-1]) 711 712 // Dispose removes the store dir the download created. 713 require.DirExists(t, local) 714 stores.Dispose() 715 require.NoDirExists(t, local) 716 } 717 718 func TestMemorySync_TheSyncIntervalHasADefaultAndAFloor(t *testing.T) { 719 client := anthropic.NewClient(option.WithAPIKey("k")) 720 build := func(d time.Duration) (*SessionMemoryStores, error) { 721 return NewSessionMemoryStores(client, SessionMemoryStoresOptions{ 722 Workdir: t.TempDir(), 723 SyncInterval: d, 724 Logger: silentLogger, 725 }) 726 } 727 for _, tc := range []struct{ in, want time.Duration }{ 728 {0, DefaultMemorySyncInterval}, 729 {MinMemorySyncInterval, MinMemorySyncInterval}, 730 {time.Minute, time.Minute}, 731 {-1, -1}, // negative passes through; the worker gates on it 732 } { 733 stores, err := build(tc.in) 734 require.NoError(t, err, tc.in) 735 require.Equal(t, tc.want, stores.syncInterval, tc.in) 736 } 737 _, err := build(time.Second) 738 require.ErrorIs(t, err, ErrMemorySyncIntervalTooShort) 739 } 740 741 func TestMemorySync_ABinaryLocalFileNeverBlocksTheOtherFiles(t *testing.T) { 742 // The store is utf-8-restricted, so a binary file the agent drops in the 743 // folder is refused at read — warned once and skipped until it changes — 744 // while every other file keeps syncing. 745 var logBuf bytes.Buffer 746 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil) 747 ctx := context.Background() 748 749 require.NoError(t, os.WriteFile(filepath.Join(local, "blob.bin"), []byte{0xff, 0xfe, 0x00, ' ', 'x'}, 0o600)) 750 writeLocal(t, local, "note.md", "v2") 751 752 runSync(ctx, stores) 753 runSync(ctx, stores) 754 755 require.Equal(t, []string{updated("note.md", "v2", "v1")}, server.ReceivedSnapshot()) 756 require.NotContains(t, server.Files, "blob.bin") 757 require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes")) 758 require.Contains(t, logBuf.String(), "path=blob.bin") 759 } 760 761 func TestMemorySync_AnOversizedFileIsSkippedUntilItShrinks(t *testing.T) { 762 // A file the server refuses as too large is warned once and skipped — 763 // not retried while unchanged, never blocking the other files — and 764 // syncs as soon as an edit brings it under the cap. 765 var logBuf bytes.Buffer 766 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil) 767 ctx := context.Background() 768 server.MaxContentBytes = 64 769 770 writeLocal(t, local, "big.md", strings.Repeat("x", 100)) 771 writeLocal(t, local, "note.md", "v2") 772 773 runSync(ctx, stores) 774 runSync(ctx, stores) 775 776 // The small edit landed; the big file was attempted exactly once. 777 received := server.ReceivedSnapshot() 778 require.Contains(t, received, updated("note.md", "v2", "v1")) 779 creates := 0 780 for _, r := range received { 781 if strings.HasPrefix(r, "create ") { 782 creates++ 783 } 784 } 785 require.Equal(t, 1, creates) 786 require.NotContains(t, server.Files, "big.md") 787 require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes")) 788 require.Contains(t, logBuf.String(), "path=big.md") 789 790 writeLocal(t, local, "big.md", "small now") 791 runSync(ctx, stores) 792 require.Equal(t, "small now", server.Files["big.md"]) 793 } 794 795 func TestMemorySync_ARefusalThatSaysNothingAboutTheContentIsRetriedNextSync(t *testing.T) { 796 // Anything but a content refusal is retried by the next sync. 797 for _, status := range []int{401, 403, 404} { 798 t.Run(fmt.Sprint(status), func(t *testing.T) { 799 var logBuf bytes.Buffer 800 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil) 801 ctx := context.Background() 802 writeLocal(t, local, "note.md", "v2") 803 writeLocal(t, local, "new.md", "fresh") 804 805 server.FailUploads = status 806 runSync(ctx, stores) 807 server.FailUploads = 0 808 runSync(ctx, stores) 809 810 require.Equal(t, map[string]string{"note.md": "v2", "new.md": "fresh"}, server.Files) 811 require.NotContains(t, logBuf.String(), "stays un-synced") 812 }) 813 } 814 } 815 816 func TestMemorySync_AnUploadThatLosesTheRacePullsTheWinnerNextSync(t *testing.T) { 817 // The server changes between this sync's list and its update: the 818 // content_sha256 precondition rejects the push (409), and the next sync 819 // applies the conflict rule — the winning remote edit is pulled, the 820 // local edit is never re-pushed. 821 var logBuf bytes.Buffer 822 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil) 823 ctx := context.Background() 824 825 writeLocal(t, local, "note.md", "local edit") 826 server.OnUpdate = func() { 827 // Race: the remote changes after this sync's list, before its 828 // update. (Called with the server mutex held.) 829 server.Files["note.md"] = "raced remote edit" 830 } 831 runSync(ctx, stores) 832 server.OnUpdate = nil 833 834 // The precondition rejected the push (one guarded attempt, no retry) and 835 // the server kept its raced edit. 836 require.Equal(t, []string{updated("note.md", "local edit", "v1")}, server.ReceivedSnapshot()) 837 require.Equal(t, "raced remote edit", server.Files["note.md"]) 838 require.Contains(t, logBuf.String(), "changed both locally and remotely") 839 // The warning says what actually happened: the push was dropped and the 840 // local file is the stale one. Nothing "kept the remote version" here — 841 // disk still holds the losing edit until the next sync overwrites it. 842 require.Contains(t, logBuf.String(), "the upload was refused and the local edit loses") 843 require.Equal(t, "local edit", readLocal(t, local, "note.md")) 844 845 // The next sync pulls the winner instead of re-pushing. 846 runSync(ctx, stores) 847 require.Equal(t, "raced remote edit", readLocal(t, local, "note.md")) 848 require.Len(t, server.ReceivedSnapshot(), 1) 849 } 850 851 // storesAt wires a SessionMemoryStores to server over workdir without 852 // downloading — for the tests that assert on Download's own refusals. 853 func storesAt(t *testing.T, server *memoryServer, workdir string) *SessionMemoryStores { 854 t.Helper() 855 httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP)) 856 t.Cleanup(httpSrv.Close) 857 client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0)) 858 return mustStores(t, client, SessionMemoryStoresOptions{Workdir: workdir, Logger: silentLogger}) 859 } 860 861 func TestMemorySync_ABrokenStoreFailsTheWholeDownload(t *testing.T) { 862 // The first attached store's listing explodes mid-download: its half-written 863 // folder self-destructs and the whole download fails. A session served 864 // without a folder its system prompt names would run with amnesia, so the 865 // worker turns this into a failed work item rather than a warning. 866 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 867 server.Broken = true 868 workdir := t.TempDir() 869 stores := storesAt(t, server, workdir) 870 871 err := stores.Download(context.Background(), fetchSession(t, stores.client)) 872 var memErr *SessionMemoryError 873 require.ErrorAs(t, err, &memErr) 874 require.Equal(t, "memstore_broken", memErr.MemoryStoreID) 875 876 // The broken store got one file down before exploding; nothing survives. 877 // The healthy store behind it was never reached, and nothing is tracked. 878 require.NoDirExists(t, filepath.Join(workdir, "memory", "broken")) 879 require.NoDirExists(t, filepath.Join(workdir, "memory", "notes")) 880 require.Empty(t, stores.ReadOnlyRoots()) 881 require.Empty(t, server.ReceivedSnapshot()) 882 } 883 884 func TestMemorySync_ALegalMountPathIsHonored(t *testing.T) { 885 // mount_path is the location the agent's system prompt names. A clean 886 // absolute one is honored verbatim, and Dispose removes it when the 887 // download created it. 888 tmp := t.TempDir() 889 mount := filepath.Join(tmp, "mnt", "notes") 890 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 891 server.MountPath = mount 892 stores := storesAt(t, server, filepath.Join(tmp, "wd")) 893 894 require.NoError(t, stores.Download(context.Background(), fetchSession(t, stores.client))) 895 require.Equal(t, "v1", readLocal(t, mount, "note.md")) 896 require.Equal(t, []string{mount}, stores.Roots()) 897 stores.Dispose() 898 require.NoDirExists(t, mount) 899 } 900 901 func TestMemorySync_AnUncleanMountPathFailsTheDownload(t *testing.T) { 902 // A mount_path we cannot use verbatim fails the session rather than quietly 903 // relocating the store: the agent would read an empty folder at the path it 904 // was told about, and write notes somewhere the next session never looks. 905 for _, tc := range []struct { 906 name string 907 mount string 908 // "" means the resource carries no mount_path at all — that one is 909 // allowed, and falls back to the workdir, because nothing pointed the 910 // agent anywhere. 911 ok bool 912 }{ 913 {"escapes with ..", "/mnt/../../etc/cron.d", false}, 914 {"relative, with a separator", "relative/notes", false}, 915 {"relative, a bare name", "notes", false}, 916 {"no mount path at all", "", true}, 917 } { 918 t.Run(tc.name, func(t *testing.T) { 919 tmp := t.TempDir() 920 workdir := filepath.Join(tmp, "wd") 921 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 922 server.MountPath = tc.mount 923 stores := storesAt(t, server, workdir) 924 925 err := stores.Download(context.Background(), fetchSession(t, stores.client)) 926 if tc.ok { 927 require.NoError(t, err) 928 require.Equal(t, "v1", readLocal(t, filepath.Join(workdir, "memory", "notes"), "note.md")) 929 return 930 } 931 932 var memErr *SessionMemoryError 933 require.ErrorAs(t, err, &memErr) 934 require.Equal(t, "memstore_notes", memErr.MemoryStoreID) 935 require.ErrorContains(t, err, "not a clean absolute path") 936 937 // Nothing was written anywhere, nothing was sent, nothing is tracked. 938 require.NoDirExists(t, workdir) 939 require.Empty(t, server.ReceivedSnapshot()) 940 require.Empty(t, stores.ReadOnlyRoots()) 941 }) 942 } 943 } 944 945 func TestMemorySync_AnUnwritableMountPathFailsTheDownload(t *testing.T) { 946 // A mount the host cannot provide is an environment failure: the download 947 // fails loudly (failing the work item) instead of degrading to a session 948 // with silently-empty memory. 949 if os.Geteuid() == 0 { 950 t.Skip("directory modes do not bind root") 951 } 952 tmp := t.TempDir() 953 parent := filepath.Join(tmp, "mnt") 954 require.NoError(t, os.Mkdir(parent, 0o700)) 955 require.NoError(t, os.Chmod(parent, 0o500)) // the worker may not create the store's folder here 956 t.Cleanup(func() { _ = os.Chmod(parent, 0o700) }) 957 958 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 959 server.MountPath = filepath.Join(parent, "notes") 960 stores := storesAt(t, server, filepath.Join(tmp, "wd")) 961 962 err := stores.Download(context.Background(), fetchSession(t, stores.client)) 963 var memErr *SessionMemoryError 964 require.ErrorAs(t, err, &memErr) 965 require.Equal(t, "memstore_notes", memErr.MemoryStoreID) 966 require.ErrorContains(t, err, "writable") 967 require.Empty(t, server.ReceivedSnapshot()) 968 } 969 970 func TestMemorySync_AStoreDirectoryThatAlreadyExistsIsRefused(t *testing.T) { 971 // Nothing but a dead run leaves a folder at the store's path. Syncing into 972 // it would fold that run's debris into the customer's store, so refuse — 973 // and leave the folder exactly as it was found. 974 workdir := t.TempDir() 975 stale := filepath.Join(workdir, "memory", "notes") 976 require.NoError(t, os.MkdirAll(stale, 0o755)) 977 writeLocal(t, stale, "debris.md", "left by another session") 978 979 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 980 stores := storesAt(t, server, workdir) 981 982 err := stores.Download(context.Background(), fetchSession(t, stores.client)) 983 var memErr *SessionMemoryError 984 require.ErrorAs(t, err, &memErr) 985 require.Equal(t, "memstore_notes", memErr.MemoryStoreID) 986 require.ErrorContains(t, err, "already exists") 987 988 // Refused, not adopted and not deleted — not even by Dispose. 989 require.Equal(t, "left by another session", readLocal(t, stale, "debris.md")) 990 require.NoFileExists(t, filepath.Join(stale, "note.md")) 991 require.Empty(t, server.ReceivedSnapshot()) 992 require.Empty(t, stores.ReadOnlyRoots()) 993 stores.Dispose() 994 require.DirExists(t, stale) 995 996 // A sibling directory in the same parent is none of our business. 997 other := t.TempDir() 998 require.NoError(t, os.MkdirAll(filepath.Join(other, "memory", "unrelated"), 0o755)) 999 server2 := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 1000 stores2 := storesAt(t, server2, other) 1001 1002 require.NoError(t, stores2.Download(context.Background(), fetchSession(t, stores2.client))) 1003 require.Equal(t, "v1", readLocal(t, filepath.Join(other, "memory", "notes"), "note.md")) 1004 require.DirExists(t, filepath.Join(other, "memory", "unrelated")) 1005 } 1006 1007 func TestMemorySync_PrefixItemsAndForeignResourcesLeaveNoTrace(t *testing.T) { 1008 // Only memory_store resources are downloaded and only "memory" items 1009 // carry content — a "file" resource and memory_prefix rollups pass 1010 // through without touching the disk or the server. 1011 server := newMemoryServer(t, map[string]string{"ok.md": "fine"}, "") 1012 server.Noise = true 1013 workdir := t.TempDir() 1014 stores := storesAt(t, server, workdir) 1015 ctx := context.Background() 1016 require.NoError(t, stores.Download(ctx, fetchSession(t, stores.client))) 1017 local := filepath.Join(workdir, "memory", "notes") 1018 1019 require.Equal(t, "fine", readLocal(t, local, "ok.md")) 1020 entries, err := os.ReadDir(filepath.Join(workdir, "memory")) 1021 require.NoError(t, err) 1022 require.Len(t, entries, 1) 1023 require.Equal(t, "notes", entries[0].Name()) 1024 require.NoDirExists(t, filepath.Join(local, "projects")) 1025 1026 runSync(ctx, stores) 1027 require.Empty(t, server.ReceivedSnapshot()) 1028 } 1029 1030 // ---- the sync matrix ------------------------------------------------------- 1031 1032 // Matrix cells: keep means the side did nothing, gone means it deleted the 1033 // path. Any other value is the content that side holds now. The sentinels 1034 // cannot collide with real content. 1035 const ( 1036 keep = "\x00keep" 1037 gone = "\x00gone" 1038 ) 1039 1040 func TestMemorySync_TheSyncMatrix(t *testing.T) { 1041 // Every combination of (local moved?, remote moved?) for a path the store 1042 // downloaded at "v1". local and remote say what happened since; disk and 1043 // store are the state the two sides must agree on afterwards, and sent is 1044 // exactly what went over the wire. 1045 for _, tc := range []struct { 1046 name string 1047 local, remote string 1048 disk, store string 1049 sent []string 1050 }{ 1051 {"untouched", keep, keep, "v1", "v1", nil}, 1052 {"local edit", "v2", keep, "v2", "v2", []string{updated("f.md", "v2", "v1")}}, 1053 {"remote edit", keep, "v2", "v2", "v2", nil}, 1054 {"both edit, the server wins", "v2 local", "v2 remote", "v2 remote", "v2 remote", nil}, 1055 {"local delete", gone, keep, gone, gone, []string{deletedReq("f.md", "v1")}}, 1056 {"remote delete", keep, gone, gone, gone, nil}, 1057 {"both delete", gone, gone, gone, gone, nil}, 1058 {"local delete, remote edit", gone, "v2", "v2", "v2", nil}, 1059 {"local edit, remote delete", "v2", gone, "v2", "v2", []string{created("f.md", "v2")}}, 1060 } { 1061 t.Run(tc.name, func(t *testing.T) { 1062 local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, nil) 1063 clock := withClock(stores) 1064 ctx := context.Background() 1065 1066 switch tc.local { 1067 case keep: 1068 case gone: 1069 require.NoError(t, os.Remove(filepath.Join(local, "f.md"))) 1070 default: 1071 writeLocal(t, local, "f.md", tc.local) 1072 } 1073 switch tc.remote { 1074 case keep: 1075 case gone: 1076 server.Delete("f.md") 1077 default: 1078 server.Write("f.md", tc.remote) 1079 } 1080 1081 settled := func() { 1082 t.Helper() 1083 if tc.disk == gone { 1084 require.NoFileExists(t, filepath.Join(local, "f.md")) 1085 } else { 1086 require.Equal(t, tc.disk, readLocal(t, local, "f.md")) 1087 } 1088 if tc.store == gone { 1089 require.NotContains(t, server.Files, "f.md") 1090 } else { 1091 require.Equal(t, tc.store, server.Files["f.md"]) 1092 } 1093 require.Equal(t, tc.sent, server.ReceivedSnapshot()) 1094 } 1095 1096 // Two passes: a local deletion is only sent after its 1097 // corroboration window. 1098 runSync(ctx, stores) 1099 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1100 runSync(ctx, stores) 1101 settled() 1102 1103 // Settled: a second pass sends nothing more and changes nothing. 1104 runSync(ctx, stores) 1105 settled() 1106 }) 1107 } 1108 } 1109 1110 // ---- the failure column ---------------------------------------------------- 1111 // 1112 // The same paths, one operation broken. A failed operation must leave disk and 1113 // server in a state the next sync can still reconcile — never one where the 1114 // retry does the opposite of what was asked. 1115 1116 func TestMemorySync_AFailedUploadKeepsTheLocalEditAndRetries(t *testing.T) { 1117 local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, nil) 1118 ctx := context.Background() 1119 writeLocal(t, local, "f.md", "v2 from agent") 1120 1121 server.FailUpdates = true 1122 runSync(ctx, stores) 1123 1124 // The agent's edit survives; the server is untouched. 1125 require.Equal(t, "v2 from agent", readLocal(t, local, "f.md")) 1126 require.Equal(t, "v1", server.Files["f.md"]) 1127 1128 // The baseline still says v1, so the next (working) sync retries the push 1129 // with the same precondition. 1130 server.FailUpdates = false 1131 runSync(ctx, stores) 1132 require.Equal(t, "v2 from agent", server.Files["f.md"]) 1133 push := updated("f.md", "v2 from agent", "v1") 1134 require.Equal(t, []string{push, push}, server.ReceivedSnapshot()) 1135 } 1136 1137 func TestMemorySync_AFailedPullLeavesTheOldFileAndRetries(t *testing.T) { 1138 disk := &faults{} 1139 local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, disk) 1140 ctx := context.Background() 1141 server.Write("f.md", "v2 from server") 1142 1143 disk.put = always 1144 runSync(ctx, stores) 1145 disk.put = nil 1146 1147 // The stale file is still there and the baseline never advanced... 1148 require.Equal(t, "v1", readLocal(t, local, "f.md")) 1149 require.Empty(t, server.ReceivedSnapshot()) 1150 1151 // ...so the next sync pulls it, rather than mistaking v1 for a local edit 1152 // and pushing it back over the server's v2. 1153 runSync(ctx, stores) 1154 require.Equal(t, "v2 from server", readLocal(t, local, "f.md")) 1155 require.Empty(t, server.ReceivedSnapshot()) 1156 } 1157 1158 func TestMemorySync_AFailedLocalRemovalNeverReUploadsTheDeletedMemory(t *testing.T) { 1159 // The server deleted the memory; removing our copy fails. The baseline must 1160 // stay put — dropping it would read the leftover file as a new local file 1161 // and create the memory the server just deleted. 1162 var logBuf bytes.Buffer 1163 disk := &faults{} 1164 local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", &logBuf, disk) 1165 ctx := context.Background() 1166 server.Delete("f.md") 1167 1168 disk.remove = always 1169 runSync(ctx, stores) 1170 disk.remove = nil 1171 1172 require.Equal(t, "v1", readLocal(t, local, "f.md")) 1173 require.Empty(t, server.ReceivedSnapshot()) 1174 require.NotContains(t, server.Files, "f.md") 1175 require.Contains(t, logBuf.String(), "failed to remove memory deleted remotely") 1176 1177 // The retry deletes it locally; it is never pushed back. 1178 runSync(ctx, stores) 1179 require.NoFileExists(t, filepath.Join(local, "f.md")) 1180 require.Empty(t, server.ReceivedSnapshot()) 1181 require.NotContains(t, server.Files, "f.md") 1182 } 1183 1184 // twoStoreSession builds a session attaching two memory stores at the given 1185 // mount paths, the way the control plane would send it. 1186 func twoStoreSession(t *testing.T, aRoot, bRoot string) *anthropic.BetaManagedAgentsSession { 1187 t.Helper() 1188 body := fmt.Sprintf(`{"resources":[ 1189 {"type":"memory_store","memory_store_id":"store_a","mount_path":%q,"name":"a"}, 1190 {"type":"memory_store","memory_store_id":"store_b","mount_path":%q,"name":"b"} 1191 ]}`, aRoot, bRoot) 1192 var session anthropic.BetaManagedAgentsSession 1193 require.NoError(t, json.Unmarshal([]byte(body), &session)) 1194 return &session 1195 } 1196 1197 func TestMemorySync_TwoStoresDownloadToTheirOwnFolders(t *testing.T) { 1198 wd := t.TempDir() 1199 server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "") 1200 httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP)) 1201 t.Cleanup(httpSrv.Close) 1202 client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0)) 1203 stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: wd, Logger: silentLogger}) 1204 1205 a, b := filepath.Join(wd, "m", "a"), filepath.Join(wd, "m", "b") 1206 require.NoError(t, stores.Download(context.Background(), twoStoreSession(t, a, b))) 1207 require.Equal(t, "v1", readLocal(t, a, "note.md")) 1208 require.Equal(t, "v1", readLocal(t, b, "note.md")) 1209 } 1210 1211 func TestMemorySync_StoresSyncInParallelAndOneFailureSparesTheRest(t *testing.T) { 1212 // Store A's listing waits for store B's listing to start — only a 1213 // parallel sync satisfies that — then fails. A serial sync times the 1214 // wait out instead (a different error message), and store B's edit must 1215 // land either way. 1216 wd := t.TempDir() 1217 slow := newMemoryServer(t, map[string]string{"a.md": "v1"}, "") 1218 fast := newMemoryServer(t, map[string]string{"b.md": "v1"}, "") 1219 var syncing atomic.Bool 1220 fastStarted := make(chan struct{}) 1221 var once sync.Once 1222 fail := func(w http.ResponseWriter, msg string) { 1223 w.Header().Set("Content-Type", "application/json") 1224 w.WriteHeader(http.StatusInternalServerError) 1225 _, _ = fmt.Fprintf(w, `{"error":{"type":"api_error","message":%q}}`, msg) 1226 } 1227 httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 1228 isList := r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/memories") 1229 if strings.Contains(r.URL.Path, "store_a") { 1230 if syncing.Load() && isList { 1231 select { 1232 case <-fastStarted: 1233 fail(w, "slow store exploded") 1234 case <-time.After(10 * time.Second): 1235 fail(w, "fast store never started") 1236 } 1237 return 1238 } 1239 slow.serveHTTP(w, r) 1240 return 1241 } 1242 if syncing.Load() && isList { 1243 once.Do(func() { close(fastStarted) }) 1244 } 1245 fast.serveHTTP(w, r) 1246 })) 1247 t.Cleanup(httpSrv.Close) 1248 client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0)) 1249 1250 var logBuf bytes.Buffer 1251 logger := slog.New(slog.NewTextHandler(&logBuf, nil)) 1252 stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: wd, Logger: logger}) 1253 aRoot, bRoot := filepath.Join(wd, "m", "a"), filepath.Join(wd, "m", "b") 1254 ctx := context.Background() 1255 require.NoError(t, stores.Download(ctx, twoStoreSession(t, aRoot, bRoot))) 1256 1257 syncing.Store(true) 1258 writeLocal(t, bRoot, "b.md", "v2 from agent") 1259 runSync(ctx, stores) 1260 1261 // Store A failed *after* seeing store B's listing start — proof the two 1262 // ran concurrently — and store B still pushed. 1263 require.Contains(t, logBuf.String(), "slow store exploded") 1264 require.Equal(t, []string{updated("b.md", "v2 from agent", "v1")}, fast.ReceivedSnapshot()) 1265 // The failed store's baseline was kept, so nothing is re-uploaded once it heals. 1266 require.Equal(t, "v1", readLocal(t, aRoot, "a.md")) 1267 } 1268 1269 // ---- folder-trust gating --------------------------------------------------- 1270 // 1271 // A missing, emptied, or swapped store folder is never trusted as a basis 1272 // for deletions or uploads. 1273 // 1274 // Download stamps MarkerPath (holding the store's id) into the folder. A 1275 // sync pass that finds the folder destroyed re-downloads it and pushes 1276 // nothing; a sync that finds files without a matching marker leaves the 1277 // folder as found and syncs nothing at all. 1278 1279 const ( 1280 folderTrustStoreID = "memstore_notes" 1281 folderTrustMarker = "version 1\n" + folderTrustStoreID 1282 ) 1283 1284 func TestFolderTrust_MarkerBytesArePinned(t *testing.T) { 1285 // A format change breaks CI here rather than as a marker mismatch at runtime. 1286 // printf 'version 1\nmemstore_test' | sha256sum 1287 require.Equal(t, "b8c0fbafe3e9ea979f0fbe5d984ba8834e7a2771d859462b3f7aa556fdb55a67", 1288 markerSHA("memstore_test")) 1289 } 1290 1291 func TestFolderTrust_ADeletedFolderIssuesNoDeletesAndIsRebuilt(t *testing.T) { 1292 // rm -rf of the whole folder between syncs: the next sync must not read 1293 // the emptiness as "the agent deleted everything" and wipe the server. 1294 var logBuf bytes.Buffer 1295 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil) 1296 clock := withClock(stores) 1297 ctx := context.Background() 1298 require.NoError(t, os.RemoveAll(local)) 1299 1300 runSync(ctx, stores) 1301 1302 require.Empty(t, server.ReceivedSnapshot()) 1303 require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files) 1304 require.Equal(t, "v1", readLocal(t, local, "a.md")) 1305 require.Equal(t, "v1", readLocal(t, local, "b.md")) 1306 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1307 require.Contains(t, logBuf.String(), "the folder or its marker is gone; re-downloading") 1308 1309 // The rebuilt folder syncs normally: an edit and a deletion both 1310 // propagate (the deletion after its waiting window). 1311 writeLocal(t, local, "a.md", "v2") 1312 require.NoError(t, os.Remove(filepath.Join(local, "b.md"))) 1313 runSync(ctx, stores) 1314 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1315 runSync(ctx, stores) 1316 require.Equal(t, []string{updated("a.md", "v2", "v1"), deletedReq("b.md", "v1")}, server.ReceivedSnapshot()) 1317 require.Equal(t, map[string]string{"a.md": "v2"}, server.Files) 1318 } 1319 1320 func TestFolderTrust_AnEmptiedFolderIssuesNoDeletesAndIsRepopulated(t *testing.T) { 1321 var logBuf bytes.Buffer 1322 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil) 1323 clock := withClock(stores) 1324 ctx := context.Background() 1325 entries, _ := os.ReadDir(local) 1326 for _, e := range entries { 1327 require.NoError(t, os.Remove(filepath.Join(local, e.Name()))) 1328 } 1329 1330 runSync(ctx, stores) 1331 1332 require.Empty(t, server.ReceivedSnapshot()) 1333 require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files) 1334 require.Equal(t, "v1", readLocal(t, local, "a.md")) 1335 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1336 require.Contains(t, logBuf.String(), "the folder or its marker is gone; re-downloading") 1337 1338 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1339 runSync(ctx, stores) 1340 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1341 runSync(ctx, stores) 1342 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1343 } 1344 1345 func TestFolderTrust_AnEmptiedFolderWithTheMarkerLeftBehindIssuesNoDeletes(t *testing.T) { 1346 // rm <mount>/*: shell globs skip dotfiles, so the marker survives the 1347 // wipe — an intact marker alone must not clear a folder whose every 1348 // memory file vanished at once. 1349 var logBuf bytes.Buffer 1350 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil) 1351 clock := withClock(stores) 1352 ctx := context.Background() 1353 entries, _ := os.ReadDir(local) 1354 for _, e := range entries { 1355 if e.Name() != MarkerPath { 1356 require.NoError(t, os.Remove(filepath.Join(local, e.Name()))) 1357 } 1358 } 1359 1360 runSync(ctx, stores) 1361 1362 require.Empty(t, server.ReceivedSnapshot()) 1363 require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files) 1364 require.Equal(t, "v1", readLocal(t, local, "a.md")) 1365 require.Contains(t, logBuf.String(), "every memory file is gone at once; re-downloading") 1366 1367 // With files present again, an ordinary single deletion still 1368 // propagates (after its waiting window). 1369 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1370 runSync(ctx, stores) 1371 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1372 runSync(ctx, stores) 1373 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1374 } 1375 1376 func TestFolderTrust_ASwappedFolderNeitherDeletesNorUploadsEver(t *testing.T) { 1377 // A folder carrying another store's marker is foreign: the pass must not 1378 // adopt it — no deletes, no uploads, no restamp — on this pass or any 1379 // later one, or the foreign files leak into the customer's store one 1380 // pass later. 1381 var logBuf bytes.Buffer 1382 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil) 1383 ctx := context.Background() 1384 require.NoError(t, os.RemoveAll(local)) 1385 require.NoError(t, os.MkdirAll(local, 0o755)) 1386 writeLocal(t, local, MarkerPath, "memstore_other") 1387 writeLocal(t, local, "foreign.md", "someone else's notes") 1388 1389 runSync(ctx, stores) 1390 runSync(ctx, stores) 1391 1392 require.Empty(t, server.ReceivedSnapshot()) 1393 require.Equal(t, map[string]string{"a.md": "v1"}, server.Files) 1394 require.Equal(t, "memstore_other", readLocal(t, local, MarkerPath)) 1395 require.Equal(t, "someone else's notes", readLocal(t, local, "foreign.md")) 1396 require.NoFileExists(t, filepath.Join(local, "a.md")) 1397 require.Contains(t, logBuf.String(), "the marker file does not match this store; leaving the memory store folder as found") 1398 } 1399 1400 func TestFolderTrust_FilesWithoutAMarkerAreNeitherPushedNorPulledOver(t *testing.T) { 1401 // Only the marker was deleted (a dotfile cleanup): the remaining files 1402 // may hold un-pushed edits, so the pass must neither upload them, nor 1403 // delete by them, nor destroy them by re-downloading over the folder. 1404 var logBuf bytes.Buffer 1405 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil) 1406 ctx := context.Background() 1407 require.NoError(t, os.Remove(filepath.Join(local, MarkerPath))) 1408 writeLocal(t, local, "a.md", "un-pushed edit") 1409 require.NoError(t, os.Remove(filepath.Join(local, "b.md"))) 1410 1411 runSync(ctx, stores) 1412 1413 require.Empty(t, server.ReceivedSnapshot()) 1414 require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files) 1415 require.Equal(t, "un-pushed edit", readLocal(t, local, "a.md")) 1416 require.NoFileExists(t, filepath.Join(local, MarkerPath)) 1417 require.Contains(t, logBuf.String(), "the marker file is gone; leaving the memory store folder as found") 1418 } 1419 1420 func TestFolderTrust_ASingleLocalDeletionWithTheMarkerIntactStillPropagates(t *testing.T) { 1421 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", nil, nil) 1422 clock := withClock(stores) 1423 ctx := context.Background() 1424 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1425 1426 runSync(ctx, stores) 1427 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1428 runSync(ctx, stores) 1429 1430 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1431 require.Equal(t, map[string]string{"b.md": "v1"}, server.Files) 1432 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1433 } 1434 1435 func TestFolderTrust_TheMarkerNeverSyncsInEitherDirection(t *testing.T) { 1436 var logBuf bytes.Buffer 1437 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil) 1438 ctx := context.Background() 1439 1440 // The marker sits in the folder yet is never uploaded or remote-deleted. 1441 runSync(ctx, stores) 1442 require.Empty(t, server.ReceivedSnapshot()) 1443 1444 // A marker path the server lists is refused: skipped with a warning, 1445 // never written over the real marker on disk. 1446 server.Write(MarkerPath, "not a store id") 1447 runSync(ctx, stores) 1448 require.Empty(t, server.ReceivedSnapshot()) 1449 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1450 require.Contains(t, logBuf.String(), "reserved marker path") 1451 } 1452 1453 func TestFolderTrust_DisposeToleratesAnAlreadyDeletedFolder(t *testing.T) { 1454 local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil) 1455 require.NoError(t, os.RemoveAll(local)) 1456 1457 stores.Dispose() 1458 1459 require.NoDirExists(t, local) 1460 } 1461 1462 func TestFolderTrust_DisposeKeepsAFolderThatFailsTheMarkerCheck(t *testing.T) { 1463 // Sync left the folder as found after a failed marker check; teardown 1464 // must not then delete the files sync refused to touch. 1465 var logBuf bytes.Buffer 1466 local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil) 1467 require.NoError(t, os.Remove(filepath.Join(local, MarkerPath))) 1468 writeLocal(t, local, "a.md", "un-pushed edit") 1469 1470 stores.Dispose() 1471 1472 require.Equal(t, "un-pushed edit", readLocal(t, local, "a.md")) 1473 require.Contains(t, logBuf.String(), "leaving the memory store folder on disk") 1474 } 1475 1476 func TestFolderTrust_DisposeNeverPanicsWhenAStoreRootIsReplacedByAFile(t *testing.T) { 1477 // The worker calls Dispose from its teardown: a store whose folder 1478 // something replaced with a regular file must log and be skipped, not 1479 // panic out of the teardown. 1480 local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil) 1481 require.NoError(t, os.RemoveAll(local)) 1482 require.NoError(t, os.WriteFile(local, []byte("a file where the folder was"), 0o600)) 1483 1484 stores.Dispose() 1485 1486 got, err := os.ReadFile(local) 1487 require.NoError(t, err) 1488 require.Equal(t, "a file where the folder was", string(got)) 1489 } 1490 1491 func TestFolderTrust_AMarkerFromAnOlderFormatNoLongerClearsTheFolder(t *testing.T) { 1492 // The marker carries a format version so an SDK upgrade can retire every 1493 // folder written before it: a marker in the old format (the bare store 1494 // id) must fail the check, and the folder is left as found. 1495 var logBuf bytes.Buffer 1496 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil) 1497 ctx := context.Background() 1498 writeLocal(t, local, MarkerPath, folderTrustStoreID) 1499 writeLocal(t, local, "a.md", "edit under the old marker") 1500 require.NoError(t, os.Remove(filepath.Join(local, "b.md"))) 1501 1502 runSync(ctx, stores) 1503 1504 require.Empty(t, server.ReceivedSnapshot()) 1505 require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files) 1506 require.Equal(t, "edit under the old marker", readLocal(t, local, "a.md")) 1507 require.Contains(t, logBuf.String(), "the marker file does not match this store") 1508 } 1509 1510 func TestFolderTrust_AFolderWipedWhileAPullIsInFlightIsRebuiltNotDistrusted(t *testing.T) { 1511 // rm -rf lands between a sync's scan and its content write: the write 1512 // must not bring the folder back without its marker, or every later sync 1513 // reads the folder as foreign and the session's memory writes are lost. 1514 // A flat path fails at the temp-file open, a nested one at the mkdir. 1515 for _, tc := range []struct{ name, rel string }{{"flat", "z.md"}, {"nested", "sub/z.md"}} { 1516 t.Run(tc.name, func(t *testing.T) { 1517 var logBuf bytes.Buffer 1518 local, server, stores := downloadedStores(t, 1519 map[string]string{"x.md": "v1", "y.md": "v1", tc.rel: "v1"}, "", &logBuf, nil) 1520 ctx := context.Background() 1521 server.Write(tc.rel, "v2") 1522 server.SetOnFetch(func(string) int { 1523 _ = os.RemoveAll(local) 1524 return 0 1525 }) 1526 1527 runSync(ctx, stores) 1528 1529 require.NoDirExists(t, local) 1530 require.Empty(t, server.ReceivedSnapshot()) 1531 require.Contains(t, logBuf.String(), "failed to write memory") 1532 1533 server.SetOnFetch(nil) 1534 runSync(ctx, stores) 1535 1536 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1537 require.Equal(t, "v1", readLocal(t, local, "x.md")) 1538 require.Equal(t, "v2", readLocal(t, local, tc.rel)) 1539 require.Contains(t, logBuf.String(), "re-downloading the memory store folder") 1540 require.NotContains(t, logBuf.String(), "leaving the memory store folder as found") 1541 1542 stores.Finish(ctx) 1543 stores.FlushWrites(ctx) 1544 stores.Dispose() 1545 require.NoDirExists(t, local) 1546 require.Empty(t, server.ReceivedSnapshot()) 1547 require.Equal(t, map[string]string{"x.md": "v1", "y.md": "v1", tc.rel: "v2"}, server.Files) 1548 }) 1549 } 1550 } 1551 1552 func TestFolderTrust_AFolderWipedWhileALocallyDeletedMemoryIsBeingRestoredIsRebuilt(t *testing.T) { 1553 // A file deleted locally but changed on the server is restored by the 1554 // next sync; an rm -rf that lands while that restore is downloading must 1555 // leave no folder behind, the sync after it must rebuild the whole store, 1556 // and the earlier absence must never become a server delete. 1557 var logBuf bytes.Buffer 1558 local, server, stores := downloadedStores(t, 1559 map[string]string{"x.md": "v1", "y.md": "v1", "z.md": "v1"}, "", &logBuf, nil) 1560 clock := withClock(stores) 1561 ctx := context.Background() 1562 require.NoError(t, os.Remove(filepath.Join(local, "z.md"))) 1563 server.Write("z.md", "v2") 1564 server.SetOnFetch(func(string) int { 1565 _ = os.RemoveAll(local) 1566 return 0 1567 }) 1568 1569 runSync(ctx, stores) 1570 require.NoDirExists(t, local) 1571 1572 server.SetOnFetch(nil) 1573 runSync(ctx, stores) 1574 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1575 require.Equal(t, "v1", readLocal(t, local, "x.md")) 1576 require.Equal(t, "v1", readLocal(t, local, "y.md")) 1577 require.Equal(t, "v2", readLocal(t, local, "z.md")) 1578 1579 clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second) 1580 runSync(ctx, stores) 1581 require.Empty(t, server.ReceivedSnapshot()) 1582 require.Equal(t, map[string]string{"x.md": "v1", "y.md": "v1", "z.md": "v2"}, server.Files) 1583 require.NotContains(t, logBuf.String(), "leaving the memory store folder as found") 1584 } 1585 1586 func TestFolderTrust_AFolderWipedAgainMidRecoveryIsRebuiltByTheNextSync(t *testing.T) { 1587 // A re-download writes the marker first and the memories after it; a 1588 // second rm -rf between those writes must not leave memories on disk 1589 // without a marker — the next sync re-downloads once more instead. 1590 var logBuf bytes.Buffer 1591 disk := &faults{} 1592 local, server, stores := downloadedStores(t, 1593 map[string]string{"a.md": "v1", "b.md": "v1", "c.md": "v1"}, "", &logBuf, disk) 1594 ctx := context.Background() 1595 require.NoError(t, os.RemoveAll(local)) 1596 1597 memoryPuts := 0 1598 disk.onPut = func(rel string) { 1599 if rel == MarkerPath { 1600 return 1601 } 1602 memoryPuts++ 1603 if memoryPuts == 2 { // the marker and one memory are back on disk 1604 _ = os.RemoveAll(local) 1605 } 1606 } 1607 runSync(ctx, stores) 1608 disk.onPut = nil 1609 1610 require.NoDirExists(t, local) 1611 require.Empty(t, server.ReceivedSnapshot()) 1612 1613 runSync(ctx, stores) 1614 1615 require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath)) 1616 for _, name := range []string{"a.md", "b.md", "c.md"} { 1617 require.Equal(t, "v1", readLocal(t, local, name)) 1618 } 1619 require.Equal(t, 2, strings.Count(logBuf.String(), "re-downloading the memory store folder")) 1620 require.NotContains(t, logBuf.String(), "leaving the memory store folder as found") 1621 require.Empty(t, server.ReceivedSnapshot()) 1622 1623 stores.Dispose() 1624 require.NoDirExists(t, local) 1625 } 1626 1627 // ---- server-delete discipline ---------------------------------------------- 1628 // 1629 // A locally deleted file is deleted on the server only after a second sync 1630 // confirms it, at a bounded rate per sync, and only when server deletes 1631 // are enabled at all. 1632 // 1633 // The first sync that sees the file missing records it; a later sync — 1634 // after DeleteCorroborationWindow (skipped on the session's final Finish, the 1635 // session's last sync) and a re-check that the file is still gone and the 1636 // marker still intact — sends the DELETE. Each sync sends a bounded 1637 // number of deletes; SyncDeletions = MemorySyncDeletionsLogOnly only logs them, 1638 // and MemorySyncDeletionsDisabled sends none, ever. 1639 1640 var deleteElapsed = DeleteCorroborationWindow + time.Second 1641 1642 // downloadedMode is downloadedStores with a SyncDeletions mode and a fake 1643 // clock at time zero. 1644 func downloadedMode(t *testing.T, initial map[string]string, mode MemoryDeleteMode, logBuf *bytes.Buffer) (string, *memoryServer, *SessionMemoryStores, *testClock) { 1645 t.Helper() 1646 local, server, stores := downloadedStores(t, initial, "", logBuf, nil) 1647 stores.syncDeletions = mode 1648 clock := withClock(stores) 1649 return local, server, stores, clock 1650 } 1651 1652 func TestDeleteDiscipline_ALocalDeletionIsConfirmedBeforeItReachesTheServer(t *testing.T) { 1653 local, server, stores, clock := downloadedMode(t, 1654 map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsEnabled, nil) 1655 ctx := context.Background() 1656 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1657 1658 // The first pass only records the absence. 1659 runSync(ctx, stores) 1660 require.Empty(t, server.ReceivedSnapshot()) 1661 require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files) 1662 1663 // Still inside the window: nothing goes out. 1664 clock.now = clock.now.Add(DeleteCorroborationWindow - time.Second) 1665 runSync(ctx, stores) 1666 require.Empty(t, server.ReceivedSnapshot()) 1667 1668 // Window elapsed and the file is still gone: the guarded delete is sent. 1669 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1670 runSync(ctx, stores) 1671 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1672 require.Equal(t, map[string]string{"keep.md": "v1"}, server.Files) 1673 } 1674 1675 func TestDeleteDiscipline_AFileThatReappearsIsNotDeletedAndLeavesPending(t *testing.T) { 1676 local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1"}, MemorySyncDeletionsEnabled, nil) 1677 ctx := context.Background() 1678 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1679 runSync(ctx, stores) 1680 1681 writeLocal(t, local, "a.md", "v1") 1682 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1683 runSync(ctx, stores) 1684 require.Empty(t, server.ReceivedSnapshot()) 1685 require.Equal(t, map[string]string{"a.md": "v1"}, server.Files) 1686 1687 // The reappearance cleared the pending entry: deleting again starts a 1688 // fresh window instead of firing on the stale observation. 1689 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1690 runSync(ctx, stores) 1691 require.Empty(t, server.ReceivedSnapshot()) 1692 clock.now = time.Unix(0, 0).Add(2 * deleteElapsed) 1693 runSync(ctx, stores) 1694 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1695 } 1696 1697 func TestDeleteDiscipline_DisabledModeNeverDeletesButStillSyncs(t *testing.T) { 1698 var logBuf bytes.Buffer 1699 local, server, stores, clock := downloadedMode(t, 1700 map[string]string{"gone.md": "v1", "push.md": "v1", "pull.md": "v1"}, MemorySyncDeletionsDisabled, &logBuf) 1701 ctx := context.Background() 1702 require.NoError(t, os.Remove(filepath.Join(local, "gone.md"))) 1703 writeLocal(t, local, "push.md", "v2 from agent") 1704 server.Write("pull.md", "v2 from server") 1705 1706 runSync(ctx, stores) 1707 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1708 runSync(ctx, stores) 1709 clock.now = time.Unix(0, 0).Add(2 * deleteElapsed) 1710 runSync(ctx, stores) 1711 1712 // Uploads and pulls still flow; the deletion never does. 1713 require.Equal(t, []string{updated("push.md", "v2 from agent", "v1")}, server.ReceivedSnapshot()) 1714 require.Equal(t, map[string]string{"gone.md": "v1", "push.md": "v2 from agent", "pull.md": "v2 from server"}, server.Files) 1715 require.Equal(t, "v2 from server", readLocal(t, local, "pull.md")) 1716 require.Contains(t, logBuf.String(), "server deletes are disabled") 1717 } 1718 1719 func TestDeleteDiscipline_ThePerPassCapSpreadsDeletesOverPasses(t *testing.T) { 1720 var logBuf bytes.Buffer 1721 files := map[string]string{} 1722 for i := 0; i < 14; i++ { 1723 files[fmt.Sprintf("f%02d.md", i)] = "v1" 1724 } 1725 local, server, stores, clock := downloadedMode(t, files, MemorySyncDeletionsEnabled, &logBuf) 1726 ctx := context.Background() 1727 var doomed []string 1728 for k := range files { 1729 doomed = append(doomed, k) 1730 } 1731 slices.Sort(doomed) 1732 doomed = doomed[:12] 1733 for _, name := range doomed { 1734 require.NoError(t, os.Remove(filepath.Join(local, name))) 1735 } 1736 1737 runSync(ctx, stores) 1738 require.Empty(t, server.ReceivedSnapshot()) 1739 1740 // cap = max(8, min(50, 14/4)) = 8: the first confirming sync stops there. 1741 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1742 runSync(ctx, stores) 1743 var want []string 1744 for _, name := range doomed[:8] { 1745 want = append(want, deletedReq(name, "v1")) 1746 } 1747 require.Equal(t, want, server.ReceivedSnapshot()) 1748 require.Contains(t, logBuf.String(), "delete cap reached: sent 8 deletes, held 4 for later syncs") 1749 1750 // The held-back deletions confirm on the next pass. 1751 runSync(ctx, stores) 1752 for _, name := range doomed[8:] { 1753 want = append(want, deletedReq(name, "v1")) 1754 } 1755 require.Equal(t, want, server.ReceivedSnapshot()) 1756 remaining := map[string]string{} 1757 for i := 12; i < 14; i++ { 1758 remaining[fmt.Sprintf("f%02d.md", i)] = "v1" 1759 } 1760 require.Equal(t, remaining, server.Files) 1761 } 1762 1763 func TestDeleteDiscipline_TheFinalSyncWaivesTheWindowButStillReChecks(t *testing.T) { 1764 // A deletion made in a session shorter than the waiting window still 1765 // goes through: the final sync skips the wait, and the re-check just 1766 // before the delete stands in for the second observation. 1767 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, "", nil, nil) 1768 ctx := context.Background() 1769 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1770 1771 stores.Finish(ctx) 1772 1773 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1774 require.Equal(t, map[string]string{"keep.md": "v1"}, server.Files) 1775 } 1776 1777 func TestDeleteDiscipline_AFolderDestroyedMidSyncSendsNoDeletes(t *testing.T) { 1778 // The directory scan saw an intact folder, then the folder vanished 1779 // while the server listing was in flight: the re-check just before 1780 // the delete must notice the marker is gone and hold every delete. 1781 local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsEnabled, nil) 1782 ctx := context.Background() 1783 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1784 runSync(ctx, stores) 1785 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1786 1787 server.SetOnList(func() { _ = os.RemoveAll(local) }) 1788 runSync(ctx, stores) 1789 server.SetOnList(nil) 1790 1791 require.Empty(t, server.ReceivedSnapshot()) 1792 require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files) 1793 } 1794 1795 func TestDeleteDiscipline_AnUnreadableSubdirectoryFailsThePassInsteadOfReadingAsDeletions(t *testing.T) { 1796 // Files under a directory the scan cannot enter are not treated as deleted. 1797 if os.Geteuid() == 0 { 1798 t.Skip("permission checks do not bind root") 1799 } 1800 var logBuf bytes.Buffer 1801 local, server, stores, clock := downloadedMode(t, 1802 map[string]string{"sub/a.md": "v1", "sub/b.md": "v1", "top.md": "v1"}, MemorySyncDeletionsEnabled, &logBuf) 1803 ctx := context.Background() 1804 writeLocal(t, local, "top.md", "v2") 1805 sub := filepath.Join(local, "sub") 1806 require.NoError(t, os.Chmod(sub, 0o000)) 1807 t.Cleanup(func() { _ = os.Chmod(sub, 0o700) }) 1808 1809 runSync(ctx, stores) 1810 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1811 runSync(ctx, stores) 1812 require.Empty(t, server.ReceivedSnapshot()) 1813 require.Equal(t, 2, strings.Count(logBuf.String(), "memory sync failed")) 1814 1815 // Readable again: the edit goes out and nothing is deleted. 1816 require.NoError(t, os.Chmod(sub, 0o700)) 1817 clock.now = time.Unix(0, 0).Add(2 * deleteElapsed) 1818 runSync(ctx, stores) 1819 require.Equal(t, []string{updated("top.md", "v2", "v1")}, server.ReceivedSnapshot()) 1820 require.Equal(t, map[string]string{"sub/a.md": "v1", "sub/b.md": "v1", "top.md": "v2"}, server.Files) 1821 } 1822 1823 func TestDeleteDiscipline_ARecoveryReDownloadClearsPendingDeletes(t *testing.T) { 1824 // The re-download rebuilt the disk, so an absence observed before it 1825 // says nothing about the rebuilt folder. 1826 local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1"}, MemorySyncDeletionsEnabled, nil) 1827 ctx := context.Background() 1828 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1829 runSync(ctx, stores) 1830 1831 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1832 require.NoError(t, os.RemoveAll(local)) 1833 runSync(ctx, stores) 1834 require.Equal(t, "v1", readLocal(t, local, "a.md")) 1835 1836 // A fresh deletion starts a fresh window — the pre-recovery 1837 // observation must not corroborate it. 1838 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1839 runSync(ctx, stores) 1840 require.Empty(t, server.ReceivedSnapshot()) 1841 clock.now = time.Unix(0, 0).Add(2 * deleteElapsed) 1842 runSync(ctx, stores) 1843 require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot()) 1844 } 1845 1846 func TestDeleteDiscipline_LogOnlyModeLogsTheDeleteButNeverSendsIt(t *testing.T) { 1847 // The dry-run setting: the full pipeline runs — window, re-checks, 1848 // cap — but the confirmed delete is logged instead of sent, on every 1849 // sync while the file stays missing. 1850 var logBuf bytes.Buffer 1851 local, server, stores, clock := downloadedMode(t, 1852 map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsLogOnly, &logBuf) 1853 ctx := context.Background() 1854 require.NoError(t, os.Remove(filepath.Join(local, "a.md"))) 1855 1856 runSync(ctx, stores) 1857 require.NotContains(t, logBuf.String(), "log-only") // still inside the waiting window 1858 1859 clock.now = time.Unix(0, 0).Add(deleteElapsed) 1860 runSync(ctx, stores) 1861 runSync(ctx, stores) 1862 1863 require.Empty(t, server.ReceivedSnapshot()) 1864 require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files) 1865 require.Equal(t, 2, strings.Count(logBuf.String(), "log-only: sync would delete this memory on the server")) 1866 require.Contains(t, logBuf.String(), "path=a.md") 1867 } 1868 1869 func TestDeleteDiscipline_FinishRunsOnceAndASecondCallPanics(t *testing.T) { 1870 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil) 1871 ctx := context.Background() 1872 writeLocal(t, local, "a.md", "v2") 1873 1874 stores.Finish(ctx) 1875 require.Equal(t, map[string]string{"a.md": "v2"}, server.Files) 1876 1877 require.PanicsWithValue(t, 1878 "Finish() was already called: it is the session's last sync and runs once", 1879 func() { stores.Finish(ctx) }) 1880 } 1881 1882 // ---- never destroy the only copy ------------------------------------------- 1883 // 1884 // A memory deleted remotely while its file holds an un-pushed edit keeps 1885 // the file — a writable store re-creates the memory from it, a read-only 1886 // store keeps it on disk unsynced. Only an unedited file follows the 1887 // remote delete off the disk. 1888 1889 func TestKeepOnlyCopy_AnEditedFileSurvivesARemoteDeleteAndIsReCreated(t *testing.T) { 1890 var logBuf bytes.Buffer 1891 local, server, stores := downloadedStores(t, 1892 map[string]string{"edited.md": "v1", "plain.md": "v1"}, "", &logBuf, nil) 1893 ctx := context.Background() 1894 writeLocal(t, local, "edited.md", "v2 from agent") 1895 server.Delete("edited.md") 1896 server.Delete("plain.md") 1897 1898 runSync(ctx, stores) 1899 1900 // The edit's only copy survives and the memory is re-created from it; 1901 // the unedited file follows the remote delete off the disk. 1902 require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md")) 1903 require.Equal(t, map[string]string{"edited.md": "v2 from agent"}, server.Files) 1904 require.Equal(t, []string{created("edited.md", "v2 from agent")}, server.ReceivedSnapshot()) 1905 require.NoFileExists(t, filepath.Join(local, "plain.md")) 1906 require.Contains(t, logBuf.String(), "re-creating it from the file") 1907 1908 // Settled: nothing more goes out. 1909 runSync(ctx, stores) 1910 require.Equal(t, []string{created("edited.md", "v2 from agent")}, server.ReceivedSnapshot()) 1911 1912 // The upload's sha became the baseline: the next edit is a guarded update. 1913 writeLocal(t, local, "edited.md", "v3") 1914 runSync(ctx, stores) 1915 got := server.ReceivedSnapshot() 1916 require.Equal(t, updated("edited.md", "v3", "v2 from agent"), got[len(got)-1]) 1917 } 1918 1919 func TestKeepOnlyCopy_AReadOnlyStoreKeepsTheEditedFileWithoutPushing(t *testing.T) { 1920 var logBuf bytes.Buffer 1921 local, server, stores := downloadedStores(t, 1922 map[string]string{"edited.md": "v1", "plain.md": "v1"}, "read_only", &logBuf, nil) 1923 ctx := context.Background() 1924 writeLocal(t, local, "edited.md", "v2 from agent") 1925 server.Delete("edited.md") 1926 server.Delete("plain.md") 1927 1928 runSync(ctx, stores) 1929 1930 require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md")) 1931 require.NoFileExists(t, filepath.Join(local, "plain.md")) 1932 require.Empty(t, server.ReceivedSnapshot()) 1933 require.Equal(t, 1, strings.Count(logBuf.String(), "cannot push")) 1934 1935 // Later passes see a local-only file in a read-only store: a quiet no-op. 1936 runSync(ctx, stores) 1937 runSync(ctx, stores) 1938 require.Empty(t, server.ReceivedSnapshot()) 1939 require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md")) 1940 require.Equal(t, 1, strings.Count(logBuf.String(), "cannot push")) 1941 } 1942 1943 func TestKeepOnlyCopy_ARefusedEditSurvivesARemoteDeleteWithoutARetry(t *testing.T) { 1944 // The server already refused this content, so the remote delete 1945 // neither removes the file nor triggers a re-create the server would 1946 // refuse again. 1947 local, server, stores := downloadedStores(t, map[string]string{"big.md": "v1"}, "", nil, nil) 1948 ctx := context.Background() 1949 server.MaxContentBytes = 64 1950 1951 writeLocal(t, local, "big.md", strings.Repeat("x", 100)) 1952 runSync(ctx, stores) 1953 require.Len(t, server.ReceivedSnapshot(), 1) // the one refused update 1954 1955 server.Delete("big.md") 1956 runSync(ctx, stores) 1957 runSync(ctx, stores) 1958 1959 require.Equal(t, strings.Repeat("x", 100), readLocal(t, local, "big.md")) 1960 require.Len(t, server.ReceivedSnapshot(), 1) 1961 require.NotContains(t, server.Files, "big.md") 1962 } 1963 1964 func TestKeepOnlyCopy_AMemoryDeletedBetweenTheListingAndTheUpdateIsReCreated(t *testing.T) { 1965 // An update that meets a 404 re-creates the memory in the same pass. 1966 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", nil, nil) 1967 ctx := context.Background() 1968 writeLocal(t, local, "note.md", "v2 from agent") 1969 1970 server.SetUploadHook(func(_ context.Context, path string) { 1971 server.Delete(path) 1972 server.SetUploadHook(nil) 1973 }) 1974 runSync(ctx, stores) 1975 1976 require.Equal(t, map[string]string{"note.md": "v2 from agent"}, server.Files) 1977 require.Equal(t, []string{updated("note.md", "v2 from agent", "v1"), created("note.md", "v2 from agent")}, server.ReceivedSnapshot()) 1978 1979 // The create's sha is the new baseline. 1980 writeLocal(t, local, "note.md", "v3") 1981 runSync(ctx, stores) 1982 received := server.ReceivedSnapshot() 1983 require.Equal(t, updated("note.md", "v3", "v2 from agent"), received[len(received)-1]) 1984 } 1985 1986 func TestKeepOnlyCopy_AnEditWrittenMidSyncStillSurvivesARemoteDelete(t *testing.T) { 1987 // The directory scan saw the file unedited; an edit is written while 1988 // the server listing is in flight. The removal must re-read the file 1989 // instead of destroying the edit based on the stale scan. 1990 local, server, stores := downloadedStores(t, map[string]string{"z.md": "v1", "keep.md": "v1"}, "", nil, nil) 1991 ctx := context.Background() 1992 server.Delete("z.md") 1993 1994 server.SetOnList(func() { require.NoError(t, os.WriteFile(filepath.Join(local, "z.md"), []byte("late edit"), 0o600)) }) 1995 runSync(ctx, stores) 1996 server.SetOnList(nil) 1997 1998 require.Equal(t, "late edit", readLocal(t, local, "z.md")) 1999 require.Equal(t, "late edit", server.Files["z.md"]) 2000 } 2001 2002 func TestKeepOnlyCopy_AReadOnlyPullOverALocalEditIsWarned(t *testing.T) { 2003 // Read-only stores always take the server version, but overwriting a 2004 // local edit must leave a trace instead of happening silently. 2005 var logBuf bytes.Buffer 2006 local, server, stores := downloadedStores(t, map[string]string{"pull.md": "v1"}, "read_only", &logBuf, nil) 2007 ctx := context.Background() 2008 writeLocal(t, local, "pull.md", "local edit") 2009 server.Write("pull.md", "v2 from server") 2010 2011 runSync(ctx, stores) 2012 2013 require.Equal(t, "v2 from server", readLocal(t, local, "pull.md")) 2014 require.Empty(t, server.ReceivedSnapshot()) 2015 require.Contains(t, logBuf.String(), "changed both locally and remotely") 2016 } 2017 2018 func TestKeepOnlyCopy_AFailedReReadDoesNotResurrectAServerDeletedMemory(t *testing.T) { 2019 // The re-read before removal can fail (a transient I/O error). That 2020 // must mean "retry next sync", not "file gone": dropping the entry 2021 // would make the next sync upload the file as new, re-creating the 2022 // memory the server just deleted. 2023 disk := &faults{} 2024 local, server, stores := downloadedStores(t, map[string]string{"z.md": "v1", "keep.md": "v1"}, "", nil, disk) 2025 ctx := context.Background() 2026 server.Delete("z.md") 2027 2028 disk.hashFile = func(rel string) bool { return rel == "z.md" } 2029 runSync(ctx, stores) 2030 disk.hashFile = nil 2031 2032 // The error held everything: file still on disk, nothing pushed. 2033 require.Equal(t, "v1", readLocal(t, local, "z.md")) 2034 require.Empty(t, server.ReceivedSnapshot()) 2035 2036 // With the error gone, the next sync completes the server delete. 2037 runSync(ctx, stores) 2038 require.NoFileExists(t, filepath.Join(local, "z.md")) 2039 require.NotContains(t, server.Files, "z.md") 2040 require.Empty(t, server.ReceivedSnapshot()) 2041 } 2042 2043 // ---- FlushWrites — the push-only shutdown pass ----------------------------- 2044 // 2045 // A session that ends on an error or cancel gets no full reconcile, but 2046 // the writes already on disk are the only copy of the agent's edits. 2047 // FlushWrites pushes those and does nothing else: no remote deletes, no 2048 // local removals, no pulls. 2049 2050 func TestFinalFlush_PushesNewAndChangedFilesAndNothingElse(t *testing.T) { 2051 var logBuf bytes.Buffer 2052 local, server, stores := downloadedStores(t, map[string]string{ 2053 "a-pull.md": "v1", 2054 "b-push.md": "v1", 2055 "d-local-gone.md": "v1", 2056 "e-remote-gone.md": "v1", 2057 "f-remote-gone-edited.md": "v1", 2058 "g-both.md": "v1", 2059 }, "", &logBuf, nil) 2060 ctx := context.Background() 2061 2062 writeLocal(t, local, "b-push.md", "v2 from agent") 2063 writeLocal(t, local, "c-new.md", "new from agent") 2064 require.NoError(t, os.Remove(filepath.Join(local, "d-local-gone.md"))) 2065 writeLocal(t, local, "f-remote-gone-edited.md", "v2 from agent") 2066 writeLocal(t, local, "g-both.md", "v2 from agent") 2067 server.Write("a-pull.md", "v2 from server") 2068 server.Delete("e-remote-gone.md") 2069 server.Delete("f-remote-gone-edited.md") 2070 server.Write("g-both.md", "v2 from server") 2071 2072 stores.FlushWrites(ctx) 2073 2074 // Only additions went out: the update, the new file, and the 2075 // re-created edited file whose remote copy was deleted. 2076 sent := []string{ 2077 updated("b-push.md", "v2 from agent", "v1"), 2078 created("c-new.md", "new from agent"), 2079 created("f-remote-gone-edited.md", "v2 from agent"), 2080 } 2081 require.ElementsMatch(t, sent, server.ReceivedSnapshot()) 2082 // A locally missing file was not deleted remotely. 2083 require.Equal(t, "v1", server.Files["d-local-gone.md"]) 2084 // A remotely deleted, locally unedited memory was not re-uploaded — 2085 // and its file was not removed from disk either. 2086 require.NotContains(t, server.Files, "e-remote-gone.md") 2087 require.Equal(t, "v1", readLocal(t, local, "e-remote-gone.md")) 2088 // Remote changes were not pulled. 2089 require.Equal(t, "v1", readLocal(t, local, "a-pull.md")) 2090 // The conflict was left to the server, and logged. 2091 require.Equal(t, "v2 from server", server.Files["g-both.md"]) 2092 require.Equal(t, "v2 from agent", readLocal(t, local, "g-both.md")) 2093 require.Contains(t, logBuf.String(), "changed both locally and remotely") 2094 2095 // Settled: a second flush pushes nothing more. 2096 stores.FlushWrites(ctx) 2097 require.ElementsMatch(t, sent, server.ReceivedSnapshot()) 2098 } 2099 2100 func TestFinalFlush_UploadsSeveralFilesAtOnceUpToTheBound(t *testing.T) { 2101 // The bound's worth of uploads are in flight together, and never one 2102 // more — a serial flush would time out waiting for the second to start. 2103 local, server, stores := downloadedStores(t, map[string]string{}, "", nil, nil) 2104 want := map[string]string{} 2105 for i := range uploadConcurrency + 3 { 2106 name, content := fmt.Sprintf("f%02d.md", i), fmt.Sprintf("file %d", i) 2107 writeLocal(t, local, name, content) 2108 want[name] = content 2109 } 2110 2111 var mu sync.Mutex 2112 inFlight, peak := 0, 0 2113 release := make(chan struct{}) 2114 var once sync.Once 2115 server.SetUploadHook(func(ctx context.Context, _ string) { 2116 mu.Lock() 2117 inFlight++ 2118 peak = max(peak, inFlight) 2119 full := inFlight == uploadConcurrency 2120 mu.Unlock() 2121 if full { 2122 once.Do(func() { 2123 // Hold the full set briefly so a flush that ignored the bound 2124 // has time to start one more before any of these finish. 2125 time.Sleep(100 * time.Millisecond) 2126 close(release) 2127 }) 2128 } 2129 select { 2130 case <-release: 2131 case <-ctx.Done(): 2132 case <-time.After(10 * time.Second): 2133 t.Error("the uploads never filled the bound: they ran serially") 2134 once.Do(func() { close(release) }) 2135 } 2136 mu.Lock() 2137 inFlight-- 2138 mu.Unlock() 2139 }) 2140 2141 stores.FlushWrites(context.Background()) 2142 2143 mu.Lock() 2144 got := peak 2145 mu.Unlock() 2146 require.Equal(t, uploadConcurrency, got) 2147 require.Equal(t, want, server.Files) 2148 } 2149 2150 func TestFinalFlush_AFlushCutOffPartWayLogsHowManyFilesItHadNotUploaded(t *testing.T) { 2151 var logBuf bytes.Buffer 2152 local, server, stores := downloadedStores(t, map[string]string{"z.md": "z0"}, "", &logBuf, nil) 2153 names := []string{"a.md", "b.md", "c.md", "y.md", "z.md"} 2154 for _, name := range names { 2155 writeLocal(t, local, name, "content of "+name) 2156 } 2157 2158 // Two uploads — one create, one update — never complete; the flush's 2159 // own deadline cuts them off. 2160 server.SetUploadHook(func(ctx context.Context, path string) { 2161 if path == "y.md" || path == "z.md" { 2162 <-ctx.Done() 2163 } 2164 }) 2165 ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) 2166 defer cancel() 2167 stores.FlushWrites(ctx) 2168 2169 require.Equal(t, 1, strings.Count(logBuf.String(), "memory flush cut off part-way")) 2170 require.Contains(t, logBuf.String(), 2171 `2 of 5 changed files had not finished uploading" memory_store_id=memstore_notes`) 2172 require.Equal(t, map[string]string{ 2173 "a.md": "content of a.md", "b.md": "content of b.md", "c.md": "content of c.md", "z.md": "z0", 2174 }, server.Files) 2175 require.NotContains(t, logBuf.String(), "failed to upload memory") 2176 require.NotContains(t, logBuf.String(), "memory flush failed") 2177 2178 // The three that made it are not sent again; the two that did not are. 2179 // A flush that finishes in time adds no cut-off line. 2180 server.SetUploadHook(nil) 2181 stores.FlushWrites(context.Background()) 2182 want := map[string]string{} 2183 for _, name := range names { 2184 want[name] = "content of " + name 2185 } 2186 require.Equal(t, want, server.Files) 2187 require.ElementsMatch(t, []string{ 2188 created("a.md", "content of a.md"), 2189 created("b.md", "content of b.md"), 2190 created("c.md", "content of c.md"), 2191 created("y.md", "content of y.md"), 2192 updated("z.md", "content of z.md", "z0"), 2193 }, server.ReceivedSnapshot()) 2194 require.Equal(t, 1, strings.Count(logBuf.String(), "cut off part-way")) 2195 } 2196 2197 func TestFinalFlush_AdoptsAFileTheServerAlreadyHolds(t *testing.T) { 2198 // A final sync cut off after uploading a file leaves the baseline stale; 2199 // the flush sees the server already holds those bytes and records that 2200 // instead of reporting a conflict. 2201 var logBuf bytes.Buffer 2202 local, server, stores := downloadedStores(t, map[string]string{"n.md": "v1"}, "", &logBuf, nil) 2203 ctx := context.Background() 2204 writeLocal(t, local, "n.md", "v2") 2205 server.Write("n.md", "v2") 2206 2207 stores.FlushWrites(ctx) 2208 2209 require.Empty(t, server.ReceivedSnapshot()) 2210 require.NotContains(t, logBuf.String(), "changed both locally and remotely") 2211 2212 // The adopted sha became the baseline: the next edit is a guarded update. 2213 writeLocal(t, local, "n.md", "v3") 2214 stores.FlushWrites(ctx) 2215 require.Equal(t, []string{updated("n.md", "v3", "v2")}, server.ReceivedSnapshot()) 2216 } 2217 2218 func TestFinalFlush_ARefusedFileDoesNotHoldBackTheOthers(t *testing.T) { 2219 // A refusal and a success are book-kept from two uploads of one flush: 2220 // the refused file is tried once and not retried, the other lands once. 2221 var logBuf bytes.Buffer 2222 local, server, stores := downloadedStores(t, map[string]string{}, "", &logBuf, nil) 2223 ctx := context.Background() 2224 server.MaxContentBytes = 64 2225 writeLocal(t, local, "big.md", strings.Repeat("x", 100)) 2226 writeLocal(t, local, "ok.md", "fits") 2227 2228 stores.FlushWrites(ctx) 2229 stores.FlushWrites(ctx) 2230 2231 require.Equal(t, "fits", server.Files["ok.md"]) 2232 require.NotContains(t, server.Files, "big.md") 2233 require.ElementsMatch(t, 2234 []string{created("big.md", strings.Repeat("x", 100)), created("ok.md", "fits")}, 2235 server.ReceivedSnapshot()) 2236 require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes")) 2237 } 2238 2239 func TestFinalFlush_NeverPushesAReadOnlyStore(t *testing.T) { 2240 local, server, stores := downloadedStores(t, map[string]string{"edit.md": "v1"}, "read_only", nil, nil) 2241 ctx := context.Background() 2242 2243 writeLocal(t, local, "edit.md", "v2 from agent") 2244 writeLocal(t, local, "new.md", "new from agent") 2245 2246 stores.FlushWrites(ctx) 2247 2248 require.Empty(t, server.ReceivedSnapshot()) 2249 require.Equal(t, map[string]string{"edit.md": "v1"}, server.Files) 2250 } 2251 2252 func TestFinalFlush_SkipsFilesTheServerAlreadyRefused(t *testing.T) { 2253 local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", nil, nil) 2254 ctx := context.Background() 2255 server.MaxContentBytes = 64 2256 2257 writeLocal(t, local, "big.md", strings.Repeat("x", 100)) 2258 runSync(ctx, stores) 2259 creates := 0 2260 for _, r := range server.ReceivedSnapshot() { 2261 if strings.HasPrefix(r, "create ") { 2262 creates++ 2263 } 2264 } 2265 require.Equal(t, 1, creates) 2266 2267 stores.FlushWrites(ctx) 2268 2269 // The refused file was not retried; nothing else needed pushing. 2270 creates = 0 2271 for _, r := range server.ReceivedSnapshot() { 2272 if strings.HasPrefix(r, "create ") { 2273 creates++ 2274 } 2275 } 2276 require.Equal(t, 1, creates) 2277 require.NotContains(t, server.Files, "big.md") 2278 } 2279 2280 func TestFinalFlush_PushesNothingWhenTheMarkerCheckFails(t *testing.T) { 2281 var logBuf bytes.Buffer 2282 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil) 2283 ctx := context.Background() 2284 2285 require.NoError(t, os.Remove(filepath.Join(local, MarkerPath))) 2286 writeLocal(t, local, "foreign.md", "someone else's notes") 2287 2288 stores.FlushWrites(ctx) 2289 2290 require.Empty(t, server.ReceivedSnapshot()) 2291 require.NotContains(t, server.Files, "foreign.md") 2292 // Unlike sync's recovery, the flush does not re-download the folder. 2293 require.NoFileExists(t, filepath.Join(local, MarkerPath)) 2294 require.Contains(t, logBuf.String(), "not uploading anything from the memory store folder") 2295 } 2296 2297 func TestFinalFlush_NeverFails(t *testing.T) { 2298 var logBuf bytes.Buffer 2299 local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil) 2300 ctx := context.Background() 2301 writeLocal(t, local, "a.md", "v2 from agent") 2302 2303 server.SetOnList(func() { panic(http.ErrAbortHandler) }) // the listing explodes 2304 stores.FlushWrites(ctx) 2305 server.SetOnList(nil) 2306 2307 require.Empty(t, server.ReceivedSnapshot()) 2308 require.Contains(t, logBuf.String(), "memory flush failed") 2309 }