github.com/anthropics/anthropic-sdk-go@v1.70.1/lib/environments/memories_test.go (about)

     1  package environments
     2  
     3  // The memory-store sync contract.
     4  //
     5  // An agent keeps a folder of notes on disk. The same folder lives on the
     6  // server. Each sync makes the two agree: each side gets the other's
     7  // changes, and when both changed the same file the server wins.
     8  //
     9  // Each test tells one story with two actors — `local` is the agent's folder
    10  // on disk, `server` is the remote copy. Paths are the same string on both
    11  // sides. server.Received holds only what the sync sent; arranging remote
    12  // state never touches it.
    13  
    14  import (
    15  	"bytes"
    16  	"context"
    17  	"crypto/sha256"
    18  	"encoding/hex"
    19  	"encoding/json"
    20  	"fmt"
    21  	"io"
    22  	"log/slog"
    23  	"net/http"
    24  	"net/http/httptest"
    25  	"net/url"
    26  	"os"
    27  	"path/filepath"
    28  	"slices"
    29  	"strings"
    30  	"sync"
    31  	"sync/atomic"
    32  	"testing"
    33  	"time"
    34  
    35  	anthropic "github.com/anthropics/anthropic-sdk-go"
    36  	"github.com/anthropics/anthropic-sdk-go/internal/filestore"
    37  	"github.com/anthropics/anthropic-sdk-go/option"
    38  	"github.com/stretchr/testify/require"
    39  )
    40  
    41  func shaHex(content string) string {
    42  	sum := sha256.Sum256([]byte(content))
    43  	return hex.EncodeToString(sum[:])
    44  }
    45  
    46  // Requests recorded on memoryServer.Received, in domain terms: paths and
    47  // content. `was` is the pre-image content the request's precondition guards
    48  // on — the builder computes the sha.
    49  
    50  func created(path, content string) string { return "create " + path + " " + content }
    51  
    52  func updated(path, content, was string) string {
    53  	return "update " + path + " " + content + " if=" + shaHex(was)
    54  }
    55  
    56  func deletedReq(path, was string) string { return "delete " + path + " if=" + shaHex(was) }
    57  
    58  // memoryServer is the test's view of one remote memory store named "notes".
    59  //
    60  // Files is its current state (path → content); Received is every write the
    61  // code under test sent it. Arranging via Write / Delete changes Files without
    62  // touching Received — only the sync's own requests are recorded. It serves
    63  // both the sessions lookup (recording Retrieves) and the memory_stores
    64  // endpoints, so one server backs the pure SessionMemoryStores tests and the
    65  // worker e2e tests alike.
    66  type memoryServer struct {
    67  	t *testing.T
    68  
    69  	mu        sync.Mutex
    70  	Files     map[string]string
    71  	Received  []string
    72  	Retrieves []string
    73  	// ContentFetches is every single-memory content fetch (the sync's
    74  	// content pass), by path, in arrival order — a test asserting "no
    75  	// content moved" checks this stays empty.
    76  	ContentFetches []string
    77  
    78  	access   string // "" (read_write) or "read_only"
    79  	OnUpdate func() // race hook: runs before an update is processed
    80  	onList   func() // race hook: runs after a listing is served; set via SetOnList
    81  	// onFetch is a race hook that runs as each content fetch arrives, with
    82  	// the server mutex held (Files is safe to touch); a non-zero return
    83  	// fails the fetch with that status. Set via SetOnFetch.
    84  	onFetch func(path string) int
    85  	// uploadHook runs as each create or update arrives, with the memory's
    86  	// path, before the request is recorded and without the server mutex —
    87  	// block in it to hold an upload open. A request whose client gave up
    88  	// while held is dropped unrecorded. Set via SetUploadHook.
    89  	uploadHook func(ctx context.Context, path string)
    90  	// FailUpdates answers every update with a 500 — recorded, so the attempt
    91  	// is visible, but never applied.
    92  	FailUpdates bool
    93  	// FailUploads answers every create or update with this status, before it
    94  	// is recorded or applied. Zero means none.
    95  	FailUploads int
    96  	// MaxContentBytes answers any create/update whose content exceeds it
    97  	// with a 400 — recorded, never applied. Zero means no cap.
    98  	MaxContentBytes int
    99  	// Broken attaches a second store, "memstore_broken", listed before the
   100  	// healthy one, whose listing yields one memory and then explodes on the
   101  	// next page — so a half-download is observable.
   102  	Broken bool
   103  	// MountPath is emitted on the "notes" store resource (null when empty).
   104  	MountPath string
   105  	// Noise adds a non-store resource to the session and a memory_prefix
   106  	// rollup to every listing.
   107  	Noise bool
   108  	// NoStores drops the "notes" store from the session, leaving one with no
   109  	// memory attached at all.
   110  	NoStores bool
   111  }
   112  
   113  func newMemoryServer(t *testing.T, initial map[string]string, access string) *memoryServer {
   114  	files := map[string]string{}
   115  	for k, v := range initial {
   116  		files[k] = v
   117  	}
   118  	return &memoryServer{t: t, Files: files, access: access}
   119  }
   120  
   121  func (m *memoryServer) Write(path, content string) {
   122  	m.mu.Lock()
   123  	defer m.mu.Unlock()
   124  	m.Files[path] = content
   125  }
   126  
   127  func (m *memoryServer) Delete(path string) {
   128  	m.mu.Lock()
   129  	defer m.mu.Unlock()
   130  	delete(m.Files, path)
   131  }
   132  
   133  func (m *memoryServer) SetOnList(fn func()) {
   134  	m.mu.Lock()
   135  	defer m.mu.Unlock()
   136  	m.onList = fn
   137  }
   138  
   139  func (m *memoryServer) SetOnFetch(fn func(path string) int) {
   140  	m.mu.Lock()
   141  	defer m.mu.Unlock()
   142  	m.onFetch = fn
   143  }
   144  
   145  func (m *memoryServer) SetUploadHook(fn func(ctx context.Context, path string)) {
   146  	m.mu.Lock()
   147  	defer m.mu.Unlock()
   148  	m.uploadHook = fn
   149  }
   150  
   151  func (m *memoryServer) ReceivedSnapshot() []string {
   152  	m.mu.Lock()
   153  	defer m.mu.Unlock()
   154  	return append([]string(nil), m.Received...)
   155  }
   156  
   157  // ContentFetchesSnapshot returns the content fetches so far, sorted — they
   158  // fan out, so arrival order is not deterministic.
   159  func (m *memoryServer) ContentFetchesSnapshot() []string {
   160  	m.mu.Lock()
   161  	defer m.mu.Unlock()
   162  	return slices.Sorted(slices.Values(m.ContentFetches))
   163  }
   164  
   165  func (m *memoryServer) ClearContentFetches() {
   166  	m.mu.Lock()
   167  	defer m.mu.Unlock()
   168  	m.ContentFetches = nil
   169  }
   170  
   171  // memID round-trips a path through the URL-safe id the fake hands out.
   172  func memID(path string) string { return "mem_" + hex.EncodeToString([]byte(path)) }
   173  
   174  func memIDPath(id string) string {
   175  	raw, _ := hex.DecodeString(strings.TrimPrefix(id, "mem_"))
   176  	return string(raw)
   177  }
   178  
   179  // itemJSON renders one memory the way the API does: the basic view carries
   180  // the sha but no content, the full view carries both.
   181  func (m *memoryServer) itemJSON(path string, full bool) map[string]any {
   182  	content := m.Files[path]
   183  	item := map[string]any{
   184  		"type": "memory", "id": memID(path), "path": "/" + path,
   185  		"content_sha256":     shaHex(content),
   186  		"content_size_bytes": len(content),
   187  		"created_at":         "2026-05-11T12:00:00Z", "updated_at": "2026-05-11T12:00:00Z",
   188  		"memory_store_id": "memstore_notes", "memory_version_id": "memver_1",
   189  	}
   190  	if full {
   191  		item["content"] = content
   192  	}
   193  	return item
   194  }
   195  
   196  func isFullView(r *http.Request) bool { return r.URL.Query().Get("view") == "full" }
   197  
   198  func (m *memoryServer) sessionJSON() string {
   199  	var access any
   200  	if m.access != "" {
   201  		access = m.access
   202  	}
   203  	resources := []map[string]any{}
   204  	if m.Broken {
   205  		resources = append(resources, map[string]any{
   206  			"type": "memory_store", "memory_store_id": "memstore_broken",
   207  			"mount_path": nil, "name": "broken", "access": nil,
   208  		})
   209  	}
   210  	if m.Noise {
   211  		resources = append(resources, map[string]any{"type": "file", "file_id": "file_1"})
   212  	}
   213  	var mount any
   214  	if m.MountPath != "" {
   215  		mount = m.MountPath
   216  	}
   217  	if !m.NoStores {
   218  		resources = append(resources, map[string]any{
   219  			"type": "memory_store", "memory_store_id": "memstore_notes",
   220  			"mount_path": mount, "name": "notes", "access": access,
   221  		})
   222  	}
   223  	body, _ := json.Marshal(map[string]any{
   224  		"id":        "s1",
   225  		"agent":     map[string]any{"skills": []any{}},
   226  		"resources": resources,
   227  	})
   228  	return string(body)
   229  }
   230  
   231  func statusJSON(w http.ResponseWriter, code int) {
   232  	w.Header().Set("Content-Type", "application/json")
   233  	w.WriteHeader(code)
   234  	fmt.Fprintf(w, `{"type":"error","error":{"type":"api_error","message":"status %d"}}`, code)
   235  }
   236  
   237  // uploadTarget reports the memory path a create or update request is for;
   238  // false for every other request. The body is drained and rewound first: a
   239  // create names its path there, and net/http only notices a client that
   240  // gave up once the body has been read to the end.
   241  func uploadTarget(r *http.Request) (string, bool) {
   242  	if r.Method != http.MethodPost {
   243  		return "", false
   244  	}
   245  	body, _ := io.ReadAll(r.Body)
   246  	r.Body = io.NopCloser(bytes.NewReader(body))
   247  	rest := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/v1/memory_stores/"), "/", 3)
   248  	switch {
   249  	case len(rest) == 2 && rest[1] == "memories":
   250  		var in struct{ Path string }
   251  		_ = json.Unmarshal(body, &in)
   252  		return strings.TrimLeft(in.Path, "/"), true
   253  	case len(rest) == 3 && rest[1] == "memories":
   254  		id, _ := url.PathUnescape(rest[2])
   255  		return memIDPath(strings.SplitN(id, "?", 2)[0]), true
   256  	default:
   257  		return "", false
   258  	}
   259  }
   260  
   261  // handleMemories serves the /v1/memory_stores/... endpoints against the
   262  // in-memory store.
   263  func (m *memoryServer) handleMemories(w http.ResponseWriter, r *http.Request) {
   264  	if rel, ok := uploadTarget(r); ok {
   265  		m.mu.Lock()
   266  		hook := m.uploadHook
   267  		m.mu.Unlock()
   268  		if hook != nil {
   269  			hook(r.Context(), rel)
   270  			if r.Context().Err() != nil {
   271  				return
   272  			}
   273  		}
   274  		if m.FailUploads != 0 {
   275  			statusJSON(w, m.FailUploads)
   276  			return
   277  		}
   278  	}
   279  	m.mu.Lock()
   280  	defer m.mu.Unlock()
   281  	w.Header().Set("Content-Type", "application/json")
   282  	rest := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/v1/memory_stores/"), "/", 3)
   283  	switch {
   284  	case rest[0] == "memstore_broken":
   285  		// One memory is listed, then the pager explodes on the next page.
   286  		if r.URL.Query().Get("page") == "" {
   287  			half := map[string]any{
   288  				"type": "memory", "id": "mem_half", "path": "/half.md",
   289  				"content_sha256":     shaHex("partial"),
   290  				"content_size_bytes": len("partial"),
   291  				"created_at":         "2026-05-11T12:00:00Z", "updated_at": "2026-05-11T12:00:00Z",
   292  				"memory_store_id": "memstore_broken", "memory_version_id": "memver_1",
   293  			}
   294  			if isFullView(r) {
   295  				half["content"] = "partial"
   296  			}
   297  			data := []any{}
   298  			if m.Noise {
   299  				data = append(data, map[string]any{"type": "memory_prefix", "path": "/projects/"})
   300  			}
   301  			data = append(data, half)
   302  			_ = json.NewEncoder(w).Encode(map[string]any{"data": data, "next_page": "boom"})
   303  			return
   304  		}
   305  		statusJSON(w, 500)
   306  	case len(rest) == 2 && rest[1] == "memories" && r.Method == http.MethodGet:
   307  		items := make([]map[string]any, 0, len(m.Files)+1)
   308  		if m.Noise {
   309  			// Depth-limited listings roll directories up as prefixes; only
   310  			// "memory" items carry content.
   311  			items = append(items, map[string]any{"type": "memory_prefix", "path": "/projects/"})
   312  		}
   313  		full := isFullView(r)
   314  		for path := range m.Files {
   315  			items = append(items, m.itemJSON(path, full))
   316  		}
   317  		_ = json.NewEncoder(w).Encode(map[string]any{"data": items, "next_page": nil})
   318  		if m.onList != nil {
   319  			m.onList()
   320  		}
   321  	case len(rest) == 2 && rest[1] == "memories" && r.Method == http.MethodPost:
   322  		var in struct{ Path, Content string }
   323  		body, _ := io.ReadAll(r.Body)
   324  		require.NoError(m.t, json.Unmarshal(body, &in))
   325  		bare := strings.TrimLeft(in.Path, "/")
   326  		m.Received = append(m.Received, created(bare, in.Content))
   327  		if m.MaxContentBytes > 0 && len(in.Content) > m.MaxContentBytes {
   328  			statusJSON(w, 400)
   329  			return
   330  		}
   331  		m.Files[bare] = in.Content
   332  		_ = json.NewEncoder(w).Encode(m.itemJSON(bare, true))
   333  	case len(rest) == 3 && rest[1] == "memories":
   334  		id, _ := url.PathUnescape(rest[2])
   335  		id = strings.SplitN(id, "?", 2)[0]
   336  		path := memIDPath(id)
   337  		switch r.Method {
   338  		case http.MethodGet: // single-memory content fetch
   339  			m.ContentFetches = append(m.ContentFetches, path)
   340  			if m.onFetch != nil {
   341  				if code := m.onFetch(path); code != 0 {
   342  					statusJSON(w, code)
   343  					return
   344  				}
   345  			}
   346  			if _, ok := m.Files[path]; !ok {
   347  				statusJSON(w, 404)
   348  				return
   349  			}
   350  			_ = json.NewEncoder(w).Encode(m.itemJSON(path, isFullView(r)))
   351  		case http.MethodPost: // update
   352  			if m.OnUpdate != nil {
   353  				m.OnUpdate()
   354  			}
   355  			var in struct {
   356  				Content      string
   357  				Precondition struct {
   358  					ContentSha256 string `json:"content_sha256"`
   359  				}
   360  			}
   361  			body, _ := io.ReadAll(r.Body)
   362  			require.NoError(m.t, json.Unmarshal(body, &in))
   363  			m.Received = append(m.Received, "update "+path+" "+in.Content+" if="+in.Precondition.ContentSha256)
   364  			if m.MaxContentBytes > 0 && len(in.Content) > m.MaxContentBytes {
   365  				statusJSON(w, 400)
   366  				return
   367  			}
   368  			if m.FailUpdates {
   369  				statusJSON(w, 500)
   370  				return
   371  			}
   372  			current, ok := m.Files[path]
   373  			if !ok {
   374  				statusJSON(w, 404)
   375  				return
   376  			}
   377  			if shaHex(current) != in.Precondition.ContentSha256 {
   378  				statusJSON(w, 409)
   379  				return
   380  			}
   381  			m.Files[path] = in.Content
   382  			_ = json.NewEncoder(w).Encode(m.itemJSON(path, true))
   383  		case http.MethodDelete:
   384  			expected := r.URL.Query().Get("expected_content_sha256")
   385  			m.Received = append(m.Received, "delete "+path+" if="+expected)
   386  			current, ok := m.Files[path]
   387  			if !ok {
   388  				statusJSON(w, 404)
   389  				return
   390  			}
   391  			if expected != "" && shaHex(current) != expected {
   392  				statusJSON(w, 409)
   393  				return
   394  			}
   395  			delete(m.Files, path)
   396  			_ = json.NewEncoder(w).Encode(map[string]any{"id": id, "type": "memory_deleted"})
   397  		default:
   398  			statusJSON(w, 405)
   399  		}
   400  	default:
   401  		m.t.Errorf("unexpected memory_stores request: %s %s", r.Method, r.URL.Path)
   402  		statusJSON(w, 404)
   403  	}
   404  }
   405  
   406  func (m *memoryServer) serveHTTP(w http.ResponseWriter, r *http.Request) {
   407  	switch {
   408  	case strings.HasPrefix(r.URL.Path, "/v1/memory_stores/"):
   409  		m.handleMemories(w, r)
   410  	case strings.HasPrefix(r.URL.Path, "/v1/sessions/"):
   411  		m.mu.Lock()
   412  		m.Retrieves = append(m.Retrieves, strings.TrimPrefix(r.URL.Path, "/v1/sessions/"))
   413  		m.mu.Unlock()
   414  		w.Header().Set("Content-Type", "application/json")
   415  		_, _ = w.Write([]byte(m.sessionJSON()))
   416  	default:
   417  		m.t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
   418  		statusJSON(w, 404)
   419  	}
   420  }
   421  
   422  // fetchSession retrieves the fake server's session. Download takes that
   423  // snapshot and never looks the session up itself, so every test fetches it the
   424  // way the worker does — once, up front.
   425  func fetchSession(t *testing.T, client anthropic.Client) *anthropic.BetaManagedAgentsSession {
   426  	t.Helper()
   427  	session, err := client.Beta.Sessions.Get(context.Background(), "s1", anthropic.BetaSessionGetParams{})
   428  	require.NoError(t, err)
   429  	return session
   430  }
   431  
   432  // mustStores is NewSessionMemoryStores for options a test knows are valid.
   433  func mustStores(t *testing.T, client anthropic.Client, opts SessionMemoryStoresOptions) *SessionMemoryStores {
   434  	t.Helper()
   435  	stores, err := NewSessionMemoryStores(client, opts)
   436  	require.NoError(t, err)
   437  	return stores
   438  }
   439  
   440  // faults selects which disk verbs fail, and for which paths, and can act on
   441  // the disk just before a write. The zero value does nothing; a test sets a
   442  // field to break the verb it is exercising and clears it again to watch the
   443  // retry. The disk events the baseline rules are written against — a write
   444  // that never lands, a removal that cannot happen, a folder wiped between two
   445  // writes — have no reliable fixture, so they are injected here.
   446  type faults struct {
   447  	put      func(rel string) bool
   448  	remove   func(rel string) bool
   449  	hashFile func(rel string) bool
   450  	// onPut runs as each Put arrives, before it reaches the disk — a race
   451  	// hook to land a change on disk between two of a pass's writes.
   452  	onPut func(rel string)
   453  }
   454  
   455  // always breaks a verb for every path.
   456  func always(string) bool { return true }
   457  
   458  // open is the [SessionMemoryStores.openStore] seam: every store opens through
   459  // a faultyStore reading these faults live.
   460  func (f *faults) open(root string, opts ...filestore.Option) (storeFiles, error) {
   461  	files, err := filestore.Open(root, opts...)
   462  	if err != nil {
   463  		return nil, err
   464  	}
   465  	return &faultyStore{storeFiles: files, faults: f}, nil
   466  }
   467  
   468  type faultyStore struct {
   469  	storeFiles
   470  	faults *faults
   471  }
   472  
   473  func (f *faultyStore) Put(rel string, data []byte, opts ...filestore.PutOption) error {
   474  	if f.faults.onPut != nil {
   475  		f.faults.onPut(rel)
   476  	}
   477  	if f.faults.put != nil && f.faults.put(rel) {
   478  		return fmt.Errorf("injected write failure: %s", rel)
   479  	}
   480  	return f.storeFiles.Put(rel, data, opts...)
   481  }
   482  
   483  func (f *faultyStore) Remove(rel string) error {
   484  	if f.faults.remove != nil && f.faults.remove(rel) {
   485  		return fmt.Errorf("injected removal failure: %s", rel)
   486  	}
   487  	return f.storeFiles.Remove(rel)
   488  }
   489  
   490  func (f *faultyStore) HashFile(rel string) (string, error) {
   491  	if f.faults.hashFile != nil && f.faults.hashFile(rel) {
   492  		return "", fmt.Errorf("injected hash failure: %s", rel)
   493  	}
   494  	return f.storeFiles.HashFile(rel)
   495  }
   496  
   497  // captureLog is a debug-level text logger writing into buf, for tests that
   498  // assert on log output.
   499  func captureLog(buf *bytes.Buffer) *slog.Logger {
   500  	return slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
   501  }
   502  
   503  // downloadedStores returns a SessionMemoryStores with the fake server's one
   504  // store already downloaded to disk, plus the local dir and the server handle.
   505  // disk, when non-nil, opens every store through a faultyStore.
   506  func downloadedStores(t *testing.T, initial map[string]string, access string, logBuf *bytes.Buffer, disk *faults) (string, *memoryServer, *SessionMemoryStores) {
   507  	t.Helper()
   508  	server := newMemoryServer(t, initial, access)
   509  	httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP))
   510  	t.Cleanup(httpSrv.Close)
   511  	client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0))
   512  
   513  	logger := silentLogger
   514  	if logBuf != nil {
   515  		logger = captureLog(logBuf)
   516  	}
   517  	workdir := t.TempDir()
   518  	stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: workdir, Logger: logger})
   519  	if disk != nil {
   520  		stores.openStore = disk.open
   521  	}
   522  	require.NoError(t, stores.Download(context.Background(), fetchSession(t, client)))
   523  	return filepath.Join(workdir, "memory", "notes"), server, stores
   524  }
   525  
   526  // testClock is a settable stand-in for the store's time source.
   527  type testClock struct{ now time.Time }
   528  
   529  func (c *testClock) Now() time.Time { return c.now }
   530  
   531  // withClock swaps in a fake clock at time zero and returns a handle to it.
   532  func withClock(stores *SessionMemoryStores) *testClock {
   533  	c := &testClock{now: time.Unix(0, 0)}
   534  	stores.setClock(c.Now)
   535  	return c
   536  }
   537  
   538  // runSync drives one reconcile pass without clock bookkeeping.
   539  //
   540  // The public cadence path (SyncIfDue plus the clock) is exercised by the
   541  // worker tests; unit tests call the pass directly.
   542  func runSync(ctx context.Context, s *SessionMemoryStores) {
   543  	s.syncAll(ctx, false)
   544  }
   545  
   546  func writeLocal(t *testing.T, dir, name, content string) {
   547  	t.Helper()
   548  	require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600))
   549  }
   550  
   551  func readLocal(t *testing.T, dir, name string) string {
   552  	t.Helper()
   553  	got, err := os.ReadFile(filepath.Join(dir, name))
   554  	require.NoError(t, err)
   555  	return string(got)
   556  }
   557  
   558  func TestMemorySync_EditsFlowBothWaysAndTheServerWinsConflicts(t *testing.T) {
   559  	var logBuf bytes.Buffer
   560  	local, server, stores := downloadedStores(t,
   561  		map[string]string{"push.md": "v1", "pull.md": "v1", "both.md": "v1", "gone.md": "v1"}, "", &logBuf, nil)
   562  	ctx := context.Background()
   563  
   564  	writeLocal(t, local, "push.md", "v2 from agent")
   565  	writeLocal(t, local, "new.md", "new from agent")
   566  	server.Write("pull.md", "v2 from server")
   567  	writeLocal(t, local, "both.md", "v2 from agent")
   568  	server.Write("both.md", "v2 from server")
   569  	server.Delete("gone.md")
   570  
   571  	runSync(ctx, stores)
   572  
   573  	// Each side has the other's change; the server won the conflict; the
   574  	// server's deletion took the local copy with it.
   575  	require.Equal(t, "v2 from agent", server.Files["push.md"])
   576  	require.Equal(t, "new from agent", server.Files["new.md"])
   577  	require.Equal(t, "v2 from server", readLocal(t, local, "pull.md"))
   578  	require.Equal(t, "v2 from server", readLocal(t, local, "both.md"))
   579  	require.NoFileExists(t, filepath.Join(local, "gone.md"))
   580  	require.NotContains(t, server.Files, "gone.md")
   581  	require.Contains(t, logBuf.String(), "changed both locally and remotely")
   582  
   583  	// The push was guarded: overwrite only if the server still had v1.
   584  	sent := []string{created("new.md", "new from agent"), updated("push.md", "v2 from agent", "v1")}
   585  	require.ElementsMatch(t, sent, server.ReceivedSnapshot())
   586  
   587  	// Settled: a second sync sends nothing more.
   588  	runSync(ctx, stores)
   589  	require.Len(t, server.ReceivedSnapshot(), len(sent))
   590  }
   591  
   592  func TestMemorySync_LocalDeletionsReachTheServerUnlessItChanged(t *testing.T) {
   593  	// One memory's download write fails, so it is never on disk to begin with
   594  	// — that absence is not a deletion.
   595  	disk := &faults{put: func(rel string) bool { return rel == "never.md" }}
   596  	// keep.md survives on disk so the pass is not an all-files-gone wipe.
   597  	local, server, stores := downloadedStores(t,
   598  		map[string]string{"drop.md": "v1", "raced.md": "v1", "never.md": "v1", "keep.md": "v1"}, "", nil, disk)
   599  	disk.put = nil
   600  	clock := withClock(stores)
   601  	ctx := context.Background()
   602  	require.NoFileExists(t, filepath.Join(local, "never.md"))
   603  
   604  	require.NoError(t, os.Remove(filepath.Join(local, "drop.md")))
   605  	require.NoError(t, os.Remove(filepath.Join(local, "raced.md")))
   606  	server.Write("raced.md", "v2 from server")
   607  
   608  	runSync(ctx, stores)
   609  	// The first pass only records the deletion; a corroborating pass
   610  	// after the window sends it.
   611  	require.Contains(t, server.Files, "drop.md")
   612  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
   613  	runSync(ctx, stores)
   614  
   615  	// Only the un-raced deletion went out, guarded by the last-synced content.
   616  	sent := []string{deletedReq("drop.md", "v1")}
   617  	require.Equal(t, sent, server.ReceivedSnapshot())
   618  	require.NotContains(t, server.Files, "drop.md")
   619  	// The raced deletion lost — the server's edit came back to disk instead.
   620  	require.Equal(t, "v2 from server", readLocal(t, local, "raced.md"))
   621  	// The never-downloaded file was pulled, not deleted.
   622  	require.Equal(t, "v1", readLocal(t, local, "never.md"))
   623  	require.Equal(t, "v1", server.Files["never.md"])
   624  
   625  	runSync(ctx, stores)
   626  	require.Equal(t, sent, server.ReceivedSnapshot())
   627  }
   628  
   629  func TestMemorySync_ReadOnlyStoresAreNeverWritten(t *testing.T) {
   630  	local, server, stores := downloadedStores(t,
   631  		map[string]string{"edit.md": "v1", "drop.md": "v1", "pull.md": "v1", "vanish.md": "v1"}, "read_only", nil, nil)
   632  	ctx := context.Background()
   633  
   634  	writeLocal(t, local, "edit.md", "v2 from agent")
   635  	require.NoError(t, os.Remove(filepath.Join(local, "drop.md")))
   636  	writeLocal(t, local, "new.md", "new from agent")
   637  	server.Write("pull.md", "v2 from server")
   638  	server.Delete("vanish.md")
   639  
   640  	runSync(ctx, stores)
   641  
   642  	require.Empty(t, server.ReceivedSnapshot())
   643  	require.Equal(t, map[string]string{"edit.md": "v1", "drop.md": "v1", "pull.md": "v2 from server"}, server.Files)
   644  	// Pulls still happen...
   645  	require.Equal(t, "v2 from server", readLocal(t, local, "pull.md"))
   646  	// ...including a removal, which is a pull and not a push: the server no
   647  	// longer holds vanish.md, so neither does the folder.
   648  	require.NoFileExists(t, filepath.Join(local, "vanish.md"))
   649  	// And the store's root is reported for the write/edit tool refusal, as
   650  	// well as for the file tools' reach.
   651  	require.Equal(t, []string{local}, stores.ReadOnlyRoots())
   652  	require.Equal(t, stores.ReadOnlyRoots(), stores.Roots())
   653  }
   654  
   655  func TestMemorySync_PathsThatEscapeTheStoreAreRefused(t *testing.T) {
   656  	var logBuf bytes.Buffer
   657  	local, server, stores := downloadedStores(t,
   658  		map[string]string{"../evil.md": "boom", "ok.md": "fine"}, "", &logBuf, nil)
   659  
   660  	runSync(context.Background(), stores)
   661  
   662  	require.Equal(t, "fine", readLocal(t, local, "ok.md"))
   663  	// Nothing named evil.md landed anywhere under the workdir.
   664  	workdir := filepath.Dir(filepath.Dir(local))
   665  	require.NoError(t, filepath.Walk(workdir, func(path string, _ os.FileInfo, err error) error {
   666  		require.NotEqual(t, "evil.md", filepath.Base(path))
   667  		return err
   668  	}))
   669  	require.Empty(t, server.ReceivedSnapshot())
   670  	require.Contains(t, logBuf.String(), "escapes")
   671  }
   672  
   673  func TestMemorySync_LifecycleCadenceAndDispose(t *testing.T) {
   674  	// The worker's call sequence: download → SyncIfDue per call → Finish →
   675  	// Dispose. A fake clock makes the cadence observable.
   676  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   677  	httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP))
   678  	t.Cleanup(httpSrv.Close)
   679  	client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0))
   680  
   681  	now := time.Unix(0, 0)
   682  	workdir := t.TempDir()
   683  	stores := mustStores(t, client, SessionMemoryStoresOptions{
   684  		Workdir:      workdir,
   685  		SyncInterval: 300 * time.Second,
   686  		Logger:       silentLogger,
   687  	})
   688  	stores.setClock(func() time.Time { return now })
   689  	ctx := context.Background()
   690  	require.NoError(t, stores.Download(ctx, fetchSession(t, client)))
   691  	local := filepath.Join(workdir, "memory", "notes")
   692  
   693  	writeLocal(t, local, "note.md", "edit 1")
   694  	stores.SyncIfDue(ctx)
   695  	require.Empty(t, server.ReceivedSnapshot())
   696  
   697  	// One long gap fires exactly once, and not again until another interval.
   698  	now = time.Unix(2000, 0)
   699  	stores.SyncIfDue(ctx)
   700  	stores.SyncIfDue(ctx)
   701  	require.Equal(t, []string{updated("note.md", "edit 1", "v1")}, server.ReceivedSnapshot())
   702  	now = time.Unix(2299, 0)
   703  	stores.SyncIfDue(ctx)
   704  	require.Len(t, server.ReceivedSnapshot(), 1)
   705  
   706  	// The last sync is Finish: unconditional, once.
   707  	writeLocal(t, local, "note.md", "edit 2")
   708  	stores.Finish(ctx)
   709  	received := server.ReceivedSnapshot()
   710  	require.Equal(t, updated("note.md", "edit 2", "edit 1"), received[len(received)-1])
   711  
   712  	// Dispose removes the store dir the download created.
   713  	require.DirExists(t, local)
   714  	stores.Dispose()
   715  	require.NoDirExists(t, local)
   716  }
   717  
   718  func TestMemorySync_TheSyncIntervalHasADefaultAndAFloor(t *testing.T) {
   719  	client := anthropic.NewClient(option.WithAPIKey("k"))
   720  	build := func(d time.Duration) (*SessionMemoryStores, error) {
   721  		return NewSessionMemoryStores(client, SessionMemoryStoresOptions{
   722  			Workdir:      t.TempDir(),
   723  			SyncInterval: d,
   724  			Logger:       silentLogger,
   725  		})
   726  	}
   727  	for _, tc := range []struct{ in, want time.Duration }{
   728  		{0, DefaultMemorySyncInterval},
   729  		{MinMemorySyncInterval, MinMemorySyncInterval},
   730  		{time.Minute, time.Minute},
   731  		{-1, -1}, // negative passes through; the worker gates on it
   732  	} {
   733  		stores, err := build(tc.in)
   734  		require.NoError(t, err, tc.in)
   735  		require.Equal(t, tc.want, stores.syncInterval, tc.in)
   736  	}
   737  	_, err := build(time.Second)
   738  	require.ErrorIs(t, err, ErrMemorySyncIntervalTooShort)
   739  }
   740  
   741  func TestMemorySync_ABinaryLocalFileNeverBlocksTheOtherFiles(t *testing.T) {
   742  	// The store is utf-8-restricted, so a binary file the agent drops in the
   743  	// folder is refused at read — warned once and skipped until it changes —
   744  	// while every other file keeps syncing.
   745  	var logBuf bytes.Buffer
   746  	local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil)
   747  	ctx := context.Background()
   748  
   749  	require.NoError(t, os.WriteFile(filepath.Join(local, "blob.bin"), []byte{0xff, 0xfe, 0x00, ' ', 'x'}, 0o600))
   750  	writeLocal(t, local, "note.md", "v2")
   751  
   752  	runSync(ctx, stores)
   753  	runSync(ctx, stores)
   754  
   755  	require.Equal(t, []string{updated("note.md", "v2", "v1")}, server.ReceivedSnapshot())
   756  	require.NotContains(t, server.Files, "blob.bin")
   757  	require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes"))
   758  	require.Contains(t, logBuf.String(), "path=blob.bin")
   759  }
   760  
   761  func TestMemorySync_AnOversizedFileIsSkippedUntilItShrinks(t *testing.T) {
   762  	// A file the server refuses as too large is warned once and skipped —
   763  	// not retried while unchanged, never blocking the other files — and
   764  	// syncs as soon as an edit brings it under the cap.
   765  	var logBuf bytes.Buffer
   766  	local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil)
   767  	ctx := context.Background()
   768  	server.MaxContentBytes = 64
   769  
   770  	writeLocal(t, local, "big.md", strings.Repeat("x", 100))
   771  	writeLocal(t, local, "note.md", "v2")
   772  
   773  	runSync(ctx, stores)
   774  	runSync(ctx, stores)
   775  
   776  	// The small edit landed; the big file was attempted exactly once.
   777  	received := server.ReceivedSnapshot()
   778  	require.Contains(t, received, updated("note.md", "v2", "v1"))
   779  	creates := 0
   780  	for _, r := range received {
   781  		if strings.HasPrefix(r, "create ") {
   782  			creates++
   783  		}
   784  	}
   785  	require.Equal(t, 1, creates)
   786  	require.NotContains(t, server.Files, "big.md")
   787  	require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes"))
   788  	require.Contains(t, logBuf.String(), "path=big.md")
   789  
   790  	writeLocal(t, local, "big.md", "small now")
   791  	runSync(ctx, stores)
   792  	require.Equal(t, "small now", server.Files["big.md"])
   793  }
   794  
   795  func TestMemorySync_ARefusalThatSaysNothingAboutTheContentIsRetriedNextSync(t *testing.T) {
   796  	// Anything but a content refusal is retried by the next sync.
   797  	for _, status := range []int{401, 403, 404} {
   798  		t.Run(fmt.Sprint(status), func(t *testing.T) {
   799  			var logBuf bytes.Buffer
   800  			local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil)
   801  			ctx := context.Background()
   802  			writeLocal(t, local, "note.md", "v2")
   803  			writeLocal(t, local, "new.md", "fresh")
   804  
   805  			server.FailUploads = status
   806  			runSync(ctx, stores)
   807  			server.FailUploads = 0
   808  			runSync(ctx, stores)
   809  
   810  			require.Equal(t, map[string]string{"note.md": "v2", "new.md": "fresh"}, server.Files)
   811  			require.NotContains(t, logBuf.String(), "stays un-synced")
   812  		})
   813  	}
   814  }
   815  
   816  func TestMemorySync_AnUploadThatLosesTheRacePullsTheWinnerNextSync(t *testing.T) {
   817  	// The server changes between this sync's list and its update: the
   818  	// content_sha256 precondition rejects the push (409), and the next sync
   819  	// applies the conflict rule — the winning remote edit is pulled, the
   820  	// local edit is never re-pushed.
   821  	var logBuf bytes.Buffer
   822  	local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", &logBuf, nil)
   823  	ctx := context.Background()
   824  
   825  	writeLocal(t, local, "note.md", "local edit")
   826  	server.OnUpdate = func() {
   827  		// Race: the remote changes after this sync's list, before its
   828  		// update. (Called with the server mutex held.)
   829  		server.Files["note.md"] = "raced remote edit"
   830  	}
   831  	runSync(ctx, stores)
   832  	server.OnUpdate = nil
   833  
   834  	// The precondition rejected the push (one guarded attempt, no retry) and
   835  	// the server kept its raced edit.
   836  	require.Equal(t, []string{updated("note.md", "local edit", "v1")}, server.ReceivedSnapshot())
   837  	require.Equal(t, "raced remote edit", server.Files["note.md"])
   838  	require.Contains(t, logBuf.String(), "changed both locally and remotely")
   839  	// The warning says what actually happened: the push was dropped and the
   840  	// local file is the stale one. Nothing "kept the remote version" here —
   841  	// disk still holds the losing edit until the next sync overwrites it.
   842  	require.Contains(t, logBuf.String(), "the upload was refused and the local edit loses")
   843  	require.Equal(t, "local edit", readLocal(t, local, "note.md"))
   844  
   845  	// The next sync pulls the winner instead of re-pushing.
   846  	runSync(ctx, stores)
   847  	require.Equal(t, "raced remote edit", readLocal(t, local, "note.md"))
   848  	require.Len(t, server.ReceivedSnapshot(), 1)
   849  }
   850  
   851  // storesAt wires a SessionMemoryStores to server over workdir without
   852  // downloading — for the tests that assert on Download's own refusals.
   853  func storesAt(t *testing.T, server *memoryServer, workdir string) *SessionMemoryStores {
   854  	t.Helper()
   855  	httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP))
   856  	t.Cleanup(httpSrv.Close)
   857  	client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0))
   858  	return mustStores(t, client, SessionMemoryStoresOptions{Workdir: workdir, Logger: silentLogger})
   859  }
   860  
   861  func TestMemorySync_ABrokenStoreFailsTheWholeDownload(t *testing.T) {
   862  	// The first attached store's listing explodes mid-download: its half-written
   863  	// folder self-destructs and the whole download fails. A session served
   864  	// without a folder its system prompt names would run with amnesia, so the
   865  	// worker turns this into a failed work item rather than a warning.
   866  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   867  	server.Broken = true
   868  	workdir := t.TempDir()
   869  	stores := storesAt(t, server, workdir)
   870  
   871  	err := stores.Download(context.Background(), fetchSession(t, stores.client))
   872  	var memErr *SessionMemoryError
   873  	require.ErrorAs(t, err, &memErr)
   874  	require.Equal(t, "memstore_broken", memErr.MemoryStoreID)
   875  
   876  	// The broken store got one file down before exploding; nothing survives.
   877  	// The healthy store behind it was never reached, and nothing is tracked.
   878  	require.NoDirExists(t, filepath.Join(workdir, "memory", "broken"))
   879  	require.NoDirExists(t, filepath.Join(workdir, "memory", "notes"))
   880  	require.Empty(t, stores.ReadOnlyRoots())
   881  	require.Empty(t, server.ReceivedSnapshot())
   882  }
   883  
   884  func TestMemorySync_ALegalMountPathIsHonored(t *testing.T) {
   885  	// mount_path is the location the agent's system prompt names. A clean
   886  	// absolute one is honored verbatim, and Dispose removes it when the
   887  	// download created it.
   888  	tmp := t.TempDir()
   889  	mount := filepath.Join(tmp, "mnt", "notes")
   890  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   891  	server.MountPath = mount
   892  	stores := storesAt(t, server, filepath.Join(tmp, "wd"))
   893  
   894  	require.NoError(t, stores.Download(context.Background(), fetchSession(t, stores.client)))
   895  	require.Equal(t, "v1", readLocal(t, mount, "note.md"))
   896  	require.Equal(t, []string{mount}, stores.Roots())
   897  	stores.Dispose()
   898  	require.NoDirExists(t, mount)
   899  }
   900  
   901  func TestMemorySync_AnUncleanMountPathFailsTheDownload(t *testing.T) {
   902  	// A mount_path we cannot use verbatim fails the session rather than quietly
   903  	// relocating the store: the agent would read an empty folder at the path it
   904  	// was told about, and write notes somewhere the next session never looks.
   905  	for _, tc := range []struct {
   906  		name  string
   907  		mount string
   908  		// "" means the resource carries no mount_path at all — that one is
   909  		// allowed, and falls back to the workdir, because nothing pointed the
   910  		// agent anywhere.
   911  		ok bool
   912  	}{
   913  		{"escapes with ..", "/mnt/../../etc/cron.d", false},
   914  		{"relative, with a separator", "relative/notes", false},
   915  		{"relative, a bare name", "notes", false},
   916  		{"no mount path at all", "", true},
   917  	} {
   918  		t.Run(tc.name, func(t *testing.T) {
   919  			tmp := t.TempDir()
   920  			workdir := filepath.Join(tmp, "wd")
   921  			server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   922  			server.MountPath = tc.mount
   923  			stores := storesAt(t, server, workdir)
   924  
   925  			err := stores.Download(context.Background(), fetchSession(t, stores.client))
   926  			if tc.ok {
   927  				require.NoError(t, err)
   928  				require.Equal(t, "v1", readLocal(t, filepath.Join(workdir, "memory", "notes"), "note.md"))
   929  				return
   930  			}
   931  
   932  			var memErr *SessionMemoryError
   933  			require.ErrorAs(t, err, &memErr)
   934  			require.Equal(t, "memstore_notes", memErr.MemoryStoreID)
   935  			require.ErrorContains(t, err, "not a clean absolute path")
   936  
   937  			// Nothing was written anywhere, nothing was sent, nothing is tracked.
   938  			require.NoDirExists(t, workdir)
   939  			require.Empty(t, server.ReceivedSnapshot())
   940  			require.Empty(t, stores.ReadOnlyRoots())
   941  		})
   942  	}
   943  }
   944  
   945  func TestMemorySync_AnUnwritableMountPathFailsTheDownload(t *testing.T) {
   946  	// A mount the host cannot provide is an environment failure: the download
   947  	// fails loudly (failing the work item) instead of degrading to a session
   948  	// with silently-empty memory.
   949  	if os.Geteuid() == 0 {
   950  		t.Skip("directory modes do not bind root")
   951  	}
   952  	tmp := t.TempDir()
   953  	parent := filepath.Join(tmp, "mnt")
   954  	require.NoError(t, os.Mkdir(parent, 0o700))
   955  	require.NoError(t, os.Chmod(parent, 0o500)) // the worker may not create the store's folder here
   956  	t.Cleanup(func() { _ = os.Chmod(parent, 0o700) })
   957  
   958  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   959  	server.MountPath = filepath.Join(parent, "notes")
   960  	stores := storesAt(t, server, filepath.Join(tmp, "wd"))
   961  
   962  	err := stores.Download(context.Background(), fetchSession(t, stores.client))
   963  	var memErr *SessionMemoryError
   964  	require.ErrorAs(t, err, &memErr)
   965  	require.Equal(t, "memstore_notes", memErr.MemoryStoreID)
   966  	require.ErrorContains(t, err, "writable")
   967  	require.Empty(t, server.ReceivedSnapshot())
   968  }
   969  
   970  func TestMemorySync_AStoreDirectoryThatAlreadyExistsIsRefused(t *testing.T) {
   971  	// Nothing but a dead run leaves a folder at the store's path. Syncing into
   972  	// it would fold that run's debris into the customer's store, so refuse —
   973  	// and leave the folder exactly as it was found.
   974  	workdir := t.TempDir()
   975  	stale := filepath.Join(workdir, "memory", "notes")
   976  	require.NoError(t, os.MkdirAll(stale, 0o755))
   977  	writeLocal(t, stale, "debris.md", "left by another session")
   978  
   979  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
   980  	stores := storesAt(t, server, workdir)
   981  
   982  	err := stores.Download(context.Background(), fetchSession(t, stores.client))
   983  	var memErr *SessionMemoryError
   984  	require.ErrorAs(t, err, &memErr)
   985  	require.Equal(t, "memstore_notes", memErr.MemoryStoreID)
   986  	require.ErrorContains(t, err, "already exists")
   987  
   988  	// Refused, not adopted and not deleted — not even by Dispose.
   989  	require.Equal(t, "left by another session", readLocal(t, stale, "debris.md"))
   990  	require.NoFileExists(t, filepath.Join(stale, "note.md"))
   991  	require.Empty(t, server.ReceivedSnapshot())
   992  	require.Empty(t, stores.ReadOnlyRoots())
   993  	stores.Dispose()
   994  	require.DirExists(t, stale)
   995  
   996  	// A sibling directory in the same parent is none of our business.
   997  	other := t.TempDir()
   998  	require.NoError(t, os.MkdirAll(filepath.Join(other, "memory", "unrelated"), 0o755))
   999  	server2 := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
  1000  	stores2 := storesAt(t, server2, other)
  1001  
  1002  	require.NoError(t, stores2.Download(context.Background(), fetchSession(t, stores2.client)))
  1003  	require.Equal(t, "v1", readLocal(t, filepath.Join(other, "memory", "notes"), "note.md"))
  1004  	require.DirExists(t, filepath.Join(other, "memory", "unrelated"))
  1005  }
  1006  
  1007  func TestMemorySync_PrefixItemsAndForeignResourcesLeaveNoTrace(t *testing.T) {
  1008  	// Only memory_store resources are downloaded and only "memory" items
  1009  	// carry content — a "file" resource and memory_prefix rollups pass
  1010  	// through without touching the disk or the server.
  1011  	server := newMemoryServer(t, map[string]string{"ok.md": "fine"}, "")
  1012  	server.Noise = true
  1013  	workdir := t.TempDir()
  1014  	stores := storesAt(t, server, workdir)
  1015  	ctx := context.Background()
  1016  	require.NoError(t, stores.Download(ctx, fetchSession(t, stores.client)))
  1017  	local := filepath.Join(workdir, "memory", "notes")
  1018  
  1019  	require.Equal(t, "fine", readLocal(t, local, "ok.md"))
  1020  	entries, err := os.ReadDir(filepath.Join(workdir, "memory"))
  1021  	require.NoError(t, err)
  1022  	require.Len(t, entries, 1)
  1023  	require.Equal(t, "notes", entries[0].Name())
  1024  	require.NoDirExists(t, filepath.Join(local, "projects"))
  1025  
  1026  	runSync(ctx, stores)
  1027  	require.Empty(t, server.ReceivedSnapshot())
  1028  }
  1029  
  1030  // ---- the sync matrix -------------------------------------------------------
  1031  
  1032  // Matrix cells: keep means the side did nothing, gone means it deleted the
  1033  // path. Any other value is the content that side holds now. The sentinels
  1034  // cannot collide with real content.
  1035  const (
  1036  	keep = "\x00keep"
  1037  	gone = "\x00gone"
  1038  )
  1039  
  1040  func TestMemorySync_TheSyncMatrix(t *testing.T) {
  1041  	// Every combination of (local moved?, remote moved?) for a path the store
  1042  	// downloaded at "v1". local and remote say what happened since; disk and
  1043  	// store are the state the two sides must agree on afterwards, and sent is
  1044  	// exactly what went over the wire.
  1045  	for _, tc := range []struct {
  1046  		name          string
  1047  		local, remote string
  1048  		disk, store   string
  1049  		sent          []string
  1050  	}{
  1051  		{"untouched", keep, keep, "v1", "v1", nil},
  1052  		{"local edit", "v2", keep, "v2", "v2", []string{updated("f.md", "v2", "v1")}},
  1053  		{"remote edit", keep, "v2", "v2", "v2", nil},
  1054  		{"both edit, the server wins", "v2 local", "v2 remote", "v2 remote", "v2 remote", nil},
  1055  		{"local delete", gone, keep, gone, gone, []string{deletedReq("f.md", "v1")}},
  1056  		{"remote delete", keep, gone, gone, gone, nil},
  1057  		{"both delete", gone, gone, gone, gone, nil},
  1058  		{"local delete, remote edit", gone, "v2", "v2", "v2", nil},
  1059  		{"local edit, remote delete", "v2", gone, "v2", "v2", []string{created("f.md", "v2")}},
  1060  	} {
  1061  		t.Run(tc.name, func(t *testing.T) {
  1062  			local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, nil)
  1063  			clock := withClock(stores)
  1064  			ctx := context.Background()
  1065  
  1066  			switch tc.local {
  1067  			case keep:
  1068  			case gone:
  1069  				require.NoError(t, os.Remove(filepath.Join(local, "f.md")))
  1070  			default:
  1071  				writeLocal(t, local, "f.md", tc.local)
  1072  			}
  1073  			switch tc.remote {
  1074  			case keep:
  1075  			case gone:
  1076  				server.Delete("f.md")
  1077  			default:
  1078  				server.Write("f.md", tc.remote)
  1079  			}
  1080  
  1081  			settled := func() {
  1082  				t.Helper()
  1083  				if tc.disk == gone {
  1084  					require.NoFileExists(t, filepath.Join(local, "f.md"))
  1085  				} else {
  1086  					require.Equal(t, tc.disk, readLocal(t, local, "f.md"))
  1087  				}
  1088  				if tc.store == gone {
  1089  					require.NotContains(t, server.Files, "f.md")
  1090  				} else {
  1091  					require.Equal(t, tc.store, server.Files["f.md"])
  1092  				}
  1093  				require.Equal(t, tc.sent, server.ReceivedSnapshot())
  1094  			}
  1095  
  1096  			// Two passes: a local deletion is only sent after its
  1097  			// corroboration window.
  1098  			runSync(ctx, stores)
  1099  			clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1100  			runSync(ctx, stores)
  1101  			settled()
  1102  
  1103  			// Settled: a second pass sends nothing more and changes nothing.
  1104  			runSync(ctx, stores)
  1105  			settled()
  1106  		})
  1107  	}
  1108  }
  1109  
  1110  // ---- the failure column ----------------------------------------------------
  1111  //
  1112  // The same paths, one operation broken. A failed operation must leave disk and
  1113  // server in a state the next sync can still reconcile — never one where the
  1114  // retry does the opposite of what was asked.
  1115  
  1116  func TestMemorySync_AFailedUploadKeepsTheLocalEditAndRetries(t *testing.T) {
  1117  	local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, nil)
  1118  	ctx := context.Background()
  1119  	writeLocal(t, local, "f.md", "v2 from agent")
  1120  
  1121  	server.FailUpdates = true
  1122  	runSync(ctx, stores)
  1123  
  1124  	// The agent's edit survives; the server is untouched.
  1125  	require.Equal(t, "v2 from agent", readLocal(t, local, "f.md"))
  1126  	require.Equal(t, "v1", server.Files["f.md"])
  1127  
  1128  	// The baseline still says v1, so the next (working) sync retries the push
  1129  	// with the same precondition.
  1130  	server.FailUpdates = false
  1131  	runSync(ctx, stores)
  1132  	require.Equal(t, "v2 from agent", server.Files["f.md"])
  1133  	push := updated("f.md", "v2 from agent", "v1")
  1134  	require.Equal(t, []string{push, push}, server.ReceivedSnapshot())
  1135  }
  1136  
  1137  func TestMemorySync_AFailedPullLeavesTheOldFileAndRetries(t *testing.T) {
  1138  	disk := &faults{}
  1139  	local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", nil, disk)
  1140  	ctx := context.Background()
  1141  	server.Write("f.md", "v2 from server")
  1142  
  1143  	disk.put = always
  1144  	runSync(ctx, stores)
  1145  	disk.put = nil
  1146  
  1147  	// The stale file is still there and the baseline never advanced...
  1148  	require.Equal(t, "v1", readLocal(t, local, "f.md"))
  1149  	require.Empty(t, server.ReceivedSnapshot())
  1150  
  1151  	// ...so the next sync pulls it, rather than mistaking v1 for a local edit
  1152  	// and pushing it back over the server's v2.
  1153  	runSync(ctx, stores)
  1154  	require.Equal(t, "v2 from server", readLocal(t, local, "f.md"))
  1155  	require.Empty(t, server.ReceivedSnapshot())
  1156  }
  1157  
  1158  func TestMemorySync_AFailedLocalRemovalNeverReUploadsTheDeletedMemory(t *testing.T) {
  1159  	// The server deleted the memory; removing our copy fails. The baseline must
  1160  	// stay put — dropping it would read the leftover file as a new local file
  1161  	// and create the memory the server just deleted.
  1162  	var logBuf bytes.Buffer
  1163  	disk := &faults{}
  1164  	local, server, stores := downloadedStores(t, map[string]string{"f.md": "v1"}, "", &logBuf, disk)
  1165  	ctx := context.Background()
  1166  	server.Delete("f.md")
  1167  
  1168  	disk.remove = always
  1169  	runSync(ctx, stores)
  1170  	disk.remove = nil
  1171  
  1172  	require.Equal(t, "v1", readLocal(t, local, "f.md"))
  1173  	require.Empty(t, server.ReceivedSnapshot())
  1174  	require.NotContains(t, server.Files, "f.md")
  1175  	require.Contains(t, logBuf.String(), "failed to remove memory deleted remotely")
  1176  
  1177  	// The retry deletes it locally; it is never pushed back.
  1178  	runSync(ctx, stores)
  1179  	require.NoFileExists(t, filepath.Join(local, "f.md"))
  1180  	require.Empty(t, server.ReceivedSnapshot())
  1181  	require.NotContains(t, server.Files, "f.md")
  1182  }
  1183  
  1184  // twoStoreSession builds a session attaching two memory stores at the given
  1185  // mount paths, the way the control plane would send it.
  1186  func twoStoreSession(t *testing.T, aRoot, bRoot string) *anthropic.BetaManagedAgentsSession {
  1187  	t.Helper()
  1188  	body := fmt.Sprintf(`{"resources":[
  1189  		{"type":"memory_store","memory_store_id":"store_a","mount_path":%q,"name":"a"},
  1190  		{"type":"memory_store","memory_store_id":"store_b","mount_path":%q,"name":"b"}
  1191  	]}`, aRoot, bRoot)
  1192  	var session anthropic.BetaManagedAgentsSession
  1193  	require.NoError(t, json.Unmarshal([]byte(body), &session))
  1194  	return &session
  1195  }
  1196  
  1197  func TestMemorySync_TwoStoresDownloadToTheirOwnFolders(t *testing.T) {
  1198  	wd := t.TempDir()
  1199  	server := newMemoryServer(t, map[string]string{"note.md": "v1"}, "")
  1200  	httpSrv := httptest.NewServer(http.HandlerFunc(server.serveHTTP))
  1201  	t.Cleanup(httpSrv.Close)
  1202  	client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0))
  1203  	stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: wd, Logger: silentLogger})
  1204  
  1205  	a, b := filepath.Join(wd, "m", "a"), filepath.Join(wd, "m", "b")
  1206  	require.NoError(t, stores.Download(context.Background(), twoStoreSession(t, a, b)))
  1207  	require.Equal(t, "v1", readLocal(t, a, "note.md"))
  1208  	require.Equal(t, "v1", readLocal(t, b, "note.md"))
  1209  }
  1210  
  1211  func TestMemorySync_StoresSyncInParallelAndOneFailureSparesTheRest(t *testing.T) {
  1212  	// Store A's listing waits for store B's listing to start — only a
  1213  	// parallel sync satisfies that — then fails. A serial sync times the
  1214  	// wait out instead (a different error message), and store B's edit must
  1215  	// land either way.
  1216  	wd := t.TempDir()
  1217  	slow := newMemoryServer(t, map[string]string{"a.md": "v1"}, "")
  1218  	fast := newMemoryServer(t, map[string]string{"b.md": "v1"}, "")
  1219  	var syncing atomic.Bool
  1220  	fastStarted := make(chan struct{})
  1221  	var once sync.Once
  1222  	fail := func(w http.ResponseWriter, msg string) {
  1223  		w.Header().Set("Content-Type", "application/json")
  1224  		w.WriteHeader(http.StatusInternalServerError)
  1225  		_, _ = fmt.Fprintf(w, `{"error":{"type":"api_error","message":%q}}`, msg)
  1226  	}
  1227  	httpSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1228  		isList := r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/memories")
  1229  		if strings.Contains(r.URL.Path, "store_a") {
  1230  			if syncing.Load() && isList {
  1231  				select {
  1232  				case <-fastStarted:
  1233  					fail(w, "slow store exploded")
  1234  				case <-time.After(10 * time.Second):
  1235  					fail(w, "fast store never started")
  1236  				}
  1237  				return
  1238  			}
  1239  			slow.serveHTTP(w, r)
  1240  			return
  1241  		}
  1242  		if syncing.Load() && isList {
  1243  			once.Do(func() { close(fastStarted) })
  1244  		}
  1245  		fast.serveHTTP(w, r)
  1246  	}))
  1247  	t.Cleanup(httpSrv.Close)
  1248  	client := anthropic.NewClient(option.WithBaseURL(httpSrv.URL), option.WithAPIKey("k"), option.WithMaxRetries(0))
  1249  
  1250  	var logBuf bytes.Buffer
  1251  	logger := slog.New(slog.NewTextHandler(&logBuf, nil))
  1252  	stores := mustStores(t, client, SessionMemoryStoresOptions{Workdir: wd, Logger: logger})
  1253  	aRoot, bRoot := filepath.Join(wd, "m", "a"), filepath.Join(wd, "m", "b")
  1254  	ctx := context.Background()
  1255  	require.NoError(t, stores.Download(ctx, twoStoreSession(t, aRoot, bRoot)))
  1256  
  1257  	syncing.Store(true)
  1258  	writeLocal(t, bRoot, "b.md", "v2 from agent")
  1259  	runSync(ctx, stores)
  1260  
  1261  	// Store A failed *after* seeing store B's listing start — proof the two
  1262  	// ran concurrently — and store B still pushed.
  1263  	require.Contains(t, logBuf.String(), "slow store exploded")
  1264  	require.Equal(t, []string{updated("b.md", "v2 from agent", "v1")}, fast.ReceivedSnapshot())
  1265  	// The failed store's baseline was kept, so nothing is re-uploaded once it heals.
  1266  	require.Equal(t, "v1", readLocal(t, aRoot, "a.md"))
  1267  }
  1268  
  1269  // ---- folder-trust gating ---------------------------------------------------
  1270  //
  1271  // A missing, emptied, or swapped store folder is never trusted as a basis
  1272  // for deletions or uploads.
  1273  //
  1274  // Download stamps MarkerPath (holding the store's id) into the folder. A
  1275  // sync pass that finds the folder destroyed re-downloads it and pushes
  1276  // nothing; a sync that finds files without a matching marker leaves the
  1277  // folder as found and syncs nothing at all.
  1278  
  1279  const (
  1280  	folderTrustStoreID = "memstore_notes"
  1281  	folderTrustMarker  = "version 1\n" + folderTrustStoreID
  1282  )
  1283  
  1284  func TestFolderTrust_MarkerBytesArePinned(t *testing.T) {
  1285  	// A format change breaks CI here rather than as a marker mismatch at runtime.
  1286  	// printf 'version 1\nmemstore_test' | sha256sum
  1287  	require.Equal(t, "b8c0fbafe3e9ea979f0fbe5d984ba8834e7a2771d859462b3f7aa556fdb55a67",
  1288  		markerSHA("memstore_test"))
  1289  }
  1290  
  1291  func TestFolderTrust_ADeletedFolderIssuesNoDeletesAndIsRebuilt(t *testing.T) {
  1292  	// rm -rf of the whole folder between syncs: the next sync must not read
  1293  	// the emptiness as "the agent deleted everything" and wipe the server.
  1294  	var logBuf bytes.Buffer
  1295  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil)
  1296  	clock := withClock(stores)
  1297  	ctx := context.Background()
  1298  	require.NoError(t, os.RemoveAll(local))
  1299  
  1300  	runSync(ctx, stores)
  1301  
  1302  	require.Empty(t, server.ReceivedSnapshot())
  1303  	require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files)
  1304  	require.Equal(t, "v1", readLocal(t, local, "a.md"))
  1305  	require.Equal(t, "v1", readLocal(t, local, "b.md"))
  1306  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1307  	require.Contains(t, logBuf.String(), "the folder or its marker is gone; re-downloading")
  1308  
  1309  	// The rebuilt folder syncs normally: an edit and a deletion both
  1310  	// propagate (the deletion after its waiting window).
  1311  	writeLocal(t, local, "a.md", "v2")
  1312  	require.NoError(t, os.Remove(filepath.Join(local, "b.md")))
  1313  	runSync(ctx, stores)
  1314  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1315  	runSync(ctx, stores)
  1316  	require.Equal(t, []string{updated("a.md", "v2", "v1"), deletedReq("b.md", "v1")}, server.ReceivedSnapshot())
  1317  	require.Equal(t, map[string]string{"a.md": "v2"}, server.Files)
  1318  }
  1319  
  1320  func TestFolderTrust_AnEmptiedFolderIssuesNoDeletesAndIsRepopulated(t *testing.T) {
  1321  	var logBuf bytes.Buffer
  1322  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil)
  1323  	clock := withClock(stores)
  1324  	ctx := context.Background()
  1325  	entries, _ := os.ReadDir(local)
  1326  	for _, e := range entries {
  1327  		require.NoError(t, os.Remove(filepath.Join(local, e.Name())))
  1328  	}
  1329  
  1330  	runSync(ctx, stores)
  1331  
  1332  	require.Empty(t, server.ReceivedSnapshot())
  1333  	require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files)
  1334  	require.Equal(t, "v1", readLocal(t, local, "a.md"))
  1335  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1336  	require.Contains(t, logBuf.String(), "the folder or its marker is gone; re-downloading")
  1337  
  1338  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1339  	runSync(ctx, stores)
  1340  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1341  	runSync(ctx, stores)
  1342  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1343  }
  1344  
  1345  func TestFolderTrust_AnEmptiedFolderWithTheMarkerLeftBehindIssuesNoDeletes(t *testing.T) {
  1346  	// rm <mount>/*: shell globs skip dotfiles, so the marker survives the
  1347  	// wipe — an intact marker alone must not clear a folder whose every
  1348  	// memory file vanished at once.
  1349  	var logBuf bytes.Buffer
  1350  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil)
  1351  	clock := withClock(stores)
  1352  	ctx := context.Background()
  1353  	entries, _ := os.ReadDir(local)
  1354  	for _, e := range entries {
  1355  		if e.Name() != MarkerPath {
  1356  			require.NoError(t, os.Remove(filepath.Join(local, e.Name())))
  1357  		}
  1358  	}
  1359  
  1360  	runSync(ctx, stores)
  1361  
  1362  	require.Empty(t, server.ReceivedSnapshot())
  1363  	require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files)
  1364  	require.Equal(t, "v1", readLocal(t, local, "a.md"))
  1365  	require.Contains(t, logBuf.String(), "every memory file is gone at once; re-downloading")
  1366  
  1367  	// With files present again, an ordinary single deletion still
  1368  	// propagates (after its waiting window).
  1369  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1370  	runSync(ctx, stores)
  1371  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1372  	runSync(ctx, stores)
  1373  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1374  }
  1375  
  1376  func TestFolderTrust_ASwappedFolderNeitherDeletesNorUploadsEver(t *testing.T) {
  1377  	// A folder carrying another store's marker is foreign: the pass must not
  1378  	// adopt it — no deletes, no uploads, no restamp — on this pass or any
  1379  	// later one, or the foreign files leak into the customer's store one
  1380  	// pass later.
  1381  	var logBuf bytes.Buffer
  1382  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil)
  1383  	ctx := context.Background()
  1384  	require.NoError(t, os.RemoveAll(local))
  1385  	require.NoError(t, os.MkdirAll(local, 0o755))
  1386  	writeLocal(t, local, MarkerPath, "memstore_other")
  1387  	writeLocal(t, local, "foreign.md", "someone else's notes")
  1388  
  1389  	runSync(ctx, stores)
  1390  	runSync(ctx, stores)
  1391  
  1392  	require.Empty(t, server.ReceivedSnapshot())
  1393  	require.Equal(t, map[string]string{"a.md": "v1"}, server.Files)
  1394  	require.Equal(t, "memstore_other", readLocal(t, local, MarkerPath))
  1395  	require.Equal(t, "someone else's notes", readLocal(t, local, "foreign.md"))
  1396  	require.NoFileExists(t, filepath.Join(local, "a.md"))
  1397  	require.Contains(t, logBuf.String(), "the marker file does not match this store; leaving the memory store folder as found")
  1398  }
  1399  
  1400  func TestFolderTrust_FilesWithoutAMarkerAreNeitherPushedNorPulledOver(t *testing.T) {
  1401  	// Only the marker was deleted (a dotfile cleanup): the remaining files
  1402  	// may hold un-pushed edits, so the pass must neither upload them, nor
  1403  	// delete by them, nor destroy them by re-downloading over the folder.
  1404  	var logBuf bytes.Buffer
  1405  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil)
  1406  	ctx := context.Background()
  1407  	require.NoError(t, os.Remove(filepath.Join(local, MarkerPath)))
  1408  	writeLocal(t, local, "a.md", "un-pushed edit")
  1409  	require.NoError(t, os.Remove(filepath.Join(local, "b.md")))
  1410  
  1411  	runSync(ctx, stores)
  1412  
  1413  	require.Empty(t, server.ReceivedSnapshot())
  1414  	require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files)
  1415  	require.Equal(t, "un-pushed edit", readLocal(t, local, "a.md"))
  1416  	require.NoFileExists(t, filepath.Join(local, MarkerPath))
  1417  	require.Contains(t, logBuf.String(), "the marker file is gone; leaving the memory store folder as found")
  1418  }
  1419  
  1420  func TestFolderTrust_ASingleLocalDeletionWithTheMarkerIntactStillPropagates(t *testing.T) {
  1421  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", nil, nil)
  1422  	clock := withClock(stores)
  1423  	ctx := context.Background()
  1424  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1425  
  1426  	runSync(ctx, stores)
  1427  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1428  	runSync(ctx, stores)
  1429  
  1430  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1431  	require.Equal(t, map[string]string{"b.md": "v1"}, server.Files)
  1432  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1433  }
  1434  
  1435  func TestFolderTrust_TheMarkerNeverSyncsInEitherDirection(t *testing.T) {
  1436  	var logBuf bytes.Buffer
  1437  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil)
  1438  	ctx := context.Background()
  1439  
  1440  	// The marker sits in the folder yet is never uploaded or remote-deleted.
  1441  	runSync(ctx, stores)
  1442  	require.Empty(t, server.ReceivedSnapshot())
  1443  
  1444  	// A marker path the server lists is refused: skipped with a warning,
  1445  	// never written over the real marker on disk.
  1446  	server.Write(MarkerPath, "not a store id")
  1447  	runSync(ctx, stores)
  1448  	require.Empty(t, server.ReceivedSnapshot())
  1449  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1450  	require.Contains(t, logBuf.String(), "reserved marker path")
  1451  }
  1452  
  1453  func TestFolderTrust_DisposeToleratesAnAlreadyDeletedFolder(t *testing.T) {
  1454  	local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil)
  1455  	require.NoError(t, os.RemoveAll(local))
  1456  
  1457  	stores.Dispose()
  1458  
  1459  	require.NoDirExists(t, local)
  1460  }
  1461  
  1462  func TestFolderTrust_DisposeKeepsAFolderThatFailsTheMarkerCheck(t *testing.T) {
  1463  	// Sync left the folder as found after a failed marker check; teardown
  1464  	// must not then delete the files sync refused to touch.
  1465  	var logBuf bytes.Buffer
  1466  	local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil)
  1467  	require.NoError(t, os.Remove(filepath.Join(local, MarkerPath)))
  1468  	writeLocal(t, local, "a.md", "un-pushed edit")
  1469  
  1470  	stores.Dispose()
  1471  
  1472  	require.Equal(t, "un-pushed edit", readLocal(t, local, "a.md"))
  1473  	require.Contains(t, logBuf.String(), "leaving the memory store folder on disk")
  1474  }
  1475  
  1476  func TestFolderTrust_DisposeNeverPanicsWhenAStoreRootIsReplacedByAFile(t *testing.T) {
  1477  	// The worker calls Dispose from its teardown: a store whose folder
  1478  	// something replaced with a regular file must log and be skipped, not
  1479  	// panic out of the teardown.
  1480  	local, _, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil)
  1481  	require.NoError(t, os.RemoveAll(local))
  1482  	require.NoError(t, os.WriteFile(local, []byte("a file where the folder was"), 0o600))
  1483  
  1484  	stores.Dispose()
  1485  
  1486  	got, err := os.ReadFile(local)
  1487  	require.NoError(t, err)
  1488  	require.Equal(t, "a file where the folder was", string(got))
  1489  }
  1490  
  1491  func TestFolderTrust_AMarkerFromAnOlderFormatNoLongerClearsTheFolder(t *testing.T) {
  1492  	// The marker carries a format version so an SDK upgrade can retire every
  1493  	// folder written before it: a marker in the old format (the bare store
  1494  	// id) must fail the check, and the folder is left as found.
  1495  	var logBuf bytes.Buffer
  1496  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "b.md": "v1"}, "", &logBuf, nil)
  1497  	ctx := context.Background()
  1498  	writeLocal(t, local, MarkerPath, folderTrustStoreID)
  1499  	writeLocal(t, local, "a.md", "edit under the old marker")
  1500  	require.NoError(t, os.Remove(filepath.Join(local, "b.md")))
  1501  
  1502  	runSync(ctx, stores)
  1503  
  1504  	require.Empty(t, server.ReceivedSnapshot())
  1505  	require.Equal(t, map[string]string{"a.md": "v1", "b.md": "v1"}, server.Files)
  1506  	require.Equal(t, "edit under the old marker", readLocal(t, local, "a.md"))
  1507  	require.Contains(t, logBuf.String(), "the marker file does not match this store")
  1508  }
  1509  
  1510  func TestFolderTrust_AFolderWipedWhileAPullIsInFlightIsRebuiltNotDistrusted(t *testing.T) {
  1511  	// rm -rf lands between a sync's scan and its content write: the write
  1512  	// must not bring the folder back without its marker, or every later sync
  1513  	// reads the folder as foreign and the session's memory writes are lost.
  1514  	// A flat path fails at the temp-file open, a nested one at the mkdir.
  1515  	for _, tc := range []struct{ name, rel string }{{"flat", "z.md"}, {"nested", "sub/z.md"}} {
  1516  		t.Run(tc.name, func(t *testing.T) {
  1517  			var logBuf bytes.Buffer
  1518  			local, server, stores := downloadedStores(t,
  1519  				map[string]string{"x.md": "v1", "y.md": "v1", tc.rel: "v1"}, "", &logBuf, nil)
  1520  			ctx := context.Background()
  1521  			server.Write(tc.rel, "v2")
  1522  			server.SetOnFetch(func(string) int {
  1523  				_ = os.RemoveAll(local)
  1524  				return 0
  1525  			})
  1526  
  1527  			runSync(ctx, stores)
  1528  
  1529  			require.NoDirExists(t, local)
  1530  			require.Empty(t, server.ReceivedSnapshot())
  1531  			require.Contains(t, logBuf.String(), "failed to write memory")
  1532  
  1533  			server.SetOnFetch(nil)
  1534  			runSync(ctx, stores)
  1535  
  1536  			require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1537  			require.Equal(t, "v1", readLocal(t, local, "x.md"))
  1538  			require.Equal(t, "v2", readLocal(t, local, tc.rel))
  1539  			require.Contains(t, logBuf.String(), "re-downloading the memory store folder")
  1540  			require.NotContains(t, logBuf.String(), "leaving the memory store folder as found")
  1541  
  1542  			stores.Finish(ctx)
  1543  			stores.FlushWrites(ctx)
  1544  			stores.Dispose()
  1545  			require.NoDirExists(t, local)
  1546  			require.Empty(t, server.ReceivedSnapshot())
  1547  			require.Equal(t, map[string]string{"x.md": "v1", "y.md": "v1", tc.rel: "v2"}, server.Files)
  1548  		})
  1549  	}
  1550  }
  1551  
  1552  func TestFolderTrust_AFolderWipedWhileALocallyDeletedMemoryIsBeingRestoredIsRebuilt(t *testing.T) {
  1553  	// A file deleted locally but changed on the server is restored by the
  1554  	// next sync; an rm -rf that lands while that restore is downloading must
  1555  	// leave no folder behind, the sync after it must rebuild the whole store,
  1556  	// and the earlier absence must never become a server delete.
  1557  	var logBuf bytes.Buffer
  1558  	local, server, stores := downloadedStores(t,
  1559  		map[string]string{"x.md": "v1", "y.md": "v1", "z.md": "v1"}, "", &logBuf, nil)
  1560  	clock := withClock(stores)
  1561  	ctx := context.Background()
  1562  	require.NoError(t, os.Remove(filepath.Join(local, "z.md")))
  1563  	server.Write("z.md", "v2")
  1564  	server.SetOnFetch(func(string) int {
  1565  		_ = os.RemoveAll(local)
  1566  		return 0
  1567  	})
  1568  
  1569  	runSync(ctx, stores)
  1570  	require.NoDirExists(t, local)
  1571  
  1572  	server.SetOnFetch(nil)
  1573  	runSync(ctx, stores)
  1574  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1575  	require.Equal(t, "v1", readLocal(t, local, "x.md"))
  1576  	require.Equal(t, "v1", readLocal(t, local, "y.md"))
  1577  	require.Equal(t, "v2", readLocal(t, local, "z.md"))
  1578  
  1579  	clock.now = clock.now.Add(DeleteCorroborationWindow + time.Second)
  1580  	runSync(ctx, stores)
  1581  	require.Empty(t, server.ReceivedSnapshot())
  1582  	require.Equal(t, map[string]string{"x.md": "v1", "y.md": "v1", "z.md": "v2"}, server.Files)
  1583  	require.NotContains(t, logBuf.String(), "leaving the memory store folder as found")
  1584  }
  1585  
  1586  func TestFolderTrust_AFolderWipedAgainMidRecoveryIsRebuiltByTheNextSync(t *testing.T) {
  1587  	// A re-download writes the marker first and the memories after it; a
  1588  	// second rm -rf between those writes must not leave memories on disk
  1589  	// without a marker — the next sync re-downloads once more instead.
  1590  	var logBuf bytes.Buffer
  1591  	disk := &faults{}
  1592  	local, server, stores := downloadedStores(t,
  1593  		map[string]string{"a.md": "v1", "b.md": "v1", "c.md": "v1"}, "", &logBuf, disk)
  1594  	ctx := context.Background()
  1595  	require.NoError(t, os.RemoveAll(local))
  1596  
  1597  	memoryPuts := 0
  1598  	disk.onPut = func(rel string) {
  1599  		if rel == MarkerPath {
  1600  			return
  1601  		}
  1602  		memoryPuts++
  1603  		if memoryPuts == 2 { // the marker and one memory are back on disk
  1604  			_ = os.RemoveAll(local)
  1605  		}
  1606  	}
  1607  	runSync(ctx, stores)
  1608  	disk.onPut = nil
  1609  
  1610  	require.NoDirExists(t, local)
  1611  	require.Empty(t, server.ReceivedSnapshot())
  1612  
  1613  	runSync(ctx, stores)
  1614  
  1615  	require.Equal(t, folderTrustMarker, readLocal(t, local, MarkerPath))
  1616  	for _, name := range []string{"a.md", "b.md", "c.md"} {
  1617  		require.Equal(t, "v1", readLocal(t, local, name))
  1618  	}
  1619  	require.Equal(t, 2, strings.Count(logBuf.String(), "re-downloading the memory store folder"))
  1620  	require.NotContains(t, logBuf.String(), "leaving the memory store folder as found")
  1621  	require.Empty(t, server.ReceivedSnapshot())
  1622  
  1623  	stores.Dispose()
  1624  	require.NoDirExists(t, local)
  1625  }
  1626  
  1627  // ---- server-delete discipline ----------------------------------------------
  1628  //
  1629  // A locally deleted file is deleted on the server only after a second sync
  1630  // confirms it, at a bounded rate per sync, and only when server deletes
  1631  // are enabled at all.
  1632  //
  1633  // The first sync that sees the file missing records it; a later sync —
  1634  // after DeleteCorroborationWindow (skipped on the session's final Finish, the
  1635  // session's last sync) and a re-check that the file is still gone and the
  1636  // marker still intact — sends the DELETE. Each sync sends a bounded
  1637  // number of deletes; SyncDeletions = MemorySyncDeletionsLogOnly only logs them,
  1638  // and MemorySyncDeletionsDisabled sends none, ever.
  1639  
  1640  var deleteElapsed = DeleteCorroborationWindow + time.Second
  1641  
  1642  // downloadedMode is downloadedStores with a SyncDeletions mode and a fake
  1643  // clock at time zero.
  1644  func downloadedMode(t *testing.T, initial map[string]string, mode MemoryDeleteMode, logBuf *bytes.Buffer) (string, *memoryServer, *SessionMemoryStores, *testClock) {
  1645  	t.Helper()
  1646  	local, server, stores := downloadedStores(t, initial, "", logBuf, nil)
  1647  	stores.syncDeletions = mode
  1648  	clock := withClock(stores)
  1649  	return local, server, stores, clock
  1650  }
  1651  
  1652  func TestDeleteDiscipline_ALocalDeletionIsConfirmedBeforeItReachesTheServer(t *testing.T) {
  1653  	local, server, stores, clock := downloadedMode(t,
  1654  		map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsEnabled, nil)
  1655  	ctx := context.Background()
  1656  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1657  
  1658  	// The first pass only records the absence.
  1659  	runSync(ctx, stores)
  1660  	require.Empty(t, server.ReceivedSnapshot())
  1661  	require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files)
  1662  
  1663  	// Still inside the window: nothing goes out.
  1664  	clock.now = clock.now.Add(DeleteCorroborationWindow - time.Second)
  1665  	runSync(ctx, stores)
  1666  	require.Empty(t, server.ReceivedSnapshot())
  1667  
  1668  	// Window elapsed and the file is still gone: the guarded delete is sent.
  1669  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1670  	runSync(ctx, stores)
  1671  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1672  	require.Equal(t, map[string]string{"keep.md": "v1"}, server.Files)
  1673  }
  1674  
  1675  func TestDeleteDiscipline_AFileThatReappearsIsNotDeletedAndLeavesPending(t *testing.T) {
  1676  	local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1"}, MemorySyncDeletionsEnabled, nil)
  1677  	ctx := context.Background()
  1678  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1679  	runSync(ctx, stores)
  1680  
  1681  	writeLocal(t, local, "a.md", "v1")
  1682  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1683  	runSync(ctx, stores)
  1684  	require.Empty(t, server.ReceivedSnapshot())
  1685  	require.Equal(t, map[string]string{"a.md": "v1"}, server.Files)
  1686  
  1687  	// The reappearance cleared the pending entry: deleting again starts a
  1688  	// fresh window instead of firing on the stale observation.
  1689  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1690  	runSync(ctx, stores)
  1691  	require.Empty(t, server.ReceivedSnapshot())
  1692  	clock.now = time.Unix(0, 0).Add(2 * deleteElapsed)
  1693  	runSync(ctx, stores)
  1694  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1695  }
  1696  
  1697  func TestDeleteDiscipline_DisabledModeNeverDeletesButStillSyncs(t *testing.T) {
  1698  	var logBuf bytes.Buffer
  1699  	local, server, stores, clock := downloadedMode(t,
  1700  		map[string]string{"gone.md": "v1", "push.md": "v1", "pull.md": "v1"}, MemorySyncDeletionsDisabled, &logBuf)
  1701  	ctx := context.Background()
  1702  	require.NoError(t, os.Remove(filepath.Join(local, "gone.md")))
  1703  	writeLocal(t, local, "push.md", "v2 from agent")
  1704  	server.Write("pull.md", "v2 from server")
  1705  
  1706  	runSync(ctx, stores)
  1707  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1708  	runSync(ctx, stores)
  1709  	clock.now = time.Unix(0, 0).Add(2 * deleteElapsed)
  1710  	runSync(ctx, stores)
  1711  
  1712  	// Uploads and pulls still flow; the deletion never does.
  1713  	require.Equal(t, []string{updated("push.md", "v2 from agent", "v1")}, server.ReceivedSnapshot())
  1714  	require.Equal(t, map[string]string{"gone.md": "v1", "push.md": "v2 from agent", "pull.md": "v2 from server"}, server.Files)
  1715  	require.Equal(t, "v2 from server", readLocal(t, local, "pull.md"))
  1716  	require.Contains(t, logBuf.String(), "server deletes are disabled")
  1717  }
  1718  
  1719  func TestDeleteDiscipline_ThePerPassCapSpreadsDeletesOverPasses(t *testing.T) {
  1720  	var logBuf bytes.Buffer
  1721  	files := map[string]string{}
  1722  	for i := 0; i < 14; i++ {
  1723  		files[fmt.Sprintf("f%02d.md", i)] = "v1"
  1724  	}
  1725  	local, server, stores, clock := downloadedMode(t, files, MemorySyncDeletionsEnabled, &logBuf)
  1726  	ctx := context.Background()
  1727  	var doomed []string
  1728  	for k := range files {
  1729  		doomed = append(doomed, k)
  1730  	}
  1731  	slices.Sort(doomed)
  1732  	doomed = doomed[:12]
  1733  	for _, name := range doomed {
  1734  		require.NoError(t, os.Remove(filepath.Join(local, name)))
  1735  	}
  1736  
  1737  	runSync(ctx, stores)
  1738  	require.Empty(t, server.ReceivedSnapshot())
  1739  
  1740  	// cap = max(8, min(50, 14/4)) = 8: the first confirming sync stops there.
  1741  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1742  	runSync(ctx, stores)
  1743  	var want []string
  1744  	for _, name := range doomed[:8] {
  1745  		want = append(want, deletedReq(name, "v1"))
  1746  	}
  1747  	require.Equal(t, want, server.ReceivedSnapshot())
  1748  	require.Contains(t, logBuf.String(), "delete cap reached: sent 8 deletes, held 4 for later syncs")
  1749  
  1750  	// The held-back deletions confirm on the next pass.
  1751  	runSync(ctx, stores)
  1752  	for _, name := range doomed[8:] {
  1753  		want = append(want, deletedReq(name, "v1"))
  1754  	}
  1755  	require.Equal(t, want, server.ReceivedSnapshot())
  1756  	remaining := map[string]string{}
  1757  	for i := 12; i < 14; i++ {
  1758  		remaining[fmt.Sprintf("f%02d.md", i)] = "v1"
  1759  	}
  1760  	require.Equal(t, remaining, server.Files)
  1761  }
  1762  
  1763  func TestDeleteDiscipline_TheFinalSyncWaivesTheWindowButStillReChecks(t *testing.T) {
  1764  	// A deletion made in a session shorter than the waiting window still
  1765  	// goes through: the final sync skips the wait, and the re-check just
  1766  	// before the delete stands in for the second observation.
  1767  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, "", nil, nil)
  1768  	ctx := context.Background()
  1769  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1770  
  1771  	stores.Finish(ctx)
  1772  
  1773  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1774  	require.Equal(t, map[string]string{"keep.md": "v1"}, server.Files)
  1775  }
  1776  
  1777  func TestDeleteDiscipline_AFolderDestroyedMidSyncSendsNoDeletes(t *testing.T) {
  1778  	// The directory scan saw an intact folder, then the folder vanished
  1779  	// while the server listing was in flight: the re-check just before
  1780  	// the delete must notice the marker is gone and hold every delete.
  1781  	local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsEnabled, nil)
  1782  	ctx := context.Background()
  1783  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1784  	runSync(ctx, stores)
  1785  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1786  
  1787  	server.SetOnList(func() { _ = os.RemoveAll(local) })
  1788  	runSync(ctx, stores)
  1789  	server.SetOnList(nil)
  1790  
  1791  	require.Empty(t, server.ReceivedSnapshot())
  1792  	require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files)
  1793  }
  1794  
  1795  func TestDeleteDiscipline_AnUnreadableSubdirectoryFailsThePassInsteadOfReadingAsDeletions(t *testing.T) {
  1796  	// Files under a directory the scan cannot enter are not treated as deleted.
  1797  	if os.Geteuid() == 0 {
  1798  		t.Skip("permission checks do not bind root")
  1799  	}
  1800  	var logBuf bytes.Buffer
  1801  	local, server, stores, clock := downloadedMode(t,
  1802  		map[string]string{"sub/a.md": "v1", "sub/b.md": "v1", "top.md": "v1"}, MemorySyncDeletionsEnabled, &logBuf)
  1803  	ctx := context.Background()
  1804  	writeLocal(t, local, "top.md", "v2")
  1805  	sub := filepath.Join(local, "sub")
  1806  	require.NoError(t, os.Chmod(sub, 0o000))
  1807  	t.Cleanup(func() { _ = os.Chmod(sub, 0o700) })
  1808  
  1809  	runSync(ctx, stores)
  1810  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1811  	runSync(ctx, stores)
  1812  	require.Empty(t, server.ReceivedSnapshot())
  1813  	require.Equal(t, 2, strings.Count(logBuf.String(), "memory sync failed"))
  1814  
  1815  	// Readable again: the edit goes out and nothing is deleted.
  1816  	require.NoError(t, os.Chmod(sub, 0o700))
  1817  	clock.now = time.Unix(0, 0).Add(2 * deleteElapsed)
  1818  	runSync(ctx, stores)
  1819  	require.Equal(t, []string{updated("top.md", "v2", "v1")}, server.ReceivedSnapshot())
  1820  	require.Equal(t, map[string]string{"sub/a.md": "v1", "sub/b.md": "v1", "top.md": "v2"}, server.Files)
  1821  }
  1822  
  1823  func TestDeleteDiscipline_ARecoveryReDownloadClearsPendingDeletes(t *testing.T) {
  1824  	// The re-download rebuilt the disk, so an absence observed before it
  1825  	// says nothing about the rebuilt folder.
  1826  	local, server, stores, clock := downloadedMode(t, map[string]string{"a.md": "v1"}, MemorySyncDeletionsEnabled, nil)
  1827  	ctx := context.Background()
  1828  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1829  	runSync(ctx, stores)
  1830  
  1831  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1832  	require.NoError(t, os.RemoveAll(local))
  1833  	runSync(ctx, stores)
  1834  	require.Equal(t, "v1", readLocal(t, local, "a.md"))
  1835  
  1836  	// A fresh deletion starts a fresh window — the pre-recovery
  1837  	// observation must not corroborate it.
  1838  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1839  	runSync(ctx, stores)
  1840  	require.Empty(t, server.ReceivedSnapshot())
  1841  	clock.now = time.Unix(0, 0).Add(2 * deleteElapsed)
  1842  	runSync(ctx, stores)
  1843  	require.Equal(t, []string{deletedReq("a.md", "v1")}, server.ReceivedSnapshot())
  1844  }
  1845  
  1846  func TestDeleteDiscipline_LogOnlyModeLogsTheDeleteButNeverSendsIt(t *testing.T) {
  1847  	// The dry-run setting: the full pipeline runs — window, re-checks,
  1848  	// cap — but the confirmed delete is logged instead of sent, on every
  1849  	// sync while the file stays missing.
  1850  	var logBuf bytes.Buffer
  1851  	local, server, stores, clock := downloadedMode(t,
  1852  		map[string]string{"a.md": "v1", "keep.md": "v1"}, MemorySyncDeletionsLogOnly, &logBuf)
  1853  	ctx := context.Background()
  1854  	require.NoError(t, os.Remove(filepath.Join(local, "a.md")))
  1855  
  1856  	runSync(ctx, stores)
  1857  	require.NotContains(t, logBuf.String(), "log-only") // still inside the waiting window
  1858  
  1859  	clock.now = time.Unix(0, 0).Add(deleteElapsed)
  1860  	runSync(ctx, stores)
  1861  	runSync(ctx, stores)
  1862  
  1863  	require.Empty(t, server.ReceivedSnapshot())
  1864  	require.Equal(t, map[string]string{"a.md": "v1", "keep.md": "v1"}, server.Files)
  1865  	require.Equal(t, 2, strings.Count(logBuf.String(), "log-only: sync would delete this memory on the server"))
  1866  	require.Contains(t, logBuf.String(), "path=a.md")
  1867  }
  1868  
  1869  func TestDeleteDiscipline_FinishRunsOnceAndASecondCallPanics(t *testing.T) {
  1870  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", nil, nil)
  1871  	ctx := context.Background()
  1872  	writeLocal(t, local, "a.md", "v2")
  1873  
  1874  	stores.Finish(ctx)
  1875  	require.Equal(t, map[string]string{"a.md": "v2"}, server.Files)
  1876  
  1877  	require.PanicsWithValue(t,
  1878  		"Finish() was already called: it is the session's last sync and runs once",
  1879  		func() { stores.Finish(ctx) })
  1880  }
  1881  
  1882  // ---- never destroy the only copy -------------------------------------------
  1883  //
  1884  // A memory deleted remotely while its file holds an un-pushed edit keeps
  1885  // the file — a writable store re-creates the memory from it, a read-only
  1886  // store keeps it on disk unsynced. Only an unedited file follows the
  1887  // remote delete off the disk.
  1888  
  1889  func TestKeepOnlyCopy_AnEditedFileSurvivesARemoteDeleteAndIsReCreated(t *testing.T) {
  1890  	var logBuf bytes.Buffer
  1891  	local, server, stores := downloadedStores(t,
  1892  		map[string]string{"edited.md": "v1", "plain.md": "v1"}, "", &logBuf, nil)
  1893  	ctx := context.Background()
  1894  	writeLocal(t, local, "edited.md", "v2 from agent")
  1895  	server.Delete("edited.md")
  1896  	server.Delete("plain.md")
  1897  
  1898  	runSync(ctx, stores)
  1899  
  1900  	// The edit's only copy survives and the memory is re-created from it;
  1901  	// the unedited file follows the remote delete off the disk.
  1902  	require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md"))
  1903  	require.Equal(t, map[string]string{"edited.md": "v2 from agent"}, server.Files)
  1904  	require.Equal(t, []string{created("edited.md", "v2 from agent")}, server.ReceivedSnapshot())
  1905  	require.NoFileExists(t, filepath.Join(local, "plain.md"))
  1906  	require.Contains(t, logBuf.String(), "re-creating it from the file")
  1907  
  1908  	// Settled: nothing more goes out.
  1909  	runSync(ctx, stores)
  1910  	require.Equal(t, []string{created("edited.md", "v2 from agent")}, server.ReceivedSnapshot())
  1911  
  1912  	// The upload's sha became the baseline: the next edit is a guarded update.
  1913  	writeLocal(t, local, "edited.md", "v3")
  1914  	runSync(ctx, stores)
  1915  	got := server.ReceivedSnapshot()
  1916  	require.Equal(t, updated("edited.md", "v3", "v2 from agent"), got[len(got)-1])
  1917  }
  1918  
  1919  func TestKeepOnlyCopy_AReadOnlyStoreKeepsTheEditedFileWithoutPushing(t *testing.T) {
  1920  	var logBuf bytes.Buffer
  1921  	local, server, stores := downloadedStores(t,
  1922  		map[string]string{"edited.md": "v1", "plain.md": "v1"}, "read_only", &logBuf, nil)
  1923  	ctx := context.Background()
  1924  	writeLocal(t, local, "edited.md", "v2 from agent")
  1925  	server.Delete("edited.md")
  1926  	server.Delete("plain.md")
  1927  
  1928  	runSync(ctx, stores)
  1929  
  1930  	require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md"))
  1931  	require.NoFileExists(t, filepath.Join(local, "plain.md"))
  1932  	require.Empty(t, server.ReceivedSnapshot())
  1933  	require.Equal(t, 1, strings.Count(logBuf.String(), "cannot push"))
  1934  
  1935  	// Later passes see a local-only file in a read-only store: a quiet no-op.
  1936  	runSync(ctx, stores)
  1937  	runSync(ctx, stores)
  1938  	require.Empty(t, server.ReceivedSnapshot())
  1939  	require.Equal(t, "v2 from agent", readLocal(t, local, "edited.md"))
  1940  	require.Equal(t, 1, strings.Count(logBuf.String(), "cannot push"))
  1941  }
  1942  
  1943  func TestKeepOnlyCopy_ARefusedEditSurvivesARemoteDeleteWithoutARetry(t *testing.T) {
  1944  	// The server already refused this content, so the remote delete
  1945  	// neither removes the file nor triggers a re-create the server would
  1946  	// refuse again.
  1947  	local, server, stores := downloadedStores(t, map[string]string{"big.md": "v1"}, "", nil, nil)
  1948  	ctx := context.Background()
  1949  	server.MaxContentBytes = 64
  1950  
  1951  	writeLocal(t, local, "big.md", strings.Repeat("x", 100))
  1952  	runSync(ctx, stores)
  1953  	require.Len(t, server.ReceivedSnapshot(), 1) // the one refused update
  1954  
  1955  	server.Delete("big.md")
  1956  	runSync(ctx, stores)
  1957  	runSync(ctx, stores)
  1958  
  1959  	require.Equal(t, strings.Repeat("x", 100), readLocal(t, local, "big.md"))
  1960  	require.Len(t, server.ReceivedSnapshot(), 1)
  1961  	require.NotContains(t, server.Files, "big.md")
  1962  }
  1963  
  1964  func TestKeepOnlyCopy_AMemoryDeletedBetweenTheListingAndTheUpdateIsReCreated(t *testing.T) {
  1965  	// An update that meets a 404 re-creates the memory in the same pass.
  1966  	local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", nil, nil)
  1967  	ctx := context.Background()
  1968  	writeLocal(t, local, "note.md", "v2 from agent")
  1969  
  1970  	server.SetUploadHook(func(_ context.Context, path string) {
  1971  		server.Delete(path)
  1972  		server.SetUploadHook(nil)
  1973  	})
  1974  	runSync(ctx, stores)
  1975  
  1976  	require.Equal(t, map[string]string{"note.md": "v2 from agent"}, server.Files)
  1977  	require.Equal(t, []string{updated("note.md", "v2 from agent", "v1"), created("note.md", "v2 from agent")}, server.ReceivedSnapshot())
  1978  
  1979  	// The create's sha is the new baseline.
  1980  	writeLocal(t, local, "note.md", "v3")
  1981  	runSync(ctx, stores)
  1982  	received := server.ReceivedSnapshot()
  1983  	require.Equal(t, updated("note.md", "v3", "v2 from agent"), received[len(received)-1])
  1984  }
  1985  
  1986  func TestKeepOnlyCopy_AnEditWrittenMidSyncStillSurvivesARemoteDelete(t *testing.T) {
  1987  	// The directory scan saw the file unedited; an edit is written while
  1988  	// the server listing is in flight. The removal must re-read the file
  1989  	// instead of destroying the edit based on the stale scan.
  1990  	local, server, stores := downloadedStores(t, map[string]string{"z.md": "v1", "keep.md": "v1"}, "", nil, nil)
  1991  	ctx := context.Background()
  1992  	server.Delete("z.md")
  1993  
  1994  	server.SetOnList(func() { require.NoError(t, os.WriteFile(filepath.Join(local, "z.md"), []byte("late edit"), 0o600)) })
  1995  	runSync(ctx, stores)
  1996  	server.SetOnList(nil)
  1997  
  1998  	require.Equal(t, "late edit", readLocal(t, local, "z.md"))
  1999  	require.Equal(t, "late edit", server.Files["z.md"])
  2000  }
  2001  
  2002  func TestKeepOnlyCopy_AReadOnlyPullOverALocalEditIsWarned(t *testing.T) {
  2003  	// Read-only stores always take the server version, but overwriting a
  2004  	// local edit must leave a trace instead of happening silently.
  2005  	var logBuf bytes.Buffer
  2006  	local, server, stores := downloadedStores(t, map[string]string{"pull.md": "v1"}, "read_only", &logBuf, nil)
  2007  	ctx := context.Background()
  2008  	writeLocal(t, local, "pull.md", "local edit")
  2009  	server.Write("pull.md", "v2 from server")
  2010  
  2011  	runSync(ctx, stores)
  2012  
  2013  	require.Equal(t, "v2 from server", readLocal(t, local, "pull.md"))
  2014  	require.Empty(t, server.ReceivedSnapshot())
  2015  	require.Contains(t, logBuf.String(), "changed both locally and remotely")
  2016  }
  2017  
  2018  func TestKeepOnlyCopy_AFailedReReadDoesNotResurrectAServerDeletedMemory(t *testing.T) {
  2019  	// The re-read before removal can fail (a transient I/O error). That
  2020  	// must mean "retry next sync", not "file gone": dropping the entry
  2021  	// would make the next sync upload the file as new, re-creating the
  2022  	// memory the server just deleted.
  2023  	disk := &faults{}
  2024  	local, server, stores := downloadedStores(t, map[string]string{"z.md": "v1", "keep.md": "v1"}, "", nil, disk)
  2025  	ctx := context.Background()
  2026  	server.Delete("z.md")
  2027  
  2028  	disk.hashFile = func(rel string) bool { return rel == "z.md" }
  2029  	runSync(ctx, stores)
  2030  	disk.hashFile = nil
  2031  
  2032  	// The error held everything: file still on disk, nothing pushed.
  2033  	require.Equal(t, "v1", readLocal(t, local, "z.md"))
  2034  	require.Empty(t, server.ReceivedSnapshot())
  2035  
  2036  	// With the error gone, the next sync completes the server delete.
  2037  	runSync(ctx, stores)
  2038  	require.NoFileExists(t, filepath.Join(local, "z.md"))
  2039  	require.NotContains(t, server.Files, "z.md")
  2040  	require.Empty(t, server.ReceivedSnapshot())
  2041  }
  2042  
  2043  // ---- FlushWrites — the push-only shutdown pass -----------------------------
  2044  //
  2045  // A session that ends on an error or cancel gets no full reconcile, but
  2046  // the writes already on disk are the only copy of the agent's edits.
  2047  // FlushWrites pushes those and does nothing else: no remote deletes, no
  2048  // local removals, no pulls.
  2049  
  2050  func TestFinalFlush_PushesNewAndChangedFilesAndNothingElse(t *testing.T) {
  2051  	var logBuf bytes.Buffer
  2052  	local, server, stores := downloadedStores(t, map[string]string{
  2053  		"a-pull.md":               "v1",
  2054  		"b-push.md":               "v1",
  2055  		"d-local-gone.md":         "v1",
  2056  		"e-remote-gone.md":        "v1",
  2057  		"f-remote-gone-edited.md": "v1",
  2058  		"g-both.md":               "v1",
  2059  	}, "", &logBuf, nil)
  2060  	ctx := context.Background()
  2061  
  2062  	writeLocal(t, local, "b-push.md", "v2 from agent")
  2063  	writeLocal(t, local, "c-new.md", "new from agent")
  2064  	require.NoError(t, os.Remove(filepath.Join(local, "d-local-gone.md")))
  2065  	writeLocal(t, local, "f-remote-gone-edited.md", "v2 from agent")
  2066  	writeLocal(t, local, "g-both.md", "v2 from agent")
  2067  	server.Write("a-pull.md", "v2 from server")
  2068  	server.Delete("e-remote-gone.md")
  2069  	server.Delete("f-remote-gone-edited.md")
  2070  	server.Write("g-both.md", "v2 from server")
  2071  
  2072  	stores.FlushWrites(ctx)
  2073  
  2074  	// Only additions went out: the update, the new file, and the
  2075  	// re-created edited file whose remote copy was deleted.
  2076  	sent := []string{
  2077  		updated("b-push.md", "v2 from agent", "v1"),
  2078  		created("c-new.md", "new from agent"),
  2079  		created("f-remote-gone-edited.md", "v2 from agent"),
  2080  	}
  2081  	require.ElementsMatch(t, sent, server.ReceivedSnapshot())
  2082  	// A locally missing file was not deleted remotely.
  2083  	require.Equal(t, "v1", server.Files["d-local-gone.md"])
  2084  	// A remotely deleted, locally unedited memory was not re-uploaded —
  2085  	// and its file was not removed from disk either.
  2086  	require.NotContains(t, server.Files, "e-remote-gone.md")
  2087  	require.Equal(t, "v1", readLocal(t, local, "e-remote-gone.md"))
  2088  	// Remote changes were not pulled.
  2089  	require.Equal(t, "v1", readLocal(t, local, "a-pull.md"))
  2090  	// The conflict was left to the server, and logged.
  2091  	require.Equal(t, "v2 from server", server.Files["g-both.md"])
  2092  	require.Equal(t, "v2 from agent", readLocal(t, local, "g-both.md"))
  2093  	require.Contains(t, logBuf.String(), "changed both locally and remotely")
  2094  
  2095  	// Settled: a second flush pushes nothing more.
  2096  	stores.FlushWrites(ctx)
  2097  	require.ElementsMatch(t, sent, server.ReceivedSnapshot())
  2098  }
  2099  
  2100  func TestFinalFlush_UploadsSeveralFilesAtOnceUpToTheBound(t *testing.T) {
  2101  	// The bound's worth of uploads are in flight together, and never one
  2102  	// more — a serial flush would time out waiting for the second to start.
  2103  	local, server, stores := downloadedStores(t, map[string]string{}, "", nil, nil)
  2104  	want := map[string]string{}
  2105  	for i := range uploadConcurrency + 3 {
  2106  		name, content := fmt.Sprintf("f%02d.md", i), fmt.Sprintf("file %d", i)
  2107  		writeLocal(t, local, name, content)
  2108  		want[name] = content
  2109  	}
  2110  
  2111  	var mu sync.Mutex
  2112  	inFlight, peak := 0, 0
  2113  	release := make(chan struct{})
  2114  	var once sync.Once
  2115  	server.SetUploadHook(func(ctx context.Context, _ string) {
  2116  		mu.Lock()
  2117  		inFlight++
  2118  		peak = max(peak, inFlight)
  2119  		full := inFlight == uploadConcurrency
  2120  		mu.Unlock()
  2121  		if full {
  2122  			once.Do(func() {
  2123  				// Hold the full set briefly so a flush that ignored the bound
  2124  				// has time to start one more before any of these finish.
  2125  				time.Sleep(100 * time.Millisecond)
  2126  				close(release)
  2127  			})
  2128  		}
  2129  		select {
  2130  		case <-release:
  2131  		case <-ctx.Done():
  2132  		case <-time.After(10 * time.Second):
  2133  			t.Error("the uploads never filled the bound: they ran serially")
  2134  			once.Do(func() { close(release) })
  2135  		}
  2136  		mu.Lock()
  2137  		inFlight--
  2138  		mu.Unlock()
  2139  	})
  2140  
  2141  	stores.FlushWrites(context.Background())
  2142  
  2143  	mu.Lock()
  2144  	got := peak
  2145  	mu.Unlock()
  2146  	require.Equal(t, uploadConcurrency, got)
  2147  	require.Equal(t, want, server.Files)
  2148  }
  2149  
  2150  func TestFinalFlush_AFlushCutOffPartWayLogsHowManyFilesItHadNotUploaded(t *testing.T) {
  2151  	var logBuf bytes.Buffer
  2152  	local, server, stores := downloadedStores(t, map[string]string{"z.md": "z0"}, "", &logBuf, nil)
  2153  	names := []string{"a.md", "b.md", "c.md", "y.md", "z.md"}
  2154  	for _, name := range names {
  2155  		writeLocal(t, local, name, "content of "+name)
  2156  	}
  2157  
  2158  	// Two uploads — one create, one update — never complete; the flush's
  2159  	// own deadline cuts them off.
  2160  	server.SetUploadHook(func(ctx context.Context, path string) {
  2161  		if path == "y.md" || path == "z.md" {
  2162  			<-ctx.Done()
  2163  		}
  2164  	})
  2165  	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
  2166  	defer cancel()
  2167  	stores.FlushWrites(ctx)
  2168  
  2169  	require.Equal(t, 1, strings.Count(logBuf.String(), "memory flush cut off part-way"))
  2170  	require.Contains(t, logBuf.String(),
  2171  		`2 of 5 changed files had not finished uploading" memory_store_id=memstore_notes`)
  2172  	require.Equal(t, map[string]string{
  2173  		"a.md": "content of a.md", "b.md": "content of b.md", "c.md": "content of c.md", "z.md": "z0",
  2174  	}, server.Files)
  2175  	require.NotContains(t, logBuf.String(), "failed to upload memory")
  2176  	require.NotContains(t, logBuf.String(), "memory flush failed")
  2177  
  2178  	// The three that made it are not sent again; the two that did not are.
  2179  	// A flush that finishes in time adds no cut-off line.
  2180  	server.SetUploadHook(nil)
  2181  	stores.FlushWrites(context.Background())
  2182  	want := map[string]string{}
  2183  	for _, name := range names {
  2184  		want[name] = "content of " + name
  2185  	}
  2186  	require.Equal(t, want, server.Files)
  2187  	require.ElementsMatch(t, []string{
  2188  		created("a.md", "content of a.md"),
  2189  		created("b.md", "content of b.md"),
  2190  		created("c.md", "content of c.md"),
  2191  		created("y.md", "content of y.md"),
  2192  		updated("z.md", "content of z.md", "z0"),
  2193  	}, server.ReceivedSnapshot())
  2194  	require.Equal(t, 1, strings.Count(logBuf.String(), "cut off part-way"))
  2195  }
  2196  
  2197  func TestFinalFlush_AdoptsAFileTheServerAlreadyHolds(t *testing.T) {
  2198  	// A final sync cut off after uploading a file leaves the baseline stale;
  2199  	// the flush sees the server already holds those bytes and records that
  2200  	// instead of reporting a conflict.
  2201  	var logBuf bytes.Buffer
  2202  	local, server, stores := downloadedStores(t, map[string]string{"n.md": "v1"}, "", &logBuf, nil)
  2203  	ctx := context.Background()
  2204  	writeLocal(t, local, "n.md", "v2")
  2205  	server.Write("n.md", "v2")
  2206  
  2207  	stores.FlushWrites(ctx)
  2208  
  2209  	require.Empty(t, server.ReceivedSnapshot())
  2210  	require.NotContains(t, logBuf.String(), "changed both locally and remotely")
  2211  
  2212  	// The adopted sha became the baseline: the next edit is a guarded update.
  2213  	writeLocal(t, local, "n.md", "v3")
  2214  	stores.FlushWrites(ctx)
  2215  	require.Equal(t, []string{updated("n.md", "v3", "v2")}, server.ReceivedSnapshot())
  2216  }
  2217  
  2218  func TestFinalFlush_ARefusedFileDoesNotHoldBackTheOthers(t *testing.T) {
  2219  	// A refusal and a success are book-kept from two uploads of one flush:
  2220  	// the refused file is tried once and not retried, the other lands once.
  2221  	var logBuf bytes.Buffer
  2222  	local, server, stores := downloadedStores(t, map[string]string{}, "", &logBuf, nil)
  2223  	ctx := context.Background()
  2224  	server.MaxContentBytes = 64
  2225  	writeLocal(t, local, "big.md", strings.Repeat("x", 100))
  2226  	writeLocal(t, local, "ok.md", "fits")
  2227  
  2228  	stores.FlushWrites(ctx)
  2229  	stores.FlushWrites(ctx)
  2230  
  2231  	require.Equal(t, "fits", server.Files["ok.md"])
  2232  	require.NotContains(t, server.Files, "big.md")
  2233  	require.ElementsMatch(t,
  2234  		[]string{created("big.md", strings.Repeat("x", 100)), created("ok.md", "fits")},
  2235  		server.ReceivedSnapshot())
  2236  	require.Equal(t, 1, strings.Count(logBuf.String(), "stays un-synced until its content changes"))
  2237  }
  2238  
  2239  func TestFinalFlush_NeverPushesAReadOnlyStore(t *testing.T) {
  2240  	local, server, stores := downloadedStores(t, map[string]string{"edit.md": "v1"}, "read_only", nil, nil)
  2241  	ctx := context.Background()
  2242  
  2243  	writeLocal(t, local, "edit.md", "v2 from agent")
  2244  	writeLocal(t, local, "new.md", "new from agent")
  2245  
  2246  	stores.FlushWrites(ctx)
  2247  
  2248  	require.Empty(t, server.ReceivedSnapshot())
  2249  	require.Equal(t, map[string]string{"edit.md": "v1"}, server.Files)
  2250  }
  2251  
  2252  func TestFinalFlush_SkipsFilesTheServerAlreadyRefused(t *testing.T) {
  2253  	local, server, stores := downloadedStores(t, map[string]string{"note.md": "v1"}, "", nil, nil)
  2254  	ctx := context.Background()
  2255  	server.MaxContentBytes = 64
  2256  
  2257  	writeLocal(t, local, "big.md", strings.Repeat("x", 100))
  2258  	runSync(ctx, stores)
  2259  	creates := 0
  2260  	for _, r := range server.ReceivedSnapshot() {
  2261  		if strings.HasPrefix(r, "create ") {
  2262  			creates++
  2263  		}
  2264  	}
  2265  	require.Equal(t, 1, creates)
  2266  
  2267  	stores.FlushWrites(ctx)
  2268  
  2269  	// The refused file was not retried; nothing else needed pushing.
  2270  	creates = 0
  2271  	for _, r := range server.ReceivedSnapshot() {
  2272  		if strings.HasPrefix(r, "create ") {
  2273  			creates++
  2274  		}
  2275  	}
  2276  	require.Equal(t, 1, creates)
  2277  	require.NotContains(t, server.Files, "big.md")
  2278  }
  2279  
  2280  func TestFinalFlush_PushesNothingWhenTheMarkerCheckFails(t *testing.T) {
  2281  	var logBuf bytes.Buffer
  2282  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil)
  2283  	ctx := context.Background()
  2284  
  2285  	require.NoError(t, os.Remove(filepath.Join(local, MarkerPath)))
  2286  	writeLocal(t, local, "foreign.md", "someone else's notes")
  2287  
  2288  	stores.FlushWrites(ctx)
  2289  
  2290  	require.Empty(t, server.ReceivedSnapshot())
  2291  	require.NotContains(t, server.Files, "foreign.md")
  2292  	// Unlike sync's recovery, the flush does not re-download the folder.
  2293  	require.NoFileExists(t, filepath.Join(local, MarkerPath))
  2294  	require.Contains(t, logBuf.String(), "not uploading anything from the memory store folder")
  2295  }
  2296  
  2297  func TestFinalFlush_NeverFails(t *testing.T) {
  2298  	var logBuf bytes.Buffer
  2299  	local, server, stores := downloadedStores(t, map[string]string{"a.md": "v1"}, "", &logBuf, nil)
  2300  	ctx := context.Background()
  2301  	writeLocal(t, local, "a.md", "v2 from agent")
  2302  
  2303  	server.SetOnList(func() { panic(http.ErrAbortHandler) }) // the listing explodes
  2304  	stores.FlushWrites(ctx)
  2305  	server.SetOnList(nil)
  2306  
  2307  	require.Empty(t, server.ReceivedSnapshot())
  2308  	require.Contains(t, logBuf.String(), "memory flush failed")
  2309  }