github.com/anthropics/anthropic-sdk-go@v1.70.1/lib/environments/poller_test.go (about)

     1  package environments
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/json"
     7  	"errors"
     8  	"io"
     9  	"log/slog"
    10  	"net/http"
    11  	"net/http/httptest"
    12  	"slices"
    13  	"strings"
    14  	"sync"
    15  	"sync/atomic"
    16  	"testing"
    17  	"time"
    18  
    19  	"github.com/anthropics/anthropic-sdk-go"
    20  	"github.com/anthropics/anthropic-sdk-go/option"
    21  	"github.com/anthropics/anthropic-sdk-go/packages/param"
    22  	"github.com/stretchr/testify/require"
    23  )
    24  
    25  // ---------------------------------------------------------------------------
    26  // Unit tests for the small helpers in poller.go. These don't need a server.
    27  // ---------------------------------------------------------------------------
    28  
    29  func TestBackoff(t *testing.T) {
    30  	tests := []struct {
    31  		description string
    32  		failures    int
    33  		want        time.Duration
    34  	}{
    35  		{"first failure backs off 2s (2^1) so a transient blip recovers quickly", 1, 2 * time.Second},
    36  		{"second consecutive failure doubles to 4s following the exponential curve", 2, 4 * time.Second},
    37  		{"fifth failure reaches 32s while still under the cap", 5, 32 * time.Second},
    38  		{"sixth failure would compute 64s but is clamped to the 60s cap", 6, 60 * time.Second},
    39  		{"very large failure counts stay clamped at the cap rather than overflowing", 30, 60 * time.Second},
    40  	}
    41  	for _, tc := range tests {
    42  		t.Run(tc.description, func(t *testing.T) {
    43  			require.Equal(t, tc.want, backoff(tc.failures))
    44  		})
    45  	}
    46  }
    47  
    48  func TestJitter(t *testing.T) {
    49  	low, high := 1*time.Second, 3*time.Second
    50  	for range 100 {
    51  		got := jitter(low, high)
    52  		require.GreaterOrEqual(t, got, low, "jitter must never undershoot lower bound")
    53  		require.Less(t, got, high, "jitter must stay strictly below upper bound")
    54  	}
    55  }
    56  
    57  func TestIsStatus(t *testing.T) {
    58  	tests := []struct {
    59  		description string
    60  		err         error
    61  		code        int
    62  		want        bool
    63  	}{
    64  		{"matching status code on a wrapped api error is detected so 409 on Stop can be suppressed",
    65  			&anthropic.Error{StatusCode: 409}, 409, true},
    66  		{"different status code returns false so only the intended code is matched",
    67  			&anthropic.Error{StatusCode: 500}, 409, false},
    68  		{"non-api error type is never matched even if the message looks similar",
    69  			errors.New("409 conflict"), 409, false},
    70  		{"nil error is treated as no-match rather than panicking",
    71  			nil, 409, false},
    72  	}
    73  	for _, tc := range tests {
    74  		t.Run(tc.description, func(t *testing.T) {
    75  			require.Equal(t, tc.want, isStatus(tc.err, tc.code))
    76  		})
    77  	}
    78  }
    79  
    80  func TestIsFatal4xx(t *testing.T) {
    81  	tests := []struct {
    82  		description string
    83  		err         error
    84  		want        bool
    85  	}{
    86  		{"400 is fatal because the request body cannot succeed on retry",
    87  			&anthropic.Error{StatusCode: 400}, true},
    88  		{"401 is fatal because a bad environment key never starts working",
    89  			&anthropic.Error{StatusCode: 401}, true},
    90  		{"403 is fatal because a forbidden caller stays forbidden",
    91  			&anthropic.Error{StatusCode: 403}, true},
    92  		{"404 is fatal because a missing environment or work item stays missing",
    93  			&anthropic.Error{StatusCode: 404}, true},
    94  		{"408 is excluded because timeouts deserve backoff, not teardown",
    95  			&anthropic.Error{StatusCode: 408}, false},
    96  		{"409 is excluded because a conflict is retried by the core client too",
    97  			&anthropic.Error{StatusCode: 409}, false},
    98  		{"412 stays fatal so a reclaimed lease still cancels the session at once",
    99  			&anthropic.Error{StatusCode: 412}, true},
   100  		{"422 is fatal because an unprocessable request cannot succeed on retry",
   101  			&anthropic.Error{StatusCode: 422}, true},
   102  		{"429 is excluded because rate-limits deserve backoff, not teardown",
   103  			&anthropic.Error{StatusCode: 429}, false},
   104  		{"500 is excluded because server-side errors retry, not abort",
   105  			&anthropic.Error{StatusCode: 500}, false},
   106  		{"non-api error is never classified as a permanent client error",
   107  			errors.New("409 conflict"), false},
   108  		{"nil error is not fatal",
   109  			nil, false},
   110  	}
   111  	for _, tc := range tests {
   112  		t.Run(tc.description, func(t *testing.T) {
   113  			require.Equal(t, tc.want, isFatal4xx(tc.err))
   114  		})
   115  	}
   116  }
   117  
   118  // ---------------------------------------------------------------------------
   119  // Integration tests against an httptest server scripting poll/ack/stop.
   120  // ---------------------------------------------------------------------------
   121  
   122  type recordedCall struct {
   123  	method string
   124  	path   string
   125  	auth   string
   126  	// apiKey captures the X-Api-Key header value on the recorded request.
   127  	// Runner helpers must NOT leak the parent client's default API key
   128  	// alongside their bearer credential; tests assert this is empty on
   129  	// every helper-issued call.
   130  	apiKey string
   131  	body   string
   132  	header http.Header
   133  }
   134  
   135  // fakeWorkServer is a minimal httptest server that records every request
   136  // and dispatches to per-endpoint handler functions provided by the test.
   137  // Tests assign Handle* fields before constructing the helper under test;
   138  // calls to any unassigned endpoint fail the test. Shared by both
   139  // poller_test.go and dispatcher_test.go.
   140  type fakeWorkServer struct {
   141  	t      *testing.T
   142  	server *httptest.Server
   143  
   144  	mu    sync.Mutex
   145  	calls []recordedCall
   146  
   147  	// Environment work endpoints (poller).
   148  	HandlePoll http.HandlerFunc
   149  	HandleAck  http.HandlerFunc
   150  	HandleStop http.HandlerFunc
   151  
   152  	// Environment work endpoints (dispatcher).
   153  	HandleHeartbeat http.HandlerFunc
   154  
   155  	// Session event endpoints (dispatcher).
   156  	HandleStream http.HandlerFunc // GET  /v1/sessions/{id}/events/stream
   157  	HandleSend   http.HandlerFunc // POST /v1/sessions/{id}/events
   158  	HandleList   http.HandlerFunc // GET  /v1/sessions/{id}/events
   159  
   160  	// Session lookup endpoint (skill setup).
   161  	HandleSessionGet http.HandlerFunc // GET /v1/sessions/{id}
   162  
   163  	// Memory-store endpoints (memory sync). One handler serves everything
   164  	// under /v1/memory_stores/.
   165  	HandleMemories http.HandlerFunc
   166  }
   167  
   168  func newFakeWorkServer(t *testing.T) *fakeWorkServer {
   169  	f := &fakeWorkServer{t: t}
   170  	f.server = httptest.NewServer(http.HandlerFunc(f.serveHTTP))
   171  	t.Cleanup(func() { f.server.Close() })
   172  	return f
   173  }
   174  
   175  func (f *fakeWorkServer) serveHTTP(w http.ResponseWriter, r *http.Request) {
   176  	body, _ := io.ReadAll(r.Body)
   177  	// Rewind so the handler can re-read the body. Httptest handlers commonly
   178  	// inspect the payload to assert what the SDK sent.
   179  	r.Body = io.NopCloser(bytes.NewReader(body))
   180  	f.mu.Lock()
   181  	f.calls = append(f.calls, recordedCall{
   182  		method: r.Method,
   183  		path:   r.URL.Path,
   184  		auth:   r.Header.Get("Authorization"),
   185  		apiKey: r.Header.Get("X-Api-Key"),
   186  		body:   string(body),
   187  		header: r.Header.Clone(),
   188  	})
   189  	f.mu.Unlock()
   190  
   191  	switch {
   192  	case strings.HasPrefix(r.URL.Path, "/v1/memory_stores/"):
   193  		require.NotNil(f.t, f.HandleMemories, "unscripted memory_stores call: %s", r.URL.Path)
   194  		f.HandleMemories(w, r)
   195  	case strings.HasSuffix(r.URL.Path, "/work/poll"):
   196  		require.NotNil(f.t, f.HandlePoll, "unscripted Poll call: %s", r.URL.Path)
   197  		f.HandlePoll(w, r)
   198  	case strings.HasSuffix(r.URL.Path, "/ack"):
   199  		require.NotNil(f.t, f.HandleAck, "unscripted Ack call: %s", r.URL.Path)
   200  		f.HandleAck(w, r)
   201  	case strings.HasSuffix(r.URL.Path, "/stop"):
   202  		require.NotNil(f.t, f.HandleStop, "unscripted Stop call: %s", r.URL.Path)
   203  		f.HandleStop(w, r)
   204  	case strings.HasSuffix(r.URL.Path, "/heartbeat"):
   205  		require.NotNil(f.t, f.HandleHeartbeat, "unscripted Heartbeat call: %s", r.URL.Path)
   206  		f.HandleHeartbeat(w, r)
   207  	case strings.HasSuffix(r.URL.Path, "/events/stream"):
   208  		require.NotNil(f.t, f.HandleStream, "unscripted StreamEvents call: %s", r.URL.Path)
   209  		f.HandleStream(w, r)
   210  	case strings.HasSuffix(r.URL.Path, "/events"):
   211  		switch r.Method {
   212  		case http.MethodPost:
   213  			require.NotNil(f.t, f.HandleSend, "unscripted events.Send call: %s", r.URL.Path)
   214  			f.HandleSend(w, r)
   215  		case http.MethodGet:
   216  			require.NotNil(f.t, f.HandleList, "unscripted events.List call: %s", r.URL.Path)
   217  			f.HandleList(w, r)
   218  		default:
   219  			f.t.Errorf("unexpected method on events endpoint: %s", r.Method)
   220  			http.Error(w, "unexpected", http.StatusMethodNotAllowed)
   221  		}
   222  	case strings.Contains(r.URL.Path, "/sessions/"):
   223  		// Bare GET /v1/sessions/{id} — the worker's one session lookup, shared
   224  		// by the skills and memory-store downloads. The /events{,/stream} cases
   225  		// above already handled the event endpoints.
   226  		require.NotNil(f.t, f.HandleSessionGet, "unscripted Sessions.Get call: %s", r.URL.Path)
   227  		f.HandleSessionGet(w, r)
   228  	default:
   229  		f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
   230  		http.Error(w, "unexpected", http.StatusNotImplemented)
   231  	}
   232  }
   233  
   234  func (f *fakeWorkServer) Calls() []recordedCall {
   235  	f.mu.Lock()
   236  	defer f.mu.Unlock()
   237  	return slices.Clone(f.calls)
   238  }
   239  
   240  func (f *fakeWorkServer) Client() anthropic.Client {
   241  	return anthropic.NewClient(
   242  		option.WithBaseURL(f.server.URL),
   243  		option.WithAPIKey("client-default-key"),
   244  		option.WithMaxRetries(0),
   245  	)
   246  }
   247  
   248  // workJSON returns a minimally-valid BetaSelfHostedWork JSON body. All
   249  // api:"required" fields are present so the SDK's strict unmarshal accepts
   250  // it. dataType is "session" or "health_check". The "secret" field is emitted
   251  // empty (codegen marks it required); workJSONWithSecret sets a real payload
   252  // for the tests covering the per-item work secret.
   253  func workJSON(workID, envID, dataType string) string {
   254  	return workJSONWithSecret(workID, envID, dataType, "")
   255  }
   256  
   257  func workJSONWithSecret(workID, envID, dataType, secret string) string {
   258  	ts := "2026-05-11T12:00:00Z"
   259  	data := map[string]any{"type": dataType, "id": "sesn_test"}
   260  	if dataType == "session" {
   261  		data["mode"] = "private"
   262  	}
   263  	body, _ := json.Marshal(map[string]any{
   264  		"id":                  workID,
   265  		"acknowledged_at":     ts,
   266  		"actor":               map[string]any{"type": "api_key", "api_key_id": "apk_test"},
   267  		"created_at":          ts,
   268  		"data":                data,
   269  		"environment_id":      envID,
   270  		"latest_heartbeat_at": ts,
   271  		"metadata":            map[string]string{},
   272  		"secret":              secret,
   273  		"started_at":          ts,
   274  		"state":               "queued",
   275  		"stop_requested_at":   "",
   276  		"stopped_at":          "",
   277  		"type":                "work",
   278  	})
   279  	return string(body)
   280  }
   281  
   282  var silentLogger = slog.New(slog.NewTextHandler(io.Discard, nil))
   283  
   284  func TestWorkPoller_YieldsAndPostsStopOnClose(t *testing.T) {
   285  	server := newFakeWorkServer(t)
   286  	work := workJSON("work_1", "env_1", "session")
   287  
   288  	pollCount := 0
   289  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   290  		pollCount++
   291  		w.Header().Set("Content-Type", "application/json")
   292  		if pollCount == 1 {
   293  			_, _ = w.Write([]byte(work))
   294  		} else {
   295  			// Empty poll — null body — return 204 with nothing.
   296  			w.WriteHeader(http.StatusNoContent)
   297  		}
   298  	}
   299  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   300  		w.Header().Set("Content-Type", "application/json")
   301  		_, _ = w.Write([]byte(work))
   302  	}
   303  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   304  		w.Header().Set("Content-Type", "application/json")
   305  		_, _ = w.Write([]byte(work))
   306  	}
   307  
   308  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   309  	defer cancel()
   310  
   311  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   312  		EnvironmentID:  "env_1",
   313  		EnvironmentKey: "env_key",
   314  		WorkerID:       "test-worker",
   315  		Logger:         silentLogger,
   316  	})
   317  
   318  	require.True(t, p.Next(), "first Next() must yield the scripted work")
   319  	require.Equal(t, "work_1", p.Current().ID)
   320  
   321  	require.NoError(t, p.Close())
   322  
   323  	calls := server.Calls()
   324  	require.Len(t, calls, 3, "expected poll + ack + stop")
   325  	require.Contains(t, calls[0].path, "/work/poll")
   326  	require.Contains(t, calls[1].path, "/ack")
   327  	require.Contains(t, calls[2].path, "/stop")
   328  
   329  	require.Equal(t, "Bearer env_key", calls[0].auth,
   330  		"Poll must use the environment key")
   331  	require.Equal(t, "Bearer env_key", calls[1].auth,
   332  		"Ack must use the environment key")
   333  	require.Equal(t, "Bearer env_key", calls[2].auth,
   334  		"Stop must use the environment key")
   335  
   336  	// option.WithAuthToken alone only *adds* an Authorization header; the
   337  	// parent client's WithAPIKey middleware still sets X-Api-Key. The
   338  	// poller's helperReqOpts must explicitly delete X-Api-Key per-request so
   339  	// both headers don't ride on the wire together.
   340  	require.Empty(t, calls[0].apiKey,
   341  		"Poll must not leak the parent client's X-Api-Key")
   342  	require.Empty(t, calls[1].apiKey,
   343  		"Ack must not leak the parent client's X-Api-Key")
   344  	require.Empty(t, calls[2].apiKey,
   345  		"Stop must not leak the parent client's X-Api-Key")
   346  }
   347  
   348  func TestWorkPoller_StopRunsBeforeNextPoll(t *testing.T) {
   349  	server := newFakeWorkServer(t)
   350  	work1 := workJSON("work_1", "env_1", "session")
   351  	work2 := workJSON("work_2", "env_1", "session")
   352  
   353  	pollCount := 0
   354  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   355  		pollCount++
   356  		w.Header().Set("Content-Type", "application/json")
   357  		switch pollCount {
   358  		case 1:
   359  			_, _ = w.Write([]byte(work1))
   360  		case 2:
   361  			_, _ = w.Write([]byte(work2))
   362  		default:
   363  			w.WriteHeader(http.StatusNoContent)
   364  		}
   365  	}
   366  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   367  		w.Header().Set("Content-Type", "application/json")
   368  		_, _ = w.Write([]byte(work1))
   369  	}
   370  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   371  		w.Header().Set("Content-Type", "application/json")
   372  		_, _ = w.Write([]byte(work1))
   373  	}
   374  
   375  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   376  	defer cancel()
   377  
   378  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   379  		EnvironmentID:  "env_1",
   380  		EnvironmentKey: "env_key",
   381  		Logger:         silentLogger,
   382  	})
   383  	t.Cleanup(func() { _ = p.Close() })
   384  
   385  	require.True(t, p.Next())
   386  	require.Equal(t, "work_1", p.Current().ID)
   387  
   388  	require.True(t, p.Next())
   389  	require.Equal(t, "work_2", p.Current().ID)
   390  
   391  	calls := server.Calls()
   392  	require.GreaterOrEqual(t, len(calls), 5)
   393  
   394  	// Expected order: poll → ack(1) → stop(1) → poll → ack(2). Stop for
   395  	// work_1 must appear BEFORE the second poll, even though work_1 is
   396  	// already a different item from the second Next().
   397  	stop1Idx := -1
   398  	poll2Idx := -1
   399  	for i, c := range calls {
   400  		if stop1Idx == -1 && strings.Contains(c.path, "work_1/stop") {
   401  			stop1Idx = i
   402  		}
   403  		if stop1Idx != -1 && poll2Idx == -1 && strings.Contains(c.path, "/work/poll") {
   404  			poll2Idx = i
   405  		}
   406  	}
   407  	require.NotEqual(t, -1, stop1Idx, "stop for work_1 was never posted")
   408  	require.NotEqual(t, -1, poll2Idx, "second poll was never posted")
   409  	require.Less(t, stop1Idx, poll2Idx,
   410  		"deferred stop for work_1 must run before the second poll")
   411  }
   412  
   413  func TestWorkPoller_AckFailureSkips(t *testing.T) {
   414  	server := newFakeWorkServer(t)
   415  	first := workJSON("work_1", "env_1", "session")
   416  	second := workJSON("work_2", "env_1", "session")
   417  
   418  	pollCount := 0
   419  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   420  		pollCount++
   421  		w.Header().Set("Content-Type", "application/json")
   422  		switch pollCount {
   423  		case 1:
   424  			_, _ = w.Write([]byte(first))
   425  		case 2:
   426  			_, _ = w.Write([]byte(second))
   427  		default:
   428  			w.WriteHeader(http.StatusNoContent)
   429  		}
   430  	}
   431  	ackCount := 0
   432  	server.HandleAck = func(w http.ResponseWriter, r *http.Request) {
   433  		ackCount++
   434  		if strings.Contains(r.URL.Path, "work_1") {
   435  			http.Error(w, `{"type":"error","error":{"type":"server_error","message":"sim ack failure"}}`,
   436  				http.StatusInternalServerError)
   437  			return
   438  		}
   439  		w.Header().Set("Content-Type", "application/json")
   440  		_, _ = w.Write([]byte(second))
   441  	}
   442  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   443  		// Stop is now posted both for the item whose ack failed (a best-effort
   444  		// force-stop so it doesn't dangle) and, on Close, for the item that
   445  		// successfully yielded.
   446  		w.Header().Set("Content-Type", "application/json")
   447  		_, _ = w.Write([]byte(second))
   448  	}
   449  
   450  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   451  	defer cancel()
   452  
   453  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   454  		EnvironmentID:  "env_1",
   455  		EnvironmentKey: "env_key",
   456  		Logger:         silentLogger,
   457  	})
   458  	t.Cleanup(func() { _ = p.Close() })
   459  
   460  	require.True(t, p.Next())
   461  	require.Equal(t, "work_2", p.Current().ID,
   462  		"work whose ack 5xx'd should be skipped; iterator yields the next item")
   463  	require.Equal(t, 2, ackCount, "both acks were attempted; only the second succeeded")
   464  
   465  	// The unprocessable item (ack 5xx'd) must be force-stopped rather than left
   466  	// dangling, before the poller moves on to the next item.
   467  	var work1Stop *recordedCall
   468  	for _, c := range server.Calls() {
   469  		if strings.Contains(c.path, "work_1/stop") {
   470  			c := c
   471  			work1Stop = &c
   472  			break
   473  		}
   474  	}
   475  	require.NotNil(t, work1Stop, "ack-failed work_1 must be force-stopped, not silently skipped")
   476  	require.Contains(t, work1Stop.body, `"force":true`, "discard stop must force")
   477  }
   478  
   479  // writeAPIError writes an API-shaped error body with the given status so the
   480  // SDK surfaces it as an *anthropic.Error carrying that status code.
   481  func writeAPIError(w http.ResponseWriter, status int) {
   482  	http.Error(w, `{"type":"error","error":{"type":"api_error","message":"scripted"}}`, status)
   483  }
   484  
   485  // TestWorkPoller_BacksOffOn409ThenYields relies on the fake server's client
   486  // using WithMaxRetries(0): with core-client retries enabled the 409 would be
   487  // absorbed before the poller's own classification ever ran.
   488  func TestWorkPoller_BacksOffOn409ThenYields(t *testing.T) {
   489  	server := newFakeWorkServer(t)
   490  	work := workJSON("work_1", "env_1", "session")
   491  
   492  	pollCount := 0
   493  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   494  		pollCount++
   495  		if pollCount == 1 {
   496  			writeAPIError(w, http.StatusConflict)
   497  			return
   498  		}
   499  		w.Header().Set("Content-Type", "application/json")
   500  		_, _ = w.Write([]byte(work))
   501  	}
   502  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   503  		w.Header().Set("Content-Type", "application/json")
   504  		_, _ = w.Write([]byte(work))
   505  	}
   506  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   507  		w.WriteHeader(http.StatusNoContent)
   508  	}
   509  
   510  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
   511  	defer cancel()
   512  
   513  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   514  		EnvironmentID:  "env_1",
   515  		EnvironmentKey: "env_key",
   516  		Logger:         silentLogger,
   517  	})
   518  	t.Cleanup(func() { _ = p.Close() })
   519  
   520  	require.True(t, p.Next(), "a 409 on poll must be retried after a backoff, not end iteration")
   521  	require.Equal(t, "work_1", p.Current().ID)
   522  	require.NoError(t, p.Err())
   523  	require.Equal(t, 2, pollCount)
   524  }
   525  
   526  func TestWorkPoller_Stop409IsSilent(t *testing.T) {
   527  	server := newFakeWorkServer(t)
   528  	work := workJSON("work_1", "env_1", "session")
   529  
   530  	pollCount := 0
   531  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   532  		pollCount++
   533  		if pollCount == 1 {
   534  			w.Header().Set("Content-Type", "application/json")
   535  			_, _ = w.Write([]byte(work))
   536  			return
   537  		}
   538  		writeEmptyPoll(w)
   539  	}
   540  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   541  		w.Header().Set("Content-Type", "application/json")
   542  		_, _ = w.Write([]byte(work))
   543  	}
   544  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   545  		writeAPIError(w, http.StatusConflict)
   546  	}
   547  
   548  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   549  	defer cancel()
   550  
   551  	var logs lockedBuffer
   552  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   553  		EnvironmentID:  "env_1",
   554  		EnvironmentKey: "env_key",
   555  		Drain:          true,
   556  		Logger:         slog.New(slog.NewTextHandler(&logs, nil)),
   557  	})
   558  	t.Cleanup(func() { _ = p.Close() })
   559  
   560  	require.True(t, p.Next())
   561  	require.False(t, p.Next(), "iteration continues past the 409 stop and drains normally")
   562  	require.NoError(t, p.Err())
   563  	require.NotContains(t, logs.String(), "stop failed", "a 409 on stop means already stopped and is not worth a warning")
   564  }
   565  
   566  func TestWorkPoller_SurfacesPermanent4xxOnPoll(t *testing.T) {
   567  	server := newFakeWorkServer(t)
   568  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   569  		writeAPIError(w, http.StatusUnauthorized)
   570  	}
   571  
   572  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   573  	defer cancel()
   574  
   575  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   576  		EnvironmentID:  "env_1",
   577  		EnvironmentKey: "env_key",
   578  		Logger:         silentLogger,
   579  	})
   580  	t.Cleanup(func() { _ = p.Close() })
   581  
   582  	require.False(t, p.Next(), "a 401 on poll will never succeed and must end iteration")
   583  	require.ErrorContains(t, p.Err(), "environments: poll failed")
   584  	require.True(t, isStatus(p.Err(), http.StatusUnauthorized), "Err must wrap the API error")
   585  	require.Len(t, server.Calls(), 1, "a permanent 4xx must not be retried")
   586  }
   587  
   588  func TestWorkPoller_CtxCancelDuringPollExitsCleanly(t *testing.T) {
   589  	server := newFakeWorkServer(t)
   590  	pollUnblocked := make(chan struct{})
   591  	server.HandlePoll = func(w http.ResponseWriter, r *http.Request) {
   592  		<-pollUnblocked
   593  		select {
   594  		case <-r.Context().Done():
   595  			// Server saw the cancellation — return 499-ish; client side
   596  			// will surface ctx.Err.
   597  			http.Error(w, "cancelled", 499)
   598  		default:
   599  			w.WriteHeader(http.StatusNoContent)
   600  		}
   601  	}
   602  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   603  		t.Fatal("ack must not be called when ctx is cancelled before any work is claimed")
   604  	}
   605  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   606  		t.Fatal("stop must not be called when nothing was claimed")
   607  	}
   608  
   609  	ctx, cancel := context.WithCancel(context.Background())
   610  
   611  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   612  		EnvironmentID:  "env_1",
   613  		EnvironmentKey: "env_key",
   614  		Logger:         silentLogger,
   615  	})
   616  
   617  	done := make(chan bool, 1)
   618  	go func() {
   619  		done <- p.Next()
   620  	}()
   621  
   622  	// Give the poll request time to be in flight, then cancel.
   623  	time.Sleep(50 * time.Millisecond)
   624  	cancel()
   625  	close(pollUnblocked)
   626  
   627  	select {
   628  	case got := <-done:
   629  		require.False(t, got, "Next() must return false after ctx cancellation")
   630  		require.NoError(t, p.Err(), "ctx cancellation is normal termination, not an error")
   631  	case <-time.After(2 * time.Second):
   632  		t.Fatal("Next() did not return after ctx cancellation")
   633  	}
   634  
   635  	require.NoError(t, p.Close())
   636  }
   637  
   638  func TestWorkPoller_CloseIsIdempotent(t *testing.T) {
   639  	server := newFakeWorkServer(t)
   640  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   641  		w.WriteHeader(http.StatusNoContent)
   642  	}
   643  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   644  		t.Fatal("ack should not be called")
   645  	}
   646  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   647  		t.Fatal("stop should not be called when nothing was claimed")
   648  	}
   649  
   650  	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
   651  	defer cancel()
   652  
   653  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   654  		EnvironmentID:  "env_1",
   655  		EnvironmentKey: "env_key",
   656  		Logger:         silentLogger,
   657  	})
   658  
   659  	require.NoError(t, p.Close())
   660  	require.NoError(t, p.Close(), "Close must be safe to call multiple times")
   661  
   662  	require.False(t, p.Next(), "Next after Close must return false immediately")
   663  }
   664  
   665  func TestWorkPoller_All_RangeOverFunc(t *testing.T) {
   666  	server := newFakeWorkServer(t)
   667  	work := workJSON("work_1", "env_1", "session")
   668  
   669  	pollCount := 0
   670  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   671  		pollCount++
   672  		w.Header().Set("Content-Type", "application/json")
   673  		if pollCount == 1 {
   674  			_, _ = w.Write([]byte(work))
   675  		} else {
   676  			w.WriteHeader(http.StatusNoContent)
   677  		}
   678  	}
   679  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   680  		w.Header().Set("Content-Type", "application/json")
   681  		_, _ = w.Write([]byte(work))
   682  	}
   683  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   684  		w.Header().Set("Content-Type", "application/json")
   685  		_, _ = w.Write([]byte(work))
   686  	}
   687  
   688  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   689  	defer cancel()
   690  
   691  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   692  		EnvironmentID:  "env_1",
   693  		EnvironmentKey: "env_key",
   694  		Logger:         silentLogger,
   695  	})
   696  	t.Cleanup(func() { _ = p.Close() })
   697  
   698  	var seen []string
   699  	for item, err := range p.All() {
   700  		require.NoError(t, err)
   701  		seen = append(seen, item.ID)
   702  		if len(seen) == 1 {
   703  			break
   704  		}
   705  	}
   706  	require.Equal(t, []string{"work_1"}, seen)
   707  }
   708  
   709  // writeEmptyPoll writes the response the Go SDK decodes as an empty queue:
   710  // 200 with a JSON `null` body, leaving the decoded *BetaSelfHostedWork nil.
   711  // (A bare 204 has no Content-Type and trips the strict decoder instead.)
   712  func writeEmptyPoll(w http.ResponseWriter) {
   713  	w.Header().Set("Content-Type", "application/json")
   714  	_, _ = w.Write([]byte("null"))
   715  }
   716  
   717  // TestWorkPoller_DrainReturnsOnEmptyQueue asserts that with Drain set the
   718  // poller ends iteration as soon as a poll comes back empty, instead of
   719  // sleeping and re-polling forever. Err must stay nil (normal termination).
   720  func TestWorkPoller_DrainReturnsOnEmptyQueue(t *testing.T) {
   721  	server := newFakeWorkServer(t)
   722  
   723  	pollCount := 0
   724  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   725  		pollCount++
   726  		writeEmptyPoll(w)
   727  	}
   728  
   729  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   730  	defer cancel()
   731  
   732  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   733  		EnvironmentID:  "env_1",
   734  		EnvironmentKey: "env_key",
   735  		Drain:          true,
   736  		Logger:         silentLogger,
   737  	})
   738  	t.Cleanup(func() { _ = p.Close() })
   739  
   740  	start := time.Now()
   741  	require.False(t, p.Next(), "Drain must end iteration on the first empty poll")
   742  	require.NoError(t, p.Err(), "draining the queue is normal termination, not an error")
   743  	require.Less(t, time.Since(start), 2*time.Second,
   744  		"Drain must return promptly rather than sleeping between empty polls")
   745  	require.Equal(t, 1, pollCount, "Drain must not re-poll after an empty queue")
   746  }
   747  
   748  // TestWorkPoller_DrainYieldsThenReturns asserts Drain still yields available
   749  // work and only returns once the queue empties — and that the last item is
   750  // still stopped on the way out (the poller's auto-stop model is unchanged).
   751  func TestWorkPoller_DrainYieldsThenReturns(t *testing.T) {
   752  	server := newFakeWorkServer(t)
   753  	work := workJSON("work_1", "env_1", "session")
   754  
   755  	pollCount := 0
   756  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   757  		pollCount++
   758  		if pollCount == 1 {
   759  			w.Header().Set("Content-Type", "application/json")
   760  			_, _ = w.Write([]byte(work))
   761  			return
   762  		}
   763  		writeEmptyPoll(w)
   764  	}
   765  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   766  		w.Header().Set("Content-Type", "application/json")
   767  		_, _ = w.Write([]byte(work))
   768  	}
   769  	stopped := 0
   770  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   771  		stopped++
   772  		w.Header().Set("Content-Type", "application/json")
   773  		_, _ = w.Write([]byte(work))
   774  	}
   775  
   776  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   777  	defer cancel()
   778  
   779  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   780  		EnvironmentID:  "env_1",
   781  		EnvironmentKey: "env_key",
   782  		Drain:          true,
   783  		Logger:         silentLogger,
   784  	})
   785  	t.Cleanup(func() { _ = p.Close() })
   786  
   787  	require.True(t, p.Next(), "first Next must yield the scripted work")
   788  	require.Equal(t, "work_1", p.Current().ID)
   789  	require.False(t, p.Next(), "second Next must drain-return on the empty queue")
   790  	require.NoError(t, p.Err())
   791  	require.Equal(t, 1, stopped, "the yielded item must still be stopped on drain exit")
   792  }
   793  
   794  // With AutoStop off the poller never posts Stop for a yielded item, neither
   795  // when the consumer advances nor on Close: the consumer owns the item's stop.
   796  func TestWorkPoller_AutoStopOffPostsNoStop(t *testing.T) {
   797  	server := newFakeWorkServer(t)
   798  	work := workJSON("work_1", "env_1", "session")
   799  
   800  	pollCount := 0
   801  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) {
   802  		pollCount++
   803  		if pollCount == 1 {
   804  			w.Header().Set("Content-Type", "application/json")
   805  			_, _ = w.Write([]byte(work))
   806  			return
   807  		}
   808  		writeEmptyPoll(w)
   809  	}
   810  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   811  		w.Header().Set("Content-Type", "application/json")
   812  		_, _ = w.Write([]byte(work))
   813  	}
   814  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   815  		w.WriteHeader(http.StatusNoContent)
   816  	}
   817  
   818  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   819  	defer cancel()
   820  
   821  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   822  		EnvironmentID:  "env_1",
   823  		EnvironmentKey: "env_key",
   824  		Drain:          true,
   825  		AutoStop:       param.NewOpt(false),
   826  		Logger:         silentLogger,
   827  	})
   828  
   829  	require.True(t, p.Next(), "first Next must yield the scripted work")
   830  	require.Equal(t, "work_1", p.Current().ID)
   831  	require.False(t, p.Next(), "second Next must drain-return on the empty queue")
   832  	require.NoError(t, p.Err())
   833  	require.NoError(t, p.Close())
   834  	require.Empty(t, callsEndingIn(server.Calls(), "/stop"), "with AutoStop off the poller must not stop the yielded item")
   835  }
   836  
   837  // TestWorkPoller_BlockMsWiring asserts how WorkPollerOptions.BlockMs and
   838  // ReclaimOlderThanMs map onto the poll query: omitted -> default 999,
   839  // param.Null -> omitted entirely (non-blocking), an explicit value passes
   840  // through, and ReclaimOlderThanMs only appears when set.
   841  func TestWorkPoller_BlockMsWiring(t *testing.T) {
   842  	cases := []struct {
   843  		name        string
   844  		opts        WorkPollerOptions
   845  		wantBlock   string // expected block_ms query value, "" means absent
   846  		wantReclaim string // expected reclaim_older_than_ms, "" means absent
   847  	}{
   848  		{
   849  			name:      "default omits to 999",
   850  			opts:      WorkPollerOptions{},
   851  			wantBlock: "999",
   852  		},
   853  		{
   854  			name:      "explicit value passes through",
   855  			opts:      WorkPollerOptions{BlockMs: param.NewOpt(int64(250))},
   856  			wantBlock: "250",
   857  		},
   858  		{
   859  			name:      "null omits block_ms for a non-blocking poll",
   860  			opts:      WorkPollerOptions{BlockMs: param.Null[int64]()},
   861  			wantBlock: "",
   862  		},
   863  		{
   864  			name:        "reclaim_older_than_ms threads through when set",
   865  			opts:        WorkPollerOptions{ReclaimOlderThanMs: param.NewOpt(int64(5000))},
   866  			wantBlock:   "999",
   867  			wantReclaim: "5000",
   868  		},
   869  	}
   870  
   871  	for _, tc := range cases {
   872  		t.Run(tc.name, func(t *testing.T) {
   873  			server := newFakeWorkServer(t)
   874  			var gotBlock, gotReclaim string
   875  			var gotBlockSet, gotReclaimSet bool
   876  			server.HandlePoll = func(w http.ResponseWriter, r *http.Request) {
   877  				q := r.URL.Query()
   878  				gotBlock, gotBlockSet = q.Get("block_ms"), q.Has("block_ms")
   879  				gotReclaim, gotReclaimSet = q.Get("reclaim_older_than_ms"), q.Has("reclaim_older_than_ms")
   880  				writeEmptyPoll(w)
   881  			}
   882  
   883  			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   884  			defer cancel()
   885  
   886  			opts := tc.opts
   887  			opts.EnvironmentID = "env_1"
   888  			opts.EnvironmentKey = "env_key"
   889  			opts.Drain = true // end after the single empty poll
   890  			opts.Logger = silentLogger
   891  			p := NewWorkPoller(ctx, server.Client(), opts)
   892  			t.Cleanup(func() { _ = p.Close() })
   893  
   894  			require.False(t, p.Next())
   895  			require.NoError(t, p.Err())
   896  
   897  			if tc.wantBlock == "" {
   898  				require.False(t, gotBlockSet, "block_ms must be absent for a non-blocking poll")
   899  			} else {
   900  				require.True(t, gotBlockSet, "block_ms must be present")
   901  				require.Equal(t, tc.wantBlock, gotBlock)
   902  			}
   903  			if tc.wantReclaim == "" {
   904  				require.False(t, gotReclaimSet, "reclaim_older_than_ms must be absent unless set")
   905  			} else {
   906  				require.True(t, gotReclaimSet, "reclaim_older_than_ms must be present")
   907  				require.Equal(t, tc.wantReclaim, gotReclaim)
   908  			}
   909  		})
   910  	}
   911  }
   912  
   913  // lockedClock is a settable time source that a handler goroutine may advance
   914  // while the code under test reads it.
   915  type lockedClock struct {
   916  	mu  sync.Mutex
   917  	now time.Time
   918  }
   919  
   920  func (c *lockedClock) Now() time.Time {
   921  	c.mu.Lock()
   922  	defer c.mu.Unlock()
   923  	return c.now
   924  }
   925  
   926  func (c *lockedClock) Advance(d time.Duration) {
   927  	c.mu.Lock()
   928  	c.now = c.now.Add(d)
   929  	c.mu.Unlock()
   930  }
   931  
   932  // recordsAt returns the captured log records at level, in order, with the
   933  // leading time attribute dropped so whole records compare stably.
   934  func recordsAt(logs string, level slog.Level) []string {
   935  	prefix := "level=" + level.String() + " "
   936  	var records []string
   937  	for _, line := range strings.Split(strings.TrimSpace(logs), "\n") {
   938  		if _, rec, ok := strings.Cut(line, " "); ok && strings.HasPrefix(rec, prefix) {
   939  			records = append(records, rec)
   940  		}
   941  	}
   942  	return records
   943  }
   944  
   945  // An idle poller shows up in the logs without an INFO line per poll: one INFO
   946  // line when it goes idle (after start-up or after releasing an item), DEBUG
   947  // per empty poll after that, and an INFO reminder every 5 minutes.
   948  func TestWorkPoller_LogsIdlePolling(t *testing.T) {
   949  	server := newFakeWorkServer(t)
   950  	work := workJSON("work_a", "env_1", "session")
   951  	clock := &lockedClock{now: time.Unix(0, 0)}
   952  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   953  	defer cancel()
   954  
   955  	// Three empty polls, a fourth 301 s later, a fifth, one claim, one more
   956  	// empty poll, then the run ends.
   957  	var polls atomic.Int32
   958  	server.HandlePoll = func(w http.ResponseWriter, r *http.Request) {
   959  		switch polls.Add(1) {
   960  		case 4:
   961  			clock.Advance(301 * time.Second)
   962  			writeEmptyPoll(w)
   963  		case 6:
   964  			w.Header().Set("Content-Type", "application/json")
   965  			_, _ = w.Write([]byte(work))
   966  		case 8:
   967  			cancel()
   968  			<-r.Context().Done()
   969  		default:
   970  			writeEmptyPoll(w)
   971  		}
   972  	}
   973  	server.HandleAck = func(w http.ResponseWriter, _ *http.Request) {
   974  		w.Header().Set("Content-Type", "application/json")
   975  		_, _ = w.Write([]byte(work))
   976  	}
   977  	server.HandleStop = func(w http.ResponseWriter, _ *http.Request) {
   978  		w.WriteHeader(http.StatusNoContent)
   979  	}
   980  
   981  	var logBuf bytes.Buffer
   982  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
   983  		EnvironmentID:  "env_1",
   984  		EnvironmentKey: "env_key",
   985  		WorkerID:       "test-worker",
   986  		Logger:         captureLog(&logBuf),
   987  	})
   988  	p.now = clock.Now
   989  	p.emptyPollWait = 0
   990  	t.Cleanup(func() { _ = p.Close() })
   991  
   992  	require.True(t, p.Next())
   993  	require.Equal(t, "work_a", p.Current().ID)
   994  	require.False(t, p.Next())
   995  	require.NoError(t, p.Err())
   996  
   997  	const scope = " component=work-poller environment_id=env_1"
   998  	logs := logBuf.String()
   999  	require.Equal(t, []string{
  1000  		`level=INFO msg="idle; polling for work"` + scope,
  1001  		`level=INFO msg="still polling; idle for 5m1s"` + scope,
  1002  		`level=INFO msg="claimed work"` + scope + ` work_id=work_a work_type=session`,
  1003  		`level=INFO msg="idle; polling for work"` + scope,
  1004  	}, recordsAt(logs, slog.LevelInfo))
  1005  	debug := `level=DEBUG msg="poll returned no work"` + scope
  1006  	require.Equal(t, []string{debug, debug, debug}, recordsAt(logs, slog.LevelDebug))
  1007  }
  1008  
  1009  // A draining poller that finds the queue empty is finishing, not idling, so it
  1010  // logs the drain and no idle line.
  1011  func TestWorkPoller_DrainDoesNotLogIdle(t *testing.T) {
  1012  	server := newFakeWorkServer(t)
  1013  	server.HandlePoll = func(w http.ResponseWriter, _ *http.Request) { writeEmptyPoll(w) }
  1014  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
  1015  	defer cancel()
  1016  
  1017  	var logBuf bytes.Buffer
  1018  	p := NewWorkPoller(ctx, server.Client(), WorkPollerOptions{
  1019  		EnvironmentID:  "env_1",
  1020  		EnvironmentKey: "env_key",
  1021  		WorkerID:       "test-worker",
  1022  		Drain:          true,
  1023  		Logger:         captureLog(&logBuf),
  1024  	})
  1025  	t.Cleanup(func() { _ = p.Close() })
  1026  
  1027  	require.False(t, p.Next())
  1028  	require.NoError(t, p.Err())
  1029  	logs := logBuf.String()
  1030  	require.Contains(t, logs, `msg="work queue drained"`)
  1031  	require.NotContains(t, logs, "idle; polling for work")
  1032  }