INNER CODE UNIT · Go

newStreamingServer

bhope/hedge · benchmark/streaming/main.go:106

func newStreamingServer(hits *atomic.Int64) *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hits.Add(1)
		w.WriteHeader(http.StatusOK)
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
		select {
		case <-time.After(sampleTTFT()):
			w.Write([]byte("tok")) //nolint:errcheck
		case <-r.Context().Done():
		}
	}))
}

// newGatewayServer delays the full response (headers + body) by TTFT. Models
// an LLM gateway that buffers until the first token is available, so TTFH ≈
// TTFT. The hedge timer fires while RoundTrip blocks waiting for headers.

View source record →