fix(auto-reply): coalesce block replies and document streaming toggles (#536) (thanks @mcinteerj)
parent
a05916bee8
commit
fd15704c77
@ -0,0 +1,132 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { TemplateContext } from "../templating.js";
|
||||
import type { FollowupRun, QueueSettings } from "./queue.js";
|
||||
import { createMockTypingController } from "./test-helpers.js";
|
||||
|
||||
const runEmbeddedPiAgentMock = vi.fn();
|
||||
|
||||
vi.mock("../../agents/model-fallback.js", () => ({
|
||||
runWithModelFallback: async ({
|
||||
provider,
|
||||
model,
|
||||
run,
|
||||
}: {
|
||||
provider: string;
|
||||
model: string;
|
||||
run: (provider: string, model: string) => Promise<unknown>;
|
||||
}) => ({
|
||||
result: await run(provider, model),
|
||||
provider,
|
||||
model,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/pi-embedded.js", () => ({
|
||||
queueEmbeddedPiMessage: vi.fn().mockReturnValue(false),
|
||||
runEmbeddedPiAgent: (params: unknown) => runEmbeddedPiAgentMock(params),
|
||||
}));
|
||||
|
||||
vi.mock("./queue.js", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("./queue.js")>("./queue.js");
|
||||
return {
|
||||
...actual,
|
||||
enqueueFollowupRun: vi.fn(),
|
||||
scheduleFollowupDrain: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { runReplyAgent } from "./agent-runner.js";
|
||||
|
||||
describe("runReplyAgent block streaming", () => {
|
||||
it("coalesces duplicate text_end block replies", async () => {
|
||||
const onBlockReply = vi.fn();
|
||||
runEmbeddedPiAgentMock.mockImplementationOnce(async (params) => {
|
||||
const block = params.onBlockReply as
|
||||
| ((payload: { text?: string }) => void)
|
||||
| undefined;
|
||||
block?.({ text: "Hello" });
|
||||
block?.({ text: "Hello" });
|
||||
return {
|
||||
payloads: [{ text: "Final message" }],
|
||||
meta: {},
|
||||
};
|
||||
});
|
||||
|
||||
const typing = createMockTypingController();
|
||||
const sessionCtx = {
|
||||
Provider: "discord",
|
||||
OriginatingTo: "channel:C1",
|
||||
AccountId: "primary",
|
||||
MessageSid: "msg",
|
||||
} as unknown as TemplateContext;
|
||||
const resolvedQueue = { mode: "interrupt" } as unknown as QueueSettings;
|
||||
const followupRun = {
|
||||
prompt: "hello",
|
||||
summaryLine: "hello",
|
||||
enqueuedAt: Date.now(),
|
||||
run: {
|
||||
sessionId: "session",
|
||||
sessionKey: "main",
|
||||
messageProvider: "discord",
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
workspaceDir: "/tmp",
|
||||
config: {
|
||||
agents: {
|
||||
defaults: {
|
||||
blockStreamingCoalesce: {
|
||||
minChars: 1,
|
||||
maxChars: 200,
|
||||
idleMs: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
skillsSnapshot: {},
|
||||
provider: "anthropic",
|
||||
model: "claude",
|
||||
thinkLevel: "low",
|
||||
verboseLevel: "off",
|
||||
elevatedLevel: "off",
|
||||
bashElevated: {
|
||||
enabled: false,
|
||||
allowed: false,
|
||||
defaultLevel: "off",
|
||||
},
|
||||
timeoutMs: 1_000,
|
||||
blockReplyBreak: "text_end",
|
||||
},
|
||||
} as unknown as FollowupRun;
|
||||
|
||||
const result = await runReplyAgent({
|
||||
commandBody: "hello",
|
||||
followupRun,
|
||||
queueKey: "main",
|
||||
resolvedQueue,
|
||||
shouldSteer: false,
|
||||
shouldFollowup: false,
|
||||
isActive: false,
|
||||
isStreaming: false,
|
||||
opts: { onBlockReply },
|
||||
typing,
|
||||
sessionCtx,
|
||||
defaultModel: "anthropic/claude-opus-4-5",
|
||||
resolvedVerboseLevel: "off",
|
||||
isNewSession: false,
|
||||
blockStreamingEnabled: true,
|
||||
blockReplyChunking: {
|
||||
minChars: 1,
|
||||
maxChars: 200,
|
||||
breakPreference: "paragraph",
|
||||
},
|
||||
resolvedBlockStreamingBreak: "text_end",
|
||||
shouldInjectGroupIntro: false,
|
||||
typingMode: "instant",
|
||||
});
|
||||
|
||||
expect(onBlockReply).toHaveBeenCalledTimes(1);
|
||||
expect(onBlockReply.mock.calls[0][0].text).toBe("Hello");
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,71 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createBlockReplyCoalescer } from "./block-reply-coalescer.js";
|
||||
|
||||
describe("block reply coalescer", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("coalesces chunks within the idle window", async () => {
|
||||
vi.useFakeTimers();
|
||||
const flushes: string[] = [];
|
||||
const coalescer = createBlockReplyCoalescer({
|
||||
config: { minChars: 1, maxChars: 200, idleMs: 100, joiner: " " },
|
||||
shouldAbort: () => false,
|
||||
onFlush: (payload) => {
|
||||
flushes.push(payload.text ?? "");
|
||||
},
|
||||
});
|
||||
|
||||
coalescer.enqueue({ text: "Hello" });
|
||||
coalescer.enqueue({ text: "world" });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(flushes).toEqual(["Hello world"]);
|
||||
coalescer.stop();
|
||||
});
|
||||
|
||||
it("waits until minChars before idle flush", async () => {
|
||||
vi.useFakeTimers();
|
||||
const flushes: string[] = [];
|
||||
const coalescer = createBlockReplyCoalescer({
|
||||
config: { minChars: 10, maxChars: 200, idleMs: 50, joiner: " " },
|
||||
shouldAbort: () => false,
|
||||
onFlush: (payload) => {
|
||||
flushes.push(payload.text ?? "");
|
||||
},
|
||||
});
|
||||
|
||||
coalescer.enqueue({ text: "short" });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
expect(flushes).toEqual([]);
|
||||
|
||||
coalescer.enqueue({ text: "message" });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
expect(flushes).toEqual(["short message"]);
|
||||
coalescer.stop();
|
||||
});
|
||||
|
||||
it("flushes buffered text before media payloads", () => {
|
||||
const flushes: Array<{ text?: string; mediaUrls?: string[] }> = [];
|
||||
const coalescer = createBlockReplyCoalescer({
|
||||
config: { minChars: 1, maxChars: 200, idleMs: 0, joiner: " " },
|
||||
shouldAbort: () => false,
|
||||
onFlush: (payload) => {
|
||||
flushes.push({
|
||||
text: payload.text,
|
||||
mediaUrls: payload.mediaUrls,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
coalescer.enqueue({ text: "Hello" });
|
||||
coalescer.enqueue({ text: "world" });
|
||||
coalescer.enqueue({ mediaUrls: ["https://example.com/a.png"] });
|
||||
void coalescer.flush({ force: true });
|
||||
|
||||
expect(flushes[0].text).toBe("Hello world");
|
||||
expect(flushes[1].mediaUrls).toEqual(["https://example.com/a.png"]);
|
||||
coalescer.stop();
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,125 @@
|
||||
import type { ReplyPayload } from "../types.js";
|
||||
import type { BlockStreamingCoalescing } from "./block-streaming.js";
|
||||
|
||||
export type BlockReplyCoalescer = {
|
||||
enqueue: (payload: ReplyPayload) => void;
|
||||
flush: (options?: { force?: boolean }) => Promise<void>;
|
||||
hasBuffered: () => boolean;
|
||||
stop: () => void;
|
||||
};
|
||||
|
||||
export function createBlockReplyCoalescer(params: {
|
||||
config: BlockStreamingCoalescing;
|
||||
shouldAbort: () => boolean;
|
||||
onFlush: (payload: ReplyPayload) => Promise<void> | void;
|
||||
}): BlockReplyCoalescer {
|
||||
const { config, shouldAbort, onFlush } = params;
|
||||
const minChars = Math.max(1, Math.floor(config.minChars));
|
||||
const maxChars = Math.max(minChars, Math.floor(config.maxChars));
|
||||
const idleMs = Math.max(0, Math.floor(config.idleMs));
|
||||
const joiner = config.joiner ?? "";
|
||||
|
||||
let bufferText = "";
|
||||
let bufferReplyToId: ReplyPayload["replyToId"];
|
||||
let bufferAudioAsVoice: ReplyPayload["audioAsVoice"];
|
||||
let idleTimer: NodeJS.Timeout | undefined;
|
||||
|
||||
const clearIdleTimer = () => {
|
||||
if (!idleTimer) return;
|
||||
clearTimeout(idleTimer);
|
||||
idleTimer = undefined;
|
||||
};
|
||||
|
||||
const resetBuffer = () => {
|
||||
bufferText = "";
|
||||
bufferReplyToId = undefined;
|
||||
bufferAudioAsVoice = undefined;
|
||||
};
|
||||
|
||||
const scheduleIdleFlush = () => {
|
||||
if (idleMs <= 0) return;
|
||||
clearIdleTimer();
|
||||
idleTimer = setTimeout(() => {
|
||||
void flush({ force: false });
|
||||
}, idleMs);
|
||||
};
|
||||
|
||||
const flush = async (options?: { force?: boolean }) => {
|
||||
clearIdleTimer();
|
||||
if (shouldAbort()) {
|
||||
resetBuffer();
|
||||
return;
|
||||
}
|
||||
if (!bufferText) return;
|
||||
if (!options?.force && bufferText.length < minChars) {
|
||||
scheduleIdleFlush();
|
||||
return;
|
||||
}
|
||||
const payload: ReplyPayload = {
|
||||
text: bufferText,
|
||||
replyToId: bufferReplyToId,
|
||||
audioAsVoice: bufferAudioAsVoice,
|
||||
};
|
||||
resetBuffer();
|
||||
await onFlush(payload);
|
||||
};
|
||||
|
||||
const enqueue = (payload: ReplyPayload) => {
|
||||
if (shouldAbort()) return;
|
||||
const hasMedia =
|
||||
Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
|
||||
const text = payload.text ?? "";
|
||||
const hasText = text.trim().length > 0;
|
||||
if (hasMedia) {
|
||||
void flush({ force: true });
|
||||
void onFlush(payload);
|
||||
return;
|
||||
}
|
||||
if (!hasText) return;
|
||||
|
||||
if (
|
||||
bufferText &&
|
||||
(bufferReplyToId !== payload.replyToId ||
|
||||
bufferAudioAsVoice !== payload.audioAsVoice)
|
||||
) {
|
||||
void flush({ force: true });
|
||||
}
|
||||
|
||||
if (!bufferText) {
|
||||
bufferReplyToId = payload.replyToId;
|
||||
bufferAudioAsVoice = payload.audioAsVoice;
|
||||
}
|
||||
|
||||
const nextText = bufferText ? `${bufferText}${joiner}${text}` : text;
|
||||
if (nextText.length > maxChars) {
|
||||
if (bufferText) {
|
||||
void flush({ force: true });
|
||||
bufferReplyToId = payload.replyToId;
|
||||
bufferAudioAsVoice = payload.audioAsVoice;
|
||||
if (text.length >= maxChars) {
|
||||
void onFlush(payload);
|
||||
return;
|
||||
}
|
||||
bufferText = text;
|
||||
scheduleIdleFlush();
|
||||
return;
|
||||
}
|
||||
void onFlush(payload);
|
||||
return;
|
||||
}
|
||||
|
||||
bufferText = nextText;
|
||||
if (bufferText.length >= maxChars) {
|
||||
void flush({ force: true });
|
||||
return;
|
||||
}
|
||||
scheduleIdleFlush();
|
||||
};
|
||||
|
||||
return {
|
||||
enqueue,
|
||||
flush,
|
||||
hasBuffered: () => Boolean(bufferText),
|
||||
stop: () => clearIdleTimer(),
|
||||
};
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
import { logVerbose } from "../../globals.js";
|
||||
import type { ReplyPayload } from "../types.js";
|
||||
import { createBlockReplyCoalescer } from "./block-reply-coalescer.js";
|
||||
import type { BlockStreamingCoalescing } from "./block-streaming.js";
|
||||
|
||||
export type BlockReplyPipeline = {
|
||||
enqueue: (payload: ReplyPayload) => void;
|
||||
flush: (options?: { force?: boolean }) => Promise<void>;
|
||||
stop: () => void;
|
||||
hasBuffered: () => boolean;
|
||||
didStream: () => boolean;
|
||||
isAborted: () => boolean;
|
||||
hasSentPayload: (payload: ReplyPayload) => boolean;
|
||||
};
|
||||
|
||||
export function createBlockReplyPayloadKey(payload: ReplyPayload): string {
|
||||
const text = payload.text?.trim() ?? "";
|
||||
const mediaList = payload.mediaUrls?.length
|
||||
? payload.mediaUrls
|
||||
: payload.mediaUrl
|
||||
? [payload.mediaUrl]
|
||||
: [];
|
||||
return JSON.stringify({
|
||||
text,
|
||||
mediaList,
|
||||
replyToId: payload.replyToId ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
const withTimeout = async <T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
timeoutError: Error,
|
||||
): Promise<T> => {
|
||||
if (!timeoutMs || timeoutMs <= 0) return promise;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(timeoutError), timeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeoutPromise]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
|
||||
export function createBlockReplyPipeline(params: {
|
||||
onBlockReply: (
|
||||
payload: ReplyPayload,
|
||||
options?: { abortSignal?: AbortSignal; timeoutMs?: number },
|
||||
) => Promise<void> | void;
|
||||
timeoutMs: number;
|
||||
coalescing?: BlockStreamingCoalescing;
|
||||
}): BlockReplyPipeline {
|
||||
const { onBlockReply, timeoutMs, coalescing } = params;
|
||||
const sentKeys = new Set<string>();
|
||||
const pendingKeys = new Set<string>();
|
||||
const seenKeys = new Set<string>();
|
||||
const bufferedKeys = new Set<string>();
|
||||
let sendChain: Promise<void> = Promise.resolve();
|
||||
let aborted = false;
|
||||
let didStream = false;
|
||||
let didLogTimeout = false;
|
||||
|
||||
const sendPayload = (payload: ReplyPayload, skipSeen?: boolean) => {
|
||||
if (aborted) return;
|
||||
const payloadKey = createBlockReplyPayloadKey(payload);
|
||||
if (!skipSeen) {
|
||||
if (seenKeys.has(payloadKey)) return;
|
||||
seenKeys.add(payloadKey);
|
||||
}
|
||||
if (sentKeys.has(payloadKey) || pendingKeys.has(payloadKey)) return;
|
||||
pendingKeys.add(payloadKey);
|
||||
|
||||
const timeoutError = new Error(
|
||||
`block reply delivery timed out after ${timeoutMs}ms`,
|
||||
);
|
||||
const abortController = new AbortController();
|
||||
sendChain = sendChain
|
||||
.then(async () => {
|
||||
if (aborted) return false;
|
||||
await withTimeout(
|
||||
onBlockReply(payload, {
|
||||
abortSignal: abortController.signal,
|
||||
timeoutMs,
|
||||
}) ?? Promise.resolve(),
|
||||
timeoutMs,
|
||||
timeoutError,
|
||||
);
|
||||
return true;
|
||||
})
|
||||
.then((didSend) => {
|
||||
if (!didSend) return;
|
||||
sentKeys.add(payloadKey);
|
||||
didStream = true;
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err === timeoutError) {
|
||||
abortController.abort();
|
||||
aborted = true;
|
||||
if (!didLogTimeout) {
|
||||
didLogTimeout = true;
|
||||
logVerbose(
|
||||
`block reply delivery timed out after ${timeoutMs}ms; skipping remaining block replies to preserve ordering`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
logVerbose(`block reply delivery failed: ${String(err)}`);
|
||||
})
|
||||
.finally(() => {
|
||||
pendingKeys.delete(payloadKey);
|
||||
});
|
||||
};
|
||||
|
||||
const coalescer = coalescing
|
||||
? createBlockReplyCoalescer({
|
||||
config: coalescing,
|
||||
shouldAbort: () => aborted,
|
||||
onFlush: (payload) => {
|
||||
bufferedKeys.clear();
|
||||
sendPayload(payload);
|
||||
},
|
||||
})
|
||||
: null;
|
||||
|
||||
const enqueue = (payload: ReplyPayload) => {
|
||||
if (aborted) return;
|
||||
const hasMedia =
|
||||
Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
|
||||
if (hasMedia) {
|
||||
void coalescer?.flush({ force: true });
|
||||
sendPayload(payload);
|
||||
return;
|
||||
}
|
||||
if (coalescer) {
|
||||
const payloadKey = createBlockReplyPayloadKey(payload);
|
||||
if (
|
||||
seenKeys.has(payloadKey) ||
|
||||
pendingKeys.has(payloadKey) ||
|
||||
bufferedKeys.has(payloadKey)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
bufferedKeys.add(payloadKey);
|
||||
coalescer.enqueue(payload);
|
||||
return;
|
||||
}
|
||||
sendPayload(payload);
|
||||
};
|
||||
|
||||
const flush = async (options?: { force?: boolean }) => {
|
||||
await coalescer?.flush(options);
|
||||
await sendChain;
|
||||
};
|
||||
|
||||
const stop = () => {
|
||||
coalescer?.stop();
|
||||
};
|
||||
|
||||
return {
|
||||
enqueue,
|
||||
flush,
|
||||
stop,
|
||||
hasBuffered: () => Boolean(coalescer?.hasBuffered()),
|
||||
didStream: () => didStream,
|
||||
isAborted: () => aborted,
|
||||
hasSentPayload: (payload) => {
|
||||
const payloadKey = createBlockReplyPayloadKey(payload);
|
||||
return sentKeys.has(payloadKey);
|
||||
},
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue