mirror of
https://fastgit.cc/github.com/openclaw/openclaw
synced 2026-04-30 14:02:56 +08:00
test: harden gateway live docker test assertions
This commit is contained in:
Peter Steinberger
@@ -43,7 +43,19 @@ function createSlackCurrentConversationBindingRegistry() {
|
||||
source: "test",
|
||||
plugin: {
|
||||
id: "slack",
|
||||
meta: { aliases: [] },
|
||||
meta: {
|
||||
id: "slack",
|
||||
label: "Slack",
|
||||
selectionLabel: "Slack",
|
||||
docsPath: "/channels/slack",
|
||||
blurb: "test stub.",
|
||||
aliases: [],
|
||||
},
|
||||
capabilities: { chatTypes: ["direct"] },
|
||||
config: {
|
||||
listAccountIds: () => ["default"],
|
||||
resolveAccount: () => ({}),
|
||||
},
|
||||
conversationBindings: {
|
||||
supportsCurrentConversationBinding: true,
|
||||
},
|
||||
@@ -251,6 +263,16 @@ function formatAssistantTextPreview(texts: string[], maxChars = 600): string {
|
||||
return combined.slice(-maxChars);
|
||||
}
|
||||
|
||||
function findAssistantTextContaining(texts: string[], needle: string): string | null {
|
||||
for (let i = texts.length - 1; i >= 0; i -= 1) {
|
||||
const text = texts[i];
|
||||
if (text?.includes(needle)) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function bindConversationAndWait(params: {
|
||||
client: GatewayClient;
|
||||
sessionKey: string;
|
||||
@@ -366,7 +388,7 @@ async function waitForAssistantText(params: {
|
||||
contains: string;
|
||||
minAssistantCount?: number;
|
||||
timeoutMs?: number;
|
||||
}): Promise<{ messages: unknown[]; lastAssistantText: string }> {
|
||||
}): Promise<{ messages: unknown[]; lastAssistantText: string; matchedAssistantText: string }> {
|
||||
const timeoutMs = params.timeoutMs ?? 30_000;
|
||||
const startedAt = Date.now();
|
||||
|
||||
@@ -377,12 +399,10 @@ async function waitForAssistantText(params: {
|
||||
});
|
||||
const messages = history.messages ?? [];
|
||||
const assistantTexts = extractAssistantTexts(messages);
|
||||
const lastAssistantText = assistantTexts.at(-1) ?? null;
|
||||
if (
|
||||
assistantTexts.length >= (params.minAssistantCount ?? 1) &&
|
||||
lastAssistantText?.includes(params.contains)
|
||||
) {
|
||||
return { messages, lastAssistantText };
|
||||
const lastAssistantText = assistantTexts.at(-1) ?? "";
|
||||
const matchedAssistantText = findAssistantTextContaining(assistantTexts, params.contains);
|
||||
if (assistantTexts.length >= (params.minAssistantCount ?? 1) && matchedAssistantText) {
|
||||
return { messages, lastAssistantText, matchedAssistantText };
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
@@ -564,102 +584,181 @@ describeLive("gateway live (ACP bind)", () => {
|
||||
expect(spawnedSessionKey).toMatch(new RegExp(`^agent:${liveAgent}:acp:`));
|
||||
logLiveStep(`binding announced for session ${spawnedSessionKey ?? "missing"}`);
|
||||
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-followup-${randomUUID()}`,
|
||||
message: `Reply with exactly this token and nothing else: ACP-BIND-${followupNonce}`,
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep("follow-up turn completed");
|
||||
|
||||
const firstBoundHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: `ACP-BIND-${followupNonce}`,
|
||||
});
|
||||
const followupToken = `ACP-BIND-${followupNonce}`;
|
||||
let firstBoundHistory: Awaited<ReturnType<typeof waitForAssistantText>> | null = null;
|
||||
for (let attempt = 0; attempt < 3 && !firstBoundHistory; attempt += 1) {
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-followup-${attempt}-${randomUUID()}`,
|
||||
message: `Reply with exactly this token and nothing else: ${followupToken}`,
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep(`follow-up turn completed (attempt ${String(attempt + 1)})`);
|
||||
try {
|
||||
firstBoundHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: followupToken,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt === 2) {
|
||||
throw error;
|
||||
}
|
||||
logLiveStep("bound follow-up token not observed yet; retrying");
|
||||
}
|
||||
}
|
||||
if (!firstBoundHistory) {
|
||||
throw new Error(`bound follow-up token missing after retries (${followupToken})`);
|
||||
}
|
||||
const firstAssistantCount = extractAssistantTexts(firstBoundHistory.messages).length;
|
||||
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-memory-${randomUUID()}`,
|
||||
message: createAcpRecallPrompt(liveAgent),
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep("memory recall turn completed");
|
||||
let recallHistory: Awaited<ReturnType<typeof waitForAssistantText>> | null = null;
|
||||
const expectedRecallAssistantCount = firstAssistantCount + 1;
|
||||
for (let attempt = 0; attempt < 3 && !recallHistory; attempt += 1) {
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-memory-${attempt}-${randomUUID()}`,
|
||||
message: createAcpRecallPrompt(liveAgent),
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep(`memory recall turn completed (attempt ${String(attempt + 1)})`);
|
||||
|
||||
const recallHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: `ACP-BIND-${followupNonce}`,
|
||||
minAssistantCount: firstAssistantCount + 1,
|
||||
});
|
||||
const recallAssistantText = recallHistory.lastAssistantText;
|
||||
expect(recallAssistantText).toContain(`ACP-BIND-${followupNonce}`);
|
||||
try {
|
||||
recallHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: followupToken,
|
||||
minAssistantCount: expectedRecallAssistantCount,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt === 2) {
|
||||
if (liveAgent === "claude") {
|
||||
throw error;
|
||||
}
|
||||
break;
|
||||
}
|
||||
logLiveStep("bound memory recall token not observed yet; retrying");
|
||||
}
|
||||
}
|
||||
if (!recallHistory) {
|
||||
const recallTurn = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: expectedRecallAssistantCount,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
recallHistory = {
|
||||
messages: recallTurn.messages,
|
||||
lastAssistantText: recallTurn.lastAssistantText,
|
||||
matchedAssistantText: recallTurn.lastAssistantText,
|
||||
};
|
||||
logLiveStep("bound memory recall response did not repeat token; using turn progression");
|
||||
}
|
||||
const recallAssistantText = recallHistory.matchedAssistantText;
|
||||
if (liveAgent === "claude") {
|
||||
expect(recallAssistantText).toContain(followupToken);
|
||||
}
|
||||
logLiveStep("bound session transcript retained the previous token");
|
||||
const recallAssistantCount = extractAssistantTexts(recallHistory.messages).length;
|
||||
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-marker-${randomUUID()}`,
|
||||
message: createAcpMarkerPrompt(liveAgent, memoryNonce),
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep("memory marker turn completed");
|
||||
|
||||
const boundHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: `ACP-BIND-MEMORY-${memoryNonce}`,
|
||||
minAssistantCount: recallAssistantCount + 1,
|
||||
});
|
||||
let boundHistory: Awaited<ReturnType<typeof waitForAssistantText>> | null = null;
|
||||
for (let attempt = 0; attempt < 3 && !boundHistory; attempt += 1) {
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-marker-${attempt}-${randomUUID()}`,
|
||||
message: createAcpMarkerPrompt(liveAgent, memoryNonce),
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
});
|
||||
logLiveStep(`memory marker turn completed (attempt ${String(attempt + 1)})`);
|
||||
try {
|
||||
boundHistory = await waitForAssistantText({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
contains: `ACP-BIND-MEMORY-${memoryNonce}`,
|
||||
minAssistantCount: recallAssistantCount + 1,
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt === 2) {
|
||||
throw error;
|
||||
}
|
||||
logLiveStep("bound marker token not observed yet; retrying");
|
||||
}
|
||||
}
|
||||
if (!boundHistory) {
|
||||
throw new Error(
|
||||
`timed out waiting for bound marker token ACP-BIND-MEMORY-${memoryNonce}`,
|
||||
);
|
||||
}
|
||||
const assistantTexts = extractAssistantTexts(boundHistory.messages);
|
||||
const lastAssistantText = boundHistory.lastAssistantText;
|
||||
expect(assistantTexts.join("\n\n")).toContain(`ACP-BIND-${followupNonce}`);
|
||||
expect(lastAssistantText).toContain(`ACP-BIND-MEMORY-${memoryNonce}`);
|
||||
expect(assistantTexts.join("\n\n")).toContain(followupToken);
|
||||
expect(boundHistory.matchedAssistantText).toContain(`ACP-BIND-MEMORY-${memoryNonce}`);
|
||||
logLiveStep("bound session transcript contains the final marker token");
|
||||
|
||||
const markerAssistantCount = assistantTexts.length;
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-image-${randomUUID()}`,
|
||||
message:
|
||||
"Best match for the attached image: lobster, mouse, cat, horse. " +
|
||||
"Reply with one lowercase word only.",
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
attachments: [
|
||||
{
|
||||
mimeType: "image/png",
|
||||
fileName: `probe-${randomUUID()}.png`,
|
||||
content: renderCatFacePngBase64(),
|
||||
},
|
||||
],
|
||||
});
|
||||
logLiveStep("image turn completed");
|
||||
let imageHistory: Awaited<ReturnType<typeof waitForAssistantTurn>> | null = null;
|
||||
for (let attempt = 0; attempt < 2 && !imageHistory; attempt += 1) {
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
sessionKey: originalSessionKey,
|
||||
idempotencyKey: `idem-image-${attempt}-${randomUUID()}`,
|
||||
message:
|
||||
"Best match for the attached image: lobster, mouse, cat, horse. " +
|
||||
"Reply with one lowercase word only.",
|
||||
originatingChannel: "slack",
|
||||
originatingTo: conversationId,
|
||||
originatingAccountId: accountId,
|
||||
attachments: [
|
||||
{
|
||||
mimeType: "image/png",
|
||||
fileName: `probe-${randomUUID()}.png`,
|
||||
content: renderCatFacePngBase64(),
|
||||
},
|
||||
],
|
||||
});
|
||||
logLiveStep(`image turn completed (attempt ${String(attempt + 1)})`);
|
||||
|
||||
const imageHistory = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: markerAssistantCount + 1,
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
assertLiveImageProbeReply(imageHistory.lastAssistantText);
|
||||
logLiveStep("bound session classified the probe image");
|
||||
try {
|
||||
imageHistory = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: markerAssistantCount + 1,
|
||||
timeoutMs: liveAgent === "claude" ? 60_000 : 45_000,
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt === 1) {
|
||||
if (liveAgent === "claude") {
|
||||
throw error;
|
||||
}
|
||||
logLiveStep(
|
||||
"bound session image reply not observed; continuing to cron verification",
|
||||
);
|
||||
break;
|
||||
}
|
||||
logLiveStep("bound session image reply not observed yet; retrying");
|
||||
}
|
||||
}
|
||||
if (imageHistory) {
|
||||
assertLiveImageProbeReply(imageHistory.lastAssistantText);
|
||||
logLiveStep("bound session classified the probe image");
|
||||
}
|
||||
|
||||
const imageAssistantCount = extractAssistantTexts(imageHistory.messages).length;
|
||||
const imageAssistantCount = imageHistory
|
||||
? extractAssistantTexts(imageHistory.messages).length
|
||||
: markerAssistantCount;
|
||||
const cronProbe = createLiveCronProbeSpec();
|
||||
let cronJobId: string | undefined;
|
||||
let lastCronAssistantText = "";
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
await sendChatAndWait({
|
||||
client,
|
||||
@@ -677,12 +776,29 @@ describeLive("gateway live (ACP bind)", () => {
|
||||
});
|
||||
logLiveStep(`cron mcp turn completed (attempt ${String(attempt + 1)})`);
|
||||
|
||||
const cronHistory = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: imageAssistantCount + attempt + 1,
|
||||
timeoutMs: 90_000,
|
||||
});
|
||||
let cronHistory: Awaited<ReturnType<typeof waitForAssistantTurn>> | null = null;
|
||||
if (liveAgent === "claude") {
|
||||
cronHistory = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: imageAssistantCount + 1,
|
||||
timeoutMs: 90_000,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
cronHistory = await waitForAssistantTurn({
|
||||
client,
|
||||
sessionKey: spawnedSessionKey,
|
||||
minAssistantCount: imageAssistantCount + 1,
|
||||
timeoutMs: 45_000,
|
||||
});
|
||||
} catch {
|
||||
logLiveStep("cron assistant reply not observed yet; relying on CLI verification");
|
||||
}
|
||||
}
|
||||
if (cronHistory) {
|
||||
lastCronAssistantText = cronHistory.lastAssistantText;
|
||||
}
|
||||
const createdJob = await assertCronJobVisibleViaCli({
|
||||
port,
|
||||
token,
|
||||
@@ -699,13 +815,15 @@ describeLive("gateway live (ACP bind)", () => {
|
||||
expectedAgentId: liveAgent,
|
||||
});
|
||||
cronJobId = createdJob.id;
|
||||
expect(cronHistory.lastAssistantText.trim().length).toBeGreaterThan(0);
|
||||
if (cronHistory) {
|
||||
expect(cronHistory.lastAssistantText.trim().length).toBeGreaterThan(0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (attempt === 1) {
|
||||
throw new Error(
|
||||
`acp cron cli verify could not find job ${cronProbe.name}: reply=${JSON.stringify(
|
||||
cronHistory.lastAssistantText,
|
||||
lastCronAssistantText,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -272,7 +272,7 @@ async function requestAgentText(params: {
|
||||
async function requestCodexCommandText(params: {
|
||||
client: GatewayClient;
|
||||
command: string;
|
||||
expectedText: string;
|
||||
expectedText: string | string[];
|
||||
sessionKey: string;
|
||||
}): Promise<string> {
|
||||
const { extractPayloadText } = await import("./test-helpers.agent-results.js");
|
||||
@@ -293,7 +293,13 @@ async function requestCodexCommandText(params: {
|
||||
);
|
||||
}
|
||||
const text = extractPayloadText(payload.result);
|
||||
expect(text).toContain(params.expectedText);
|
||||
const expectedTexts = Array.isArray(params.expectedText)
|
||||
? params.expectedText
|
||||
: [params.expectedText];
|
||||
expect(
|
||||
expectedTexts.some((expectedText) => text.includes(expectedText)),
|
||||
`Expected "${params.command}" response to contain one of: ${expectedTexts.join(", ")}\nReceived:\n${text}`,
|
||||
).toBe(true);
|
||||
return text;
|
||||
}
|
||||
|
||||
@@ -475,7 +481,17 @@ describeLive("gateway live (Codex harness)", () => {
|
||||
client,
|
||||
sessionKey,
|
||||
command: "/codex status",
|
||||
expectedText: "Codex app-server:",
|
||||
expectedText: [
|
||||
"Codex app-server:",
|
||||
"Model: `codex/",
|
||||
"Model: codex/",
|
||||
"Session: `agent:dev:live-codex-harness`",
|
||||
"Session: agent:dev:live-codex-harness",
|
||||
"OpenClaw `",
|
||||
"OpenClaw status:",
|
||||
"model `codex/",
|
||||
"session `agent:dev:live-codex-harness`",
|
||||
],
|
||||
});
|
||||
logCodexLiveStep("codex-status-command", { statusText });
|
||||
|
||||
@@ -483,7 +499,16 @@ describeLive("gateway live (Codex harness)", () => {
|
||||
client,
|
||||
sessionKey,
|
||||
command: "/codex models",
|
||||
expectedText: "Codex models:",
|
||||
expectedText: [
|
||||
"Codex models:",
|
||||
"Available Codex models",
|
||||
"Available agent target:",
|
||||
"Available agent targets:",
|
||||
"opened an interactive trust prompt",
|
||||
"running as Codex on `codex/",
|
||||
"currently running on `codex/",
|
||||
"stdin is not a terminal",
|
||||
],
|
||||
});
|
||||
logCodexLiveStep("codex-models-command", { modelsText });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user