diff --git a/CHANGELOG.md b/CHANGELOG.md index 8645a0cbfee..d943ef03727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Docs: https://docs.openclaw.ai - Channels/Telegram: continue polling when `deleteWebhook` hits a transient network failure but `getWebhookInfo` confirms no webhook is configured, so startup does not retry cleanup forever after the webhook was already removed. Refs #74086; carries forward #47384. Thanks @clovericbot. - Channels/Telegram: apply strict safe-send retry to inbound final replies when grammY wraps a pre-connect failure, while leaving ambiguous plain network envelopes single-shot to avoid duplicate visible messages. Fixes #74203. Thanks @nanli2000cn. - Channels/Telegram: surface polling liveness warnings in channel status and doctor when a running long-poller has not completed `getUpdates` after startup grace or its transport activity is stale, so silent polling failures no longer look clean. Refs #74299. Thanks @lolaopenclaw. +- Channels/Telegram: publish webhook runtime state and warn when `setWebhook` has not completed after startup grace, so webhook-mode accounts no longer look healthy while registration is still failing or retrying. Refs #74299. Thanks @lolaopenclaw and @martingarramon. - Channels/Telegram: bound native command menu `deleteMyCommands` and `setMyCommands` Bot API calls and allow the same timeout-triggered transport fallback retry as other startup control calls, so Windows/WSL network stalls cannot leave command sync hanging behind an otherwise running provider. Refs #74086. Thanks @SymbolStar. - ACP/commands: accept forwarded ACP timeout config controls in the OpenClaw bridge, treat unsupported discard-close controls as recoverable cleanup, and restore native `/verbose full` plus no-arg status behavior, so Discord command menus and nested ACP turns no longer fail on supported session controls. Thanks @vincentkoc. - TUI/status: clear stale `streaming` footer state when a final event arrives after the active run was already cleared and no tracked runs remain, while preserving concurrent-run ownership and inactive local `/btw` terminal handling. Fixes #64825; carries forward #64842, #64843, #64847, and #64862. Thanks @briandevans and @Yanhu007. diff --git a/docs/channels/telegram.md b/docs/channels/telegram.md index 5419a1ac261..76849fe7a47 100644 --- a/docs/channels/telegram.md +++ b/docs/channels/telegram.md @@ -867,7 +867,7 @@ Per-account, per-group, and per-topic overrides are supported (same inheritance - Some hosts resolve `api.telegram.org` to IPv6 first; broken IPv6 egress can cause intermittent Telegram API failures. - If logs include `TypeError: fetch failed` or `Network request for 'getUpdates' failed!`, OpenClaw now retries these as recoverable network errors. - If logs include `Polling stall detected`, OpenClaw restarts polling and rebuilds the Telegram transport after 120 seconds without completed long-poll liveness by default. - - `openclaw channels status --probe` and `openclaw doctor` warn when a running polling account has not completed `getUpdates` after startup grace, or when its last successful polling transport activity is stale. + - `openclaw channels status --probe` and `openclaw doctor` warn when a running polling account has not completed `getUpdates` after startup grace, when a running webhook account has not completed `setWebhook` after startup grace, or when the last successful polling transport activity is stale. - Increase `channels.telegram.pollingStallThresholdMs` only when long-running `getUpdates` calls are healthy but your host still reports false polling-stall restarts. Persistent stalls usually point to proxy, DNS, IPv6, or TLS egress issues between the host and `api.telegram.org`. - Telegram also honors process proxy env for Bot API transport, including `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and their lowercase variants. `NO_PROXY` / `no_proxy` can still bypass `api.telegram.org`. - If the OpenClaw managed proxy is configured through `OPENCLAW_PROXY_URL` for a service environment and no standard proxy env is present, Telegram uses that URL for Bot API transport too. diff --git a/extensions/telegram/src/monitor.test.ts b/extensions/telegram/src/monitor.test.ts index fdd4ff33812..4674cb2c30d 100644 --- a/extensions/telegram/src/monitor.test.ts +++ b/extensions/telegram/src/monitor.test.ts @@ -774,11 +774,13 @@ describe("monitorTelegramProvider (grammY)", () => { }); it("passes configured webhookHost to webhook listener", async () => { + const setStatus = vi.fn(); await monitorTelegramProvider({ token: "tok", useWebhook: true, webhookUrl: "https://example.test/telegram", webhookSecret: "secret", + setStatus, config: { agents: { defaults: { maxConcurrent: 2 } }, channels: { @@ -792,6 +794,7 @@ describe("monitorTelegramProvider (grammY)", () => { expect(startTelegramWebhookSpy).toHaveBeenCalledWith( expect.objectContaining({ host: "0.0.0.0", + setStatus, }), ); expect(runSpy).not.toHaveBeenCalled(); diff --git a/extensions/telegram/src/monitor.ts b/extensions/telegram/src/monitor.ts index 459f9b2976e..00d7ed97d6c 100644 --- a/extensions/telegram/src/monitor.ts +++ b/extensions/telegram/src/monitor.ts @@ -154,6 +154,7 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) { abortSignal: opts.abortSignal, publicUrl: opts.webhookUrl, webhookCertPath: opts.webhookCertPath, + setStatus: opts.setStatus, }); await waitForAbortSignal(opts.abortSignal); return; diff --git a/extensions/telegram/src/status-issues.ts b/extensions/telegram/src/status-issues.ts index 9e4905e90ce..4d9b41f575e 100644 --- a/extensions/telegram/src/status-issues.ts +++ b/extensions/telegram/src/status-issues.ts @@ -12,6 +12,7 @@ import { const TELEGRAM_POLLING_CONNECT_GRACE_MS = 120_000; const TELEGRAM_POLLING_STALE_TRANSPORT_MS = 30 * 60_000; +const TELEGRAM_WEBHOOK_CONNECT_GRACE_MS = 120_000; type TelegramAccountStatus = { accountId?: unknown; @@ -124,6 +125,40 @@ function collectTelegramPollingRuntimeIssues(params: { } } +function collectTelegramWebhookRuntimeIssues(params: { + account: TelegramAccountStatus; + accountId: string; + issues: ChannelStatusIssue[]; + now: number; +}) { + const { account, accountId, issues, now } = params; + if (account.running !== true || asString(account.mode) !== "webhook") { + return; + } + + if (account.connected !== false) { + return; + } + + const lastStartAt = asFiniteNumber(account.lastStartAt); + const withinStartupGrace = + lastStartAt != null && now - lastStartAt < TELEGRAM_WEBHOOK_CONNECT_GRACE_MS; + if (withinStartupGrace) { + return; + } + + issues.push({ + channel: "telegram", + accountId, + kind: "runtime", + message: appendTelegramRuntimeError( + "Telegram webhook listener is running but setWebhook has not completed since startup", + account.lastError, + ), + fix: `Run: ${formatCliCommand("openclaw channels status --probe")} (or restart the gateway). Check the webhook URL, secret, TLS/proxy reachability, and Telegram setWebhook logs if it persists.`, + }); +} + function readTelegramGroupMembershipAuditSummary( value: unknown, ): TelegramGroupMembershipAuditSummary { @@ -174,12 +209,19 @@ export function collectTelegramStatusIssues( if (!accountId) { continue; } + const now = Date.now(); collectTelegramPollingRuntimeIssues({ account, accountId, issues, - now: Date.now(), + now, + }); + collectTelegramWebhookRuntimeIssues({ + account, + accountId, + issues, + now, }); if (account.allowUnmentionedGroups === true) { diff --git a/extensions/telegram/src/status.test.ts b/extensions/telegram/src/status.test.ts index 1e4bf1b4ebf..f212417d96a 100644 --- a/extensions/telegram/src/status.test.ts +++ b/extensions/telegram/src/status.test.ts @@ -156,7 +156,7 @@ describe("collectTelegramStatusIssues", () => { expect(issues).toEqual([]); }); - it("does not report webhook accounts with polling-only runtime diagnostics", () => { + it("reports webhook runtime state that never completed setWebhook after startup grace", () => { const issues = collectTelegramStatusIssues([ { accountId: "main", @@ -166,6 +166,48 @@ describe("collectTelegramStatusIssues", () => { mode: "webhook", connected: false, lastStartAt: Date.now() - 10 * 60_000, + lastError: "fetch failed", + } as ChannelAccountSnapshot, + ]); + + expect(issues).toEqual([ + expect.objectContaining({ + channel: "telegram", + accountId: "main", + kind: "runtime", + message: expect.stringContaining("setWebhook has not completed"), + }), + ]); + expect(issues[0]?.message).toContain("fetch failed"); + expect(issues[0]?.fix).toContain("webhook URL"); + }); + + it("does not report webhook startup before the connect grace expires", () => { + const issues = collectTelegramStatusIssues([ + { + accountId: "main", + enabled: true, + configured: true, + running: true, + mode: "webhook", + connected: false, + lastStartAt: Date.now() - 60_000, + } as ChannelAccountSnapshot, + ]); + + expect(issues).toEqual([]); + }); + + it("does not report an advertised webhook just because no user updates arrived", () => { + const issues = collectTelegramStatusIssues([ + { + accountId: "main", + enabled: true, + configured: true, + running: true, + mode: "webhook", + connected: true, + lastStartAt: Date.now() - 60 * 60_000, } as ChannelAccountSnapshot, ]); diff --git a/extensions/telegram/src/webhook-status.test.ts b/extensions/telegram/src/webhook-status.test.ts new file mode 100644 index 00000000000..2444c31896d --- /dev/null +++ b/extensions/telegram/src/webhook-status.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import { createTelegramWebhookStatusPublisher } from "./webhook-status.js"; + +describe("createTelegramWebhookStatusPublisher", () => { + it("publishes start, advertised, update, failure, and stop status patches", () => { + const setStatus = vi.fn(); + const status = createTelegramWebhookStatusPublisher(setStatus); + + status.noteWebhookStart(); + status.noteWebhookAdvertised(1234); + status.noteWebhookUpdateReceived(2345); + status.noteWebhookRegistrationFailure("fetch failed"); + status.noteWebhookStop(); + + expect(setStatus).toHaveBeenNthCalledWith(1, { + mode: "webhook", + connected: false, + lastConnectedAt: null, + lastEventAt: null, + lastTransportActivityAt: null, + }); + expect(setStatus).toHaveBeenNthCalledWith(2, { + mode: "webhook", + connected: true, + lastConnectedAt: 1234, + lastEventAt: 1234, + lastError: null, + }); + expect(setStatus).toHaveBeenNthCalledWith(3, { + mode: "webhook", + connected: true, + lastConnectedAt: 2345, + lastEventAt: 2345, + lastError: null, + }); + expect(setStatus).toHaveBeenNthCalledWith(4, { + mode: "webhook", + connected: false, + lastError: "fetch failed", + }); + expect(setStatus).toHaveBeenNthCalledWith(5, { + mode: "webhook", + connected: false, + }); + }); +}); diff --git a/extensions/telegram/src/webhook-status.ts b/extensions/telegram/src/webhook-status.ts new file mode 100644 index 00000000000..cee2a58b603 --- /dev/null +++ b/extensions/telegram/src/webhook-status.ts @@ -0,0 +1,45 @@ +import type { ChannelAccountSnapshot } from "openclaw/plugin-sdk/channel-contract"; +import { createConnectedChannelStatusPatch } from "openclaw/plugin-sdk/gateway-runtime"; + +type TelegramWebhookStatusSink = (patch: Omit) => void; + +export function createTelegramWebhookStatusPublisher(setStatus?: TelegramWebhookStatusSink) { + return { + noteWebhookStart() { + setStatus?.({ + mode: "webhook", + connected: false, + lastConnectedAt: null, + lastEventAt: null, + lastTransportActivityAt: null, + }); + }, + noteWebhookAdvertised(at = Date.now()) { + setStatus?.({ + ...createConnectedChannelStatusPatch(at), + mode: "webhook", + lastError: null, + }); + }, + noteWebhookUpdateReceived(at = Date.now()) { + setStatus?.({ + ...createConnectedChannelStatusPatch(at), + mode: "webhook", + lastError: null, + }); + }, + noteWebhookRegistrationFailure(error: string) { + setStatus?.({ + mode: "webhook", + connected: false, + lastError: error, + }); + }, + noteWebhookStop() { + setStatus?.({ + mode: "webhook", + connected: false, + }); + }, + }; +} diff --git a/extensions/telegram/src/webhook.test.ts b/extensions/telegram/src/webhook.test.ts index 600955fa3a8..fba019a0978 100644 --- a/extensions/telegram/src/webhook.test.ts +++ b/extensions/telegram/src/webhook.test.ts @@ -412,6 +412,7 @@ describe("startTelegramWebhook", () => { initSpy.mockClear(); createTelegramBotSpy.mockClear(); const runtimeLog = vi.fn(); + const setStatus = vi.fn(); const cfg = { bindings: [] }; await withStartedWebhook( { @@ -419,6 +420,7 @@ describe("startTelegramWebhook", () => { accountId: "opie", config: cfg, runtime: { log: runtimeLog, error: vi.fn(), exit: vi.fn() }, + setStatus, }, async ({ port }) => { expect(createTelegramBotSpy).toHaveBeenCalledWith( @@ -438,6 +440,20 @@ describe("startTelegramWebhook", () => { expect(runtimeLog).toHaveBeenCalledWith( expect.stringContaining("webhook advertised to telegram on http://"), ); + expect(setStatus).toHaveBeenNthCalledWith(1, { + mode: "webhook", + connected: false, + lastConnectedAt: null, + lastEventAt: null, + lastTransportActivityAt: null, + }); + expect(setStatus).toHaveBeenNthCalledWith(2, { + mode: "webhook", + connected: true, + lastConnectedAt: expect.any(Number), + lastEventAt: expect.any(Number), + lastError: null, + }); }, ); }); @@ -445,6 +461,7 @@ describe("startTelegramWebhook", () => { it("keeps local listener alive and retries when setWebhook has a recoverable startup failure", async () => { const runtimeLog = vi.fn(); const runtimeError = vi.fn(); + const setStatus = vi.fn(); setWebhookSpy.mockRejectedValueOnce(new TypeError("fetch failed")).mockResolvedValueOnce(true); await withStartedWebhook( @@ -452,6 +469,7 @@ describe("startTelegramWebhook", () => { secret: TELEGRAM_SECRET, path: TELEGRAM_WEBHOOK_PATH, runtime: { log: runtimeLog, error: runtimeError, exit: vi.fn() }, + setStatus, webhookRegistrationRetryPolicy: { initialMs: 0, maxMs: 0, @@ -471,6 +489,18 @@ describe("startTelegramWebhook", () => { expect(runtimeLog).toHaveBeenCalledWith( expect.stringContaining("webhook advertised to telegram on http://"), ); + expect(setStatus).toHaveBeenCalledWith({ + mode: "webhook", + connected: false, + lastError: "fetch failed", + }); + expect(setStatus).toHaveBeenCalledWith( + expect.objectContaining({ + mode: "webhook", + connected: true, + lastError: null, + }), + ); }, ); }); @@ -532,6 +562,7 @@ describe("startTelegramWebhook", () => { it("invokes webhook handler on matching path", async () => { handleUpdateSpy.mockClear(); createTelegramBotSpy.mockClear(); + const setStatus = vi.fn(); const cfg = { bindings: [] }; await withStartedWebhook( { @@ -539,6 +570,7 @@ describe("startTelegramWebhook", () => { accountId: "opie", config: cfg, path: TELEGRAM_WEBHOOK_PATH, + setStatus, }, async ({ port }) => { expect(createTelegramBotSpy).toHaveBeenCalledWith( @@ -555,6 +587,13 @@ describe("startTelegramWebhook", () => { }); expect(response.status).toBe(200); await vi.waitFor(() => expect(handleUpdateSpy).toHaveBeenCalledWith(JSON.parse(payload))); + expect(setStatus).toHaveBeenCalledWith( + expect.objectContaining({ + mode: "webhook", + connected: true, + lastError: null, + }), + ); }, ); }); diff --git a/extensions/telegram/src/webhook.ts b/extensions/telegram/src/webhook.ts index 00502d38531..3d59acb90ec 100644 --- a/extensions/telegram/src/webhook.ts +++ b/extensions/telegram/src/webhook.ts @@ -2,6 +2,7 @@ import { createServer } from "node:http"; import type { IncomingMessage } from "node:http"; import net from "node:net"; import * as grammy from "grammy"; +import type { ChannelAccountSnapshot } from "openclaw/plugin-sdk/channel-contract"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; import { isDiagnosticsEnabled } from "openclaw/plugin-sdk/diagnostic-runtime"; import type { BackoffPolicy, RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; @@ -35,6 +36,7 @@ import { isTelegramRateLimitError, isTelegramServerError, } from "./network-errors.js"; +import { createTelegramWebhookStatusPublisher } from "./webhook-status.js"; const TELEGRAM_WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; const TELEGRAM_WEBHOOK_BODY_TIMEOUT_MS = 30_000; @@ -262,6 +264,7 @@ export async function startTelegramWebhook(opts: { publicUrl?: string; webhookCertPath?: string; webhookRegistrationRetryPolicy?: BackoffPolicy; + setStatus?: (patch: Omit) => void; }) { const path = opts.path ?? "/telegram-webhook"; const healthPath = opts.healthPath ?? "/healthz"; @@ -275,6 +278,8 @@ export async function startTelegramWebhook(opts: { ); } const runtime = opts.runtime ?? defaultRuntime; + const status = createTelegramWebhookStatusPublisher(opts.setStatus); + status.noteWebhookStart(); const webhookRegistrationRetryPolicy = opts.webhookRegistrationRetryPolicy ?? TELEGRAM_WEBHOOK_REGISTRATION_RETRY_POLICY; const diagnosticsEnabled = isDiagnosticsEnabled(opts.config); @@ -365,6 +370,7 @@ export async function startTelegramWebhook(opts: { } respondText(200); + status.noteWebhookUpdateReceived(); void (async () => { await bot.handleUpdate(body.value as Parameters[0]); @@ -433,6 +439,7 @@ export async function startTelegramWebhook(opts: { }); server.close(); void bot.stop(); + status.noteWebhookStop(); if (diagnosticsEnabled) { stopDiagnosticHeartbeat(); } @@ -447,20 +454,26 @@ export async function startTelegramWebhook(opts: { if (shutDown || opts.abortSignal?.aborted) { return; } - await withTelegramApiErrorLogging({ - operation: "setWebhook", - runtime, - fn: () => - bot.api.setWebhook(publicUrl, { - secret_token: secret, - allowed_updates: resolveTelegramAllowedUpdates(), - certificate: opts.webhookCertPath ? new InputFileCtor(opts.webhookCertPath) : undefined, - }), - }); + try { + await withTelegramApiErrorLogging({ + operation: "setWebhook", + runtime, + fn: () => + bot.api.setWebhook(publicUrl, { + secret_token: secret, + allowed_updates: resolveTelegramAllowedUpdates(), + certificate: opts.webhookCertPath ? new InputFileCtor(opts.webhookCertPath) : undefined, + }), + }); + } catch (err) { + status.noteWebhookRegistrationFailure(formatErrorMessage(err)); + throw err; + } if (shutDown) { return; } webhookAdvertised = true; + status.noteWebhookAdvertised(); runtime.log?.(`webhook advertised to telegram on ${publicUrl}`); }; const shouldRetryWebhookRegistration = (err: unknown): boolean => @@ -503,6 +516,7 @@ export async function startTelegramWebhook(opts: { shutDown = true; server.close(); void bot.stop(); + status.noteWebhookStop(); if (diagnosticsEnabled) { stopDiagnosticHeartbeat(); } diff --git a/src/gateway/channel-health-policy.test.ts b/src/gateway/channel-health-policy.test.ts index daec553d2d8..35ed67ab7ce 100644 --- a/src/gateway/channel-health-policy.test.ts +++ b/src/gateway/channel-health-policy.test.ts @@ -223,6 +223,28 @@ describe("evaluateChannelHealth", () => { expect(evaluation).toEqual({ healthy: true, reason: "healthy" }); }); + it("keeps quiet telegram webhooks healthy when they do not publish transport tracking", () => { + const evaluation = evaluateChannelHealth( + { + running: true, + connected: true, + enabled: true, + configured: true, + mode: "webhook", + lastStartAt: 0, + lastConnectedAt: 0, + lastEventAt: 0, + }, + { + channelId: "telegram", + now: 100_000, + channelConnectGraceMs: 10_000, + staleEventThresholdMs: 30_000, + }, + ); + expect(evaluation).toEqual({ healthy: true, reason: "healthy" }); + }); + it("does not flag stale sockets without an active connected socket", () => { const evaluation = evaluateDiscordHealth( {