Deliveries on master 150 what went out for master · all time · only master, last 1 hour · for you (0 unread)
Every message X-ray produced for an alert — delivered in X-ray, sent to a channel, rendered in dry-run, held or dropped — and why. One row per message: a group of a hundred findings is one row. Every row carries its message number (#N), the same one printed on the card and the bell row — click it to see the message on its alert's page, with what it was about. external channels dry-run
| When | Alert | Message | Delivered to | Event | Why |
|---|---|---|---|---|---|
| #52143 2026-11-17 10:47:59 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=ccbb0e14e729833398b26f30b0353c8a&at=1794912478&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "ccbb0e14e729833398b26f30b0353c8a",
"log": "stdout",
"ts": "2026-11-17T10:47:58.466445485Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912509,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52143,
"message_url": "https://xray.tilda.pizza/alerts/messages/52143"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52141 2026-11-17 10:46:09 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.326429667Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912399,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794912369
},
"message_id": 52141,
"message_url": "/alerts/messages/52141"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52139 2026-11-17 10:46:08 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nError retrieving CRF document",
"message": "Error retrieving CRF document",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+retrieving+CRF+document",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error retrieving CRF document",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.326163011Z"
},
"log_key": "f245e71b17cb74fb7e8ca75222d2b33f9337d8eff5a8f6754a79cb6dd1d6f98a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912399,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52139,
"message_url": "https://xray.tilda.pizza/alerts/messages/52139"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52137 2026-11-17 10:46:08 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nError downloading file",
"message": "Error downloading file",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+downloading+file",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error downloading file",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.316985588Z"
},
"log_key": "f7a5008a2692e7f6c2d00abaee03d4c4daac7424425a874c1807a48ac7d2aeef",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912398,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52137,
"message_url": "https://xray.tilda.pizza/alerts/messages/52137"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52135 2026-11-17 10:46:03 UTC |
channel test | warning Error in exportsvc (tilda) on master
show the payload{
"title": "Error in exportsvc (tilda) on master",
"severity": "warning",
"headline": "Error in exportsvc (tilda) on master",
"text": "Error in exportsvc (tilda) on master\nError processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"message": "Error processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=exportsvc&namespace=tilda&trace=71a8ac8646af2c3e14e5a5380c42696c&at=1794912357&msg=Error+processing+activity+update%3A+Failed+to+fetch+study+metadata%3A+API+request+failed%3A+500+Server+Error%3A+Internal+Server+Error+for+url%3A+https%3A%2F%2Fapi.master.tilda.pizza%2Fapi%2Ftrialexec%2Fv2%2Fstudy%2F9cfcafa0-9782-4448-8250-ae8288b92a5a%2Fprotocol%2Ff4f461b2-29a8-45a0-a114-a27762fac91b%2Fcrf",
"log": {
"severity": "ERROR",
"service": "exportsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Error processing activity update: Failed to fetch study metadata: API request failed: 500 Server Error: Internal Server Error for url: https://api.master.tilda.pizza/api/trialexec/v2/study/9cfcafa0-9782-4448-8250-ae8288b92a5a/protocol/f4f461b2-29a8-45a0-a114-a27762fac91b/crf",
"trace": "71a8ac8646af2c3e14e5a5380c42696c",
"log": "stdout",
"ts": "2026-11-17T10:45:57.36543118Z"
},
"log_key": "72a76c602bf2998e18f2948b8a3845ca6878c0ca26a8107a64f07a90fae9d1e6",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912393,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52135,
"message_url": "https://xray.tilda.pizza/alerts/messages/52135"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52132 2026-11-17 10:40:57 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nhitl_tool: pre-approval card generation failed",
"message": "hitl_tool: pre-approval card generation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=39833992f95a94666c5b27bd26d74896&at=1794912054&msg=hitl_tool%3A+pre-approval+card+generation+failed",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "hitl_tool: pre-approval card generation failed",
"trace": "39833992f95a94666c5b27bd26d74896",
"log": "stdout",
"ts": "2026-11-17T10:40:54.301233838Z"
},
"log_key": "ca87dfe8d32966c60dc52fd9c7c4a653b86eb156dda4e4937a2e62ebac8c46b7",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794912087,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52132,
"message_url": "https://xray.tilda.pizza/alerts/messages/52132"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52130 2026-11-17 10:38:48 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794911917",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:38:37.786915Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794911958,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52130,
"message_url": "https://xray.tilda.pizza/alerts/messages/52130"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52128 2026-11-17 10:33:36 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794911611",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:33:31.315218Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794911646,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52128,
"message_url": "https://xray.tilda.pizza/alerts/messages/52128"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52126 2026-11-17 10:19:19 UTC |
channel test | warning Any error → notify me (master): 2 more messages held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in schedsvc (tilda) on master; Error in tilda on master",
"body": "The alert allows 20 an hour. Most recent held: Error in schedsvc (tilda) on master; Error in tilda on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794910789,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52126,
"message_url": "https://xray.tilda.pizza/alerts/messages/52126"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #52124 2026-11-17 10:11:38 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5189cbe86b9f73d7ba71083aba382497&at=1794910294&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5189cbe86b9f73d7ba71083aba382497",
"log": "stderr",
"ts": "2026-11-17T10:11:34.929941666Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910328,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52124,
"message_url": "https://xray.tilda.pizza/alerts/messages/52124"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52122 2026-11-17 10:11:23 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f8959b8a063c66152dd52e027d24204d&at=1794910274&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f8959b8a063c66152dd52e027d24204d",
"log": "stderr",
"ts": "2026-11-17T10:11:14.554572887Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910313,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52122,
"message_url": "https://xray.tilda.pizza/alerts/messages/52122"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52120 2026-11-17 10:10:58 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f8959b8a063c66152dd52e027d24204d&at=1794910254&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f8959b8a063c66152dd52e027d24204d",
"log": "stderr",
"ts": "2026-11-17T10:10:54.930425341Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910288,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52120,
"message_url": "https://xray.tilda.pizza/alerts/messages/52120"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52118 2026-11-17 10:09:08 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794910139",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T10:08:59.462961Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794910178,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52118,
"message_url": "https://xray.tilda.pizza/alerts/messages/52118"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52116 2026-11-17 10:04:52 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\nHTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"message": "HTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=267ce0983d1ace3747730a30ed6a62d9&at=1794909889&msg=HTTP+request+failed%3A+Token+refresh+failed%3A+status%3D403+error%3Dinvalid_grant+error_description%3DUnknown+or+invalid+refresh+token.",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "HTTP request failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"trace": "267ce0983d1ace3747730a30ed6a62d9",
"log": "stderr",
"ts": "2026-11-17T10:04:49.79592372Z"
},
"log_key": "0e39edcfbd613b1d0e1fe9dc4ed8d9a19340d971265dd27d9309b644575c137a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794909924,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52116,
"message_url": "https://xray.tilda.pizza/alerts/messages/52116"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52114 2026-11-17 10:04:52 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\nToken refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"message": "Token refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=267ce0983d1ace3747730a30ed6a62d9&at=1794909889&msg=Token+refresh+failed%3A+Token+refresh+failed%3A+status%3D403+error%3Dinvalid_grant+error_description%3DUnknown+or+invalid+refresh+token.",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "Token refresh failed: Token refresh failed: status=403 error=invalid_grant error_description=Unknown or invalid refresh token.",
"trace": "267ce0983d1ace3747730a30ed6a62d9",
"log": "stderr",
"ts": "2026-11-17T10:04:49.800098019Z"
},
"log_key": "99d8242980bbfad7afcc475d5e933d703e6c20dbd35e88331bc63da82188920a",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794909922,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52114,
"message_url": "https://xray.tilda.pizza/alerts/messages/52114"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52111 2026-11-17 09:28:37 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=9b10438ba7044a6f077d276e761e23f6&at=1794907713&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "9b10438ba7044a6f077d276e761e23f6",
"log": "stdout",
"ts": "2026-11-17T09:28:33.370485649Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907717,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52111,
"message_url": "https://xray.tilda.pizza/alerts/messages/52111"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52109 2026-11-17 09:24:12 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794907447",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T09:24:07.894959Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907452,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52109,
"message_url": "https://xray.tilda.pizza/alerts/messages/52109"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52105 2026-11-17 09:19:08 UTC |
channel test | warning Any error → notify me (master): 1 more message held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 1 more message held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 1 more message held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in execsvc (tilda) on master",
"body": "The alert allows 20 an hour. Most recent held: Error in execsvc (tilda) on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794907178,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52105,
"message_url": "https://xray.tilda.pizza/alerts/messages/52105"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #52103 2026-11-17 09:18:33 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nevalResult is not a bool",
"message": "evalResult is not a bool",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907110&msg=evalResult+is+not+a+bool",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "evalResult is not a bool",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:30.579916018Z"
},
"log_key": "5fe49d3829c318f25ee6cc600a4c5170e367760f7a03b10d87afef9e46d17198",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907113,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52103,
"message_url": "https://xray.tilda.pizza/alerts/messages/52103"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #52101 2026-11-17 09:18:33 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nauto query re-evaluation failed",
"message": "auto query re-evaluation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907108&msg=auto+query+re-evaluation+failed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "auto query re-evaluation failed",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:28.670937555Z"
},
"log_key": "ab591fe230c52e574850a2b26a9eb3dcbbec2251f9d65aa2c21a908c4a887519",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907143,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794907113
},
"message_id": 52101,
"message_url": "/alerts/messages/52101"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52099 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nJob completed",
"message": "Job completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907108&msg=Job+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Job completed",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:28.728542847Z"
},
"log_key": "c74deab9b0ebaca0dfd6164704293554f866fed15541770acc539e74f689ff56",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907208,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52099,
"message_url": "https://xray.tilda.pizza/alerts/messages/52099"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52097 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to evaluate and raise auto query for participant question",
"message": "failed to evaluate and raise auto query for participant question",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907111&msg=failed+to+evaluate+and+raise+auto+query+for+participant+question",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to evaluate and raise auto query for participant question",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:31.635961449Z"
},
"log_key": "6bd3587402488be63611acc5599dc01571e825e5157878dd4c0a52de2538572f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907143,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52097,
"message_url": "https://xray.tilda.pizza/alerts/messages/52097"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52095 2026-11-17 09:18:32 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nno result from branch evaluation",
"message": "no result from branch evaluation",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f07c10b3ed56b2fdace28225487175e6&at=1794907111&msg=no+result+from+branch+evaluation",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "no result from branch evaluation",
"trace": "f07c10b3ed56b2fdace28225487175e6",
"log": "stdout",
"ts": "2026-11-17T09:18:31.635906533Z"
},
"log_key": "99389d22a81127e2d8a7b9897343d56e8b9ff07cd2a5c2eaedff03af61df8f87",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794907142,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52095,
"message_url": "https://xray.tilda.pizza/alerts/messages/52095"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52093 2026-11-17 09:08:00 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794906474",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T09:07:54.708553Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794906510,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52093,
"message_url": "https://xray.tilda.pizza/alerts/messages/52093"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52087 2026-11-17 08:36:35 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nmessage was never published because transaction was never committed",
"message": "message was never published because transaction was never committed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904591&msg=message+was+never+published+because+transaction+was+never+committed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "message was never published because transaction was never committed",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stdout",
"ts": "2026-11-17T08:36:31.66744662Z"
},
"log_key": "2f0e80c680892bdde82cb2c9e495d0250d68634179bec2f228ad86045550169f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904625,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52087,
"message_url": "https://xray.tilda.pizza/alerts/messages/52087"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52085 2026-11-17 08:34:55 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document stateTag: Either object does not exist OR object has been updated since last read",
"message": "could not update document stateTag: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904489&msg=could+not+update+document+stateTag%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document stateTag: Either object does not exist OR object has been updated since last read",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stdout",
"ts": "2026-11-17T08:34:49.319801944Z"
},
"log_key": "65148ebad9b32837f5019dc4dfbf5b7e345dfefcf618c31c9d05b3ce447d2951",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904526,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52085,
"message_url": "https://xray.tilda.pizza/alerts/messages/52085"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52083 2026-11-17 08:34:54 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=bec6e867c738f443c04b6700e23abced&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+the+document+version%3A+cannot+use+this+transaction%2C+because+it+has+been+committed+or+rolled+back+either+because+of+an+error+or+explicit+termination",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update the document version: cannot use this transaction, because it has been committed or rolled back either because of an error or explicit termination",
"trace": "bec6e867c738f443c04b6700e23abced",
"log": "stdout",
"ts": "2026-11-17T08:34:46.989314983Z"
},
"log_key": "fe66dcd80edbbbed2a532d7a5dca22acedcd7bb1f922975948ab6692ce004464",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904585,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52083,
"message_url": "https://xray.tilda.pizza/alerts/messages/52083"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52081 2026-11-17 08:34:54 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nfailed to run cypher in transaction",
"message": "failed to run cypher in transaction",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=bec6e867c738f443c04b6700e23abced&at=1794904486&msg=failed+to+run+cypher+in+transaction",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to run cypher in transaction",
"trace": "bec6e867c738f443c04b6700e23abced",
"log": "stdout",
"ts": "2026-11-17T08:34:46.988957864Z"
},
"log_key": "666d7cdf6bb8bdc0a4979d6039dfc336ef34dcfb6e2f4b80bcc9a4658c479d10",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904525,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52081,
"message_url": "https://xray.tilda.pizza/alerts/messages/52081"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52079 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=c064764e0dc90c4b062551c5af6018e9&at=1794904486&msg=docsvc+PATCH+failed+for+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for c7b214fe-b333-4a54-ae72-1e768f742ffd: 500 {\"message\":\"could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"trace": "c064764e0dc90c4b062551c5af6018e9",
"log": "stdout",
"ts": "2026-11-17T08:34:46.304992402Z"
},
"log_key": "91f919a87fb0b5e3bce7ba445a56cb71b5a8a8f1c12f621aa79a417e53fe7721",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904523,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52079,
"message_url": "https://xray.tilda.pizza/alerts/messages/52079"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52077 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\nCBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"message": "CBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=6cdb5d0c42dc9ac77a44da4bde8a901c&at=1794904489&msg=CBR+promote+failed+for+1d50555b-f57a-4a0a-afe2-a3c8487d5808%3A+HTTP+500%3A+%7B%22message%22%3A%22could+not+update+document+stateTag%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "CBR promote failed for 1d50555b-f57a-4a0a-afe2-a3c8487d5808: HTTP 500: {\"message\":\"could not update document stateTag: Either object does not exist OR object has been updated since last read\"}",
"trace": "6cdb5d0c42dc9ac77a44da4bde8a901c",
"log": "stderr",
"ts": "2026-11-17T08:34:49.324828058Z"
},
"log_key": "583360d93020a3242f3d1779a04060a7f8c2cb469c3d12e13029edd9fe5955c7",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904586,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52077,
"message_url": "https://xray.tilda.pizza/alerts/messages/52077"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52075 2026-11-17 08:34:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=b7b68a7116fb8eab82300256cdcf5798&at=1794904487&msg=docsvc+PATCH+failed+for+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+could+not+update+the+document%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for 7db44208-f1f6-467f-b83c-dbcbb23dace7: 500 {\"message\":\"could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read\"}",
"trace": "b7b68a7116fb8eab82300256cdcf5798",
"log": "stdout",
"ts": "2026-11-17T08:34:47.938613056Z"
},
"log_key": "3fc355714dff25d1afa5c817956c78ce060ff4f268207a99ab838805955c0a48",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904523,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52075,
"message_url": "https://xray.tilda.pizza/alerts/messages/52075"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52073 2026-11-17 08:34:52 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=b7b68a7116fb8eab82300256cdcf5798&at=1794904487&msg=could+not+update+document+metadata+for+doc+id+7db44208-f1f6-467f-b83c-dbcbb23dace7%3A+could+not+update+the+document%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 7db44208-f1f6-467f-b83c-dbcbb23dace7: could not update the document: Either object does not exist OR object has been updated since last read",
"trace": "b7b68a7116fb8eab82300256cdcf5798",
"log": "stdout",
"ts": "2026-11-17T08:34:47.934324153Z"
},
"log_key": "880f828c1add82344341fccf86906d149de7822f7be05c44cde89278e3d11b4e",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904522,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52073,
"message_url": "https://xray.tilda.pizza/alerts/messages/52073"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52071 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.168601732Z"
},
"log_key": "d9ed7ccbf189c2a651947a348909dd0865379215eb592d2a8da973a5afd5904c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904519,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52071,
"message_url": "https://xray.tilda.pizza/alerts/messages/52071"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52069 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ncould not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"message": "could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=c064764e0dc90c4b062551c5af6018e9&at=1794904486&msg=could+not+update+document+metadata+for+doc+id+c7b214fe-b333-4a54-ae72-1e768f742ffd%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not update document metadata for doc id c7b214fe-b333-4a54-ae72-1e768f742ffd: could not update document for categorization: Either object does not exist OR object has been updated since last read",
"trace": "c064764e0dc90c4b062551c5af6018e9",
"log": "stdout",
"ts": "2026-11-17T08:34:46.297301087Z"
},
"log_key": "c762240c0cfa5115e58ec9bcf85e2e1b67f878c6d5314feac8d66641aca61ad3",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904520,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52069,
"message_url": "https://xray.tilda.pizza/alerts/messages/52069"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52067 2026-11-17 08:34:49 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.168815602Z"
},
"log_key": "77db170144f3de14233f801b01d071ae079a710620d377007912f895aa3c5c3b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904519,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52067,
"message_url": "https://xray.tilda.pizza/alerts/messages/52067"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52065 2026-11-17 08:34:48 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\ndocsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"message": "docsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&trace=5fc5213f3dffb574d30253956b8cb23e&at=1794904486&msg=docsvc+PATCH+failed+for+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+500+%7B%22message%22%3A%22could+not+update+document+metadata+for+doc+id+8d77d110-eb2a-49d7-be95-bbe2dbabecb6%3A+could+not+update+document+for+categorization%3A+Either+object+does+not+exist+OR+object+has+been+updated+since+last+read%22%7D",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "docsvc PATCH failed for 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: 500 {\"message\":\"could not update document metadata for doc id 8d77d110-eb2a-49d7-be95-bbe2dbabecb6: could not update document for categorization: Either object does not exist OR object has been updated since last read\"}",
"trace": "5fc5213f3dffb574d30253956b8cb23e",
"log": "stdout",
"ts": "2026-11-17T08:34:46.178951376Z"
},
"log_key": "07690fd09d96fac52a6ce4743a74feab480d7581d10f395fe567fd8c0420a442",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904518,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52065,
"message_url": "https://xray.tilda.pizza/alerts/messages/52065"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52063 2026-11-17 08:34:47 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794904484",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T08:34:44.779977Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904517,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52063,
"message_url": "https://xray.tilda.pizza/alerts/messages/52063"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52061 2026-11-17 08:30:14 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nNo cached node found for activation chain ref node",
"message": "No cached node found for activation chain ref node",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=b1881d3cd7fbff839e742d6f0d372723&at=1794904210&msg=No+cached+node+found+for+activation+chain+ref+node",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "No cached node found for activation chain ref node",
"trace": "b1881d3cd7fbff839e742d6f0d372723",
"log": "stdout",
"ts": "2026-11-17T08:30:10.939546298Z"
},
"log_key": "bf042eb1b187488d26cc24b28cd653f2aa4c55bb2320a2758b030eccc34a3247",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794904244,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52061,
"message_url": "https://xray.tilda.pizza/alerts/messages/52061"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52059 2026-11-17 08:22:38 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nNo cached node found for activation chain ref node",
"message": "No cached node found for activation chain ref node",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=eb09f7ddcb816d48d074f92cc60fb3e9&at=1794903754&msg=No+cached+node+found+for+activation+chain+ref+node",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "No cached node found for activation chain ref node",
"trace": "eb09f7ddcb816d48d074f92cc60fb3e9",
"log": "stdout",
"ts": "2026-11-17T08:22:34.308527471Z"
},
"log_key": "bf042eb1b187488d26cc24b28cd653f2aa4c55bb2320a2758b030eccc34a3247",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903788,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52059,
"message_url": "https://xray.tilda.pizza/alerts/messages/52059"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52057 2026-11-17 08:12:47 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"message": "[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=a6681eb4bb478e7cef99be407d43fd1b&at=1794903163&msg=%5Breminder%5D+wo%3D75313c24-f239-4f5a-9a30-693ecab5c0a2+greeting+still+wrong+after+2+attempts+%28The+greeting+must+reproduce+the+reviewer%27s+own+line+exactly%3A+%27Estimados+%7B%7Bprincipal_investigators%7D%7D%3A%27.+The+draft+opens+%27Dear+Pacific+Clinical+Studies+Team%2C%27.%29+%E2%80%94+sending+the+best+draft+anyway",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=75313c24-f239-4f5a-9a30-693ecab5c0a2 greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Estimados {{principal_investigators}}:'. The draft opens 'Dear Pacific Clinical Studies Team,'.) \u2014 sending the best draft anyway",
"trace": "a6681eb4bb478e7cef99be407d43fd1b",
"log": "stderr",
"ts": "2026-11-17T08:12:43.036994764Z"
},
"log_key": "c4dfd34c090bb70c9801e796fcf0334c14491a4f4e4c11c9c9ca959c0c3807d6",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903197,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52057,
"message_url": "https://xray.tilda.pizza/alerts/messages/52057"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52055 2026-11-17 08:11:29 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=a6681eb4bb478e7cef99be407d43fd1b&at=1794903083&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "a6681eb4bb478e7cef99be407d43fd1b",
"log": "stderr",
"ts": "2026-11-17T08:11:23.533375065Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903119,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52055,
"message_url": "https://xray.tilda.pizza/alerts/messages/52055"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52053 2026-11-17 08:10:53 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=be54167d5123d43e4ab2ca89a1a3ed67&at=1794903049&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "be54167d5123d43e4ab2ca89a1a3ed67",
"log": "stderr",
"ts": "2026-11-17T08:10:49.708853636Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903083,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52053,
"message_url": "https://xray.tilda.pizza/alerts/messages/52053"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52051 2026-11-17 08:10:38 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=be54167d5123d43e4ab2ca89a1a3ed67&at=1794903036&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "be54167d5123d43e4ab2ca89a1a3ed67",
"log": "stderr",
"ts": "2026-11-17T08:10:36.001561484Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794903068,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52051,
"message_url": "https://xray.tilda.pizza/alerts/messages/52051"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52043 2026-11-17 07:14:35 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794899663",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T07:14:23.510106Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794899705,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52043,
"message_url": "https://xray.tilda.pizza/alerts/messages/52043"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52041 2026-11-17 07:05:57 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794899151",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T07:05:51.551406Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794899187,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52041,
"message_url": "https://xray.tilda.pizza/alerts/messages/52041"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52039 2026-11-17 06:56:40 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nhttp error response returned to client",
"message": "http error response returned to client",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&at=1794898598&msg=http+error+response+returned+to+client",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "http error response returned to client",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T06:56:38.728063147Z"
},
"log_key": "ab310e879110b225580d45a31d16bebbb9b120817df3a3e7b1642a87afc82555",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898630,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794898600
},
"message_id": 52039,
"message_url": "/alerts/messages/52039"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #52037 2026-11-17 06:56:39 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nresponse set creation failed",
"message": "response set creation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&trace=4c7fc825c23f5048ecfa6aee48c56797&at=1794898598&msg=response+set+creation+failed",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "response set creation failed",
"trace": "4c7fc825c23f5048ecfa6aee48c56797",
"log": "stdout",
"ts": "2026-11-17T06:56:38.72301539Z"
},
"log_key": "026cc253e3528321226f9cf754dcba90363259e82486a4cae61246851106fbcf",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898690,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52037,
"message_url": "https://xray.tilda.pizza/alerts/messages/52037"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52035 2026-11-17 06:56:39 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&trace=4c7fc825c23f5048ecfa6aee48c56797&at=1794898598&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "4c7fc825c23f5048ecfa6aee48c56797",
"log": "stdout",
"ts": "2026-11-17T06:56:38.728542426Z"
},
"log_key": "12706001fcfc74398af6ce45a2871ca57e27612f99b2566b34c34dcc62dbf44d",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794898629,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52035,
"message_url": "https://xray.tilda.pizza/alerts/messages/52035"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52032 2026-11-17 06:38:22 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nException in ASGI application\n",
"message": "Exception in ASGI application\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&at=1794897500&msg=Exception+in+ASGI+application%0A",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "Exception in ASGI application\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-17T06:38:20.071681218Z"
},
"log_key": "6da008b24934c57f7b3d8a2b1786e970fd9107e256c2508345279522456a2593",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794897533,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52032,
"message_url": "https://xray.tilda.pizza/alerts/messages/52032"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52030 2026-11-17 06:38:22 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nUnhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"message": "Unhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=4a123c049b28dd7d6deb25be8306ecea&at=1794897500&msg=Unhandled+TimeoutError+on+POST+%2Fapi%2Fv1%2Fchat%2F44b4f1ec-4085-4783-980e-42f1896b3f81%2Fruns%2Fstream",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "Unhandled TimeoutError on POST /api/v1/chat/44b4f1ec-4085-4783-980e-42f1896b3f81/runs/stream",
"trace": "4a123c049b28dd7d6deb25be8306ecea",
"log": "stderr",
"ts": "2026-11-17T06:38:20.046775832Z"
},
"log_key": "faf4bb714e9b98fb3971917c05665db0808c8db5f734087325c1d8fba30561fe",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794897532,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52030,
"message_url": "https://xray.tilda.pizza/alerts/messages/52030"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52028 2026-11-17 06:11:23 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=e90bf6d10ed2f1b27ff1be1d00f36ac6&at=1794895878&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "e90bf6d10ed2f1b27ff1be1d00f36ac6",
"log": "stderr",
"ts": "2026-11-17T06:11:18.495984787Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895913,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52028,
"message_url": "https://xray.tilda.pizza/alerts/messages/52028"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52026 2026-11-17 06:11:07 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5e506b9abbab0e53b3b7f5129308130a&at=1794895865&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5e506b9abbab0e53b3b7f5129308130a",
"log": "stderr",
"ts": "2026-11-17T06:11:05.007340589Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895897,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52026,
"message_url": "https://xray.tilda.pizza/alerts/messages/52026"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52024 2026-11-17 06:10:37 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=5e506b9abbab0e53b3b7f5129308130a&at=1794895836&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "5e506b9abbab0e53b3b7f5129308130a",
"log": "stderr",
"ts": "2026-11-17T06:10:36.07703947Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794895867,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52024,
"message_url": "https://xray.tilda.pizza/alerts/messages/52024"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52022 2026-11-17 05:51:02 UTC |
channel test | warning Error in schedsvc (tilda) on master
show the payload{
"title": "Error in schedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in schedsvc (tilda) on master",
"text": "Error in schedsvc (tilda) on master\nerror parsing time periods",
"message": "error parsing time periods",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=schedsvc&namespace=tilda&trace=ace7099be9aeb9adceb57a9a300458db&at=1794894662&msg=error+parsing+time+periods",
"log": {
"severity": "ERROR",
"service": "schedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "error parsing time periods",
"trace": "ace7099be9aeb9adceb57a9a300458db",
"log": "stdout",
"ts": "2026-11-17T05:51:02.034780594Z"
},
"log_key": "383df22466f28d9307b23fa6ffc5e8048ba0976f608f8d339e427d4b7b05b389",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794894692,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52022,
"message_url": "https://xray.tilda.pizza/alerts/messages/52022"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52020 2026-11-17 05:35:00 UTC |
channel test | info Tilda namespace findings (master): all clear — 1 resolved
show the card🔵 Tilda namespace findings (master): all clear — 1 resolved alert: Tilda namespace findings (master) · changed 1 resolved ✅ execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx) message 2 for this group · last one 48m ago · X-ray message #52020 |
#dev-notifications sent#in-app delivered |
group | ok in-app: 52 inbox row(s) for 52 people on X-ray |
| #52014 2026-11-17 05:28:06 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794893278",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:27:58.598886Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794893316,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52014,
"message_url": "https://xray.tilda.pizza/alerts/messages/52014"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52012 2026-11-17 05:20:17 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794892809",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:20:09.007504Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794892847,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52012,
"message_url": "https://xray.tilda.pizza/alerts/messages/52012"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52009 2026-11-17 05:01:57 UTC |
channel test | warning Error in dashrsvc (tilda) on master
show the payload{
"title": "Error in dashrsvc (tilda) on master",
"severity": "warning",
"headline": "Error in dashrsvc (tilda) on master",
"text": "Error in dashrsvc (tilda) on master\nFailed to resolve document_id from ai_run_id",
"message": "Failed to resolve document_id from ai_run_id",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=dashrsvc&namespace=tilda&trace=cc6c195e6bb4691b947c00a66eabf24e&at=1794891712&msg=Failed+to+resolve+document_id+from+ai_run_id",
"log": {
"severity": "ERROR",
"service": "dashrsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to resolve document_id from ai_run_id",
"trace": "cc6c195e6bb4691b947c00a66eabf24e",
"log": "stdout",
"ts": "2026-11-17T05:01:52.642482707Z"
},
"log_key": "6020cbf0f8aa1e940471331827fa9fb81f077ddd5b118c4b905177169da8ae06",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794891747,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52009,
"message_url": "https://xray.tilda.pizza/alerts/messages/52009"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52007 2026-11-17 05:01:31 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794891689",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-17T05:01:29.004453Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794891721,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52007,
"message_url": "https://xray.tilda.pizza/alerts/messages/52007"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52005 2026-11-17 04:46:53 UTC |
channel test | critical execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx)
show the card🟠 HIGH · execsvc server-error rate 89.19% vs a normal 0.00% ± 1.00% for Tue 04:00 UTC (3 of the last 4 checks) — mostly POST /api/trialexec/v2/study/:id/activity/status (787× 5xx) master · Error-rate anomaly · high execsvc shows an 89.2% error rate on master, significantly above the normal 0.0% ± 1.0% baseline for this weekly time slot. The anomaly spans 3 of the last 4 five-minute buckets (z = 89.2), based on 4 weeks of historical data. The failing endpoint is POST /api/trialexec/v2/study/:id/activity/status, which experienced 787 errors out of 787 total calls. User impact depends on endpoint traffic distribution. The facts measure: err5_pct unit: % value: 89.19 expected_median: 0.00 expected_mad: 1.00 z: 89.19 rollup · observed 56s ago · 1× so far · X-ray message #52005 |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 52 inbox row(s) for 52 people on X-ray |
| #52003 2026-11-17 04:34:27 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\noauth token refresh rejected by provider",
"message": "oauth token refresh rejected by provider",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&at=1794890065&msg=oauth+token+refresh+rejected+by+provider",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "oauth token refresh rejected by provider",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T04:34:25.32672437Z"
},
"log_key": "6ca27a7d0989ad0479f52d6ff96c685d6de5df4d24fb66d7063f2174ae33cea0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794890099,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52003,
"message_url": "https://xray.tilda.pizza/alerts/messages/52003"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #52001 2026-11-17 04:34:27 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\nfailed to refresh token; session refresh loop stopping",
"message": "failed to refresh token; session refresh loop stopping",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&at=1794890065&msg=failed+to+refresh+token%3B+session+refresh+loop+stopping",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to refresh token; session refresh loop stopping",
"trace": null,
"log": "stdout",
"ts": "2026-11-17T04:34:25.32677349Z"
},
"log_key": "3e851c0107918c12cccdc5c5232f60c9aff54bcdca4164121b97e06e20deb4e8",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794890097,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 52001,
"message_url": "https://xray.tilda.pizza/alerts/messages/52001"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51998 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not match participant questionnaire/question/table/doc-upload to activity graph",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=155a3bef7d375e6fd6a4cef1db49f6e3&at=1794888974&msg=could+not+match+participant+questionnaire%2Fquestion%2Ftable%2Fdoc-upload+to+activity+graph",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"trace": "155a3bef7d375e6fd6a4cef1db49f6e3",
"log": "stdout",
"ts": "2026-11-17T04:16:14.500150165Z"
},
"log_key": "d6dc1706a7113bb3b7c2d69ab60ed30804cb9bf28f56d4f2280bed1aa61b32a0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889069,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51998,
"message_url": "https://xray.tilda.pizza/alerts/messages/51998"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51996 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nFailed to fetch activity status",
"message": "Failed to fetch activity status",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=7ecc0ff1e56ae17c05c88a1b04780a82&at=1794888974&msg=Failed+to+fetch+activity+status",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to fetch activity status",
"trace": "7ecc0ff1e56ae17c05c88a1b04780a82",
"log": "stdout",
"ts": "2026-11-17T04:16:14.586932118Z"
},
"log_key": "826d3d9347bec580c70fbebf5014a82dd6d897ebccddd610c2fc8ba98cdfe3ae",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889008,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51996,
"message_url": "https://xray.tilda.pizza/alerts/messages/51996"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51994 2026-11-17 04:16:17 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f1d365830b94cf97bf5c96d5e36d30d5&at=1794888972&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "f1d365830b94cf97bf5c96d5e36d30d5",
"log": "stdout",
"ts": "2026-11-17T04:16:12.502809085Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794889007,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51994,
"message_url": "https://xray.tilda.pizza/alerts/messages/51994"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51992 2026-11-17 04:11:33 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=018a75f866afc8521606b9f807ebcc9b&at=1794888682&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "018a75f866afc8521606b9f807ebcc9b",
"log": "stderr",
"ts": "2026-11-17T04:11:22.915624207Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888723,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51992,
"message_url": "https://xray.tilda.pizza/alerts/messages/51992"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51990 2026-11-17 04:11:12 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=690fc5aec2b80f9deacfd4742f575fb8&at=1794888668&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "690fc5aec2b80f9deacfd4742f575fb8",
"log": "stderr",
"ts": "2026-11-17T04:11:08.793961152Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888702,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51990,
"message_url": "https://xray.tilda.pizza/alerts/messages/51990"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51988 2026-11-17 04:11:02 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=690fc5aec2b80f9deacfd4742f575fb8&at=1794888657&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "690fc5aec2b80f9deacfd4742f575fb8",
"log": "stderr",
"ts": "2026-11-17T04:10:57.801069754Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888692,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51988,
"message_url": "https://xray.tilda.pizza/alerts/messages/51988"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51986 2026-11-17 04:10:29 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not match participant questionnaire/question/table/doc-upload to activity graph",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=53245cec7eb9fed7222dc5d85462b620&at=1794888620&msg=could+not+match+participant+questionnaire%2Fquestion%2Ftable%2Fdoc-upload+to+activity+graph",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"trace": "53245cec7eb9fed7222dc5d85462b620",
"log": "stdout",
"ts": "2026-11-17T04:10:20.257053543Z"
},
"log_key": "d6dc1706a7113bb3b7c2d69ab60ed30804cb9bf28f56d4f2280bed1aa61b32a0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888659,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51986,
"message_url": "https://xray.tilda.pizza/alerts/messages/51986"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51984 2026-11-17 04:10:23 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nFailed to fetch activity status",
"message": "Failed to fetch activity status",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f09f75be21bec7599e192eb15f9913f4&at=1794888619&msg=Failed+to+fetch+activity+status",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to fetch activity status",
"trace": "f09f75be21bec7599e192eb15f9913f4",
"log": "stdout",
"ts": "2026-11-17T04:10:19.411731586Z"
},
"log_key": "826d3d9347bec580c70fbebf5014a82dd6d897ebccddd610c2fc8ba98cdfe3ae",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888654,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51984,
"message_url": "https://xray.tilda.pizza/alerts/messages/51984"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51982 2026-11-17 04:10:23 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f09f75be21bec7599e192eb15f9913f4&at=1794888619&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "f09f75be21bec7599e192eb15f9913f4",
"log": "stdout",
"ts": "2026-11-17T04:10:19.411868326Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888653,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51982,
"message_url": "https://xray.tilda.pizza/alerts/messages/51982"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51980 2026-11-17 04:05:23 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not match participant questionnaire/question/table/doc-upload to activity graph",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=edd65f07b355d0296082bcef52bf7665&at=1794888320&msg=could+not+match+participant+questionnaire%2Fquestion%2Ftable%2Fdoc-upload+to+activity+graph",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not match participant questionnaire/question/table/doc-upload to activity graph",
"trace": "edd65f07b355d0296082bcef52bf7665",
"log": "stdout",
"ts": "2026-11-17T04:05:20.520041023Z"
},
"log_key": "d6dc1706a7113bb3b7c2d69ab60ed30804cb9bf28f56d4f2280bed1aa61b32a0",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888353,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51980,
"message_url": "https://xray.tilda.pizza/alerts/messages/51980"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51978 2026-11-17 04:05:22 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nFailed to fetch activity status",
"message": "Failed to fetch activity status",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=cfaf1c8ba6d68db114230cbbd0f63418&at=1794888317&msg=Failed+to+fetch+activity+status",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Failed to fetch activity status",
"trace": "cfaf1c8ba6d68db114230cbbd0f63418",
"log": "stdout",
"ts": "2026-11-17T04:05:17.163201915Z"
},
"log_key": "826d3d9347bec580c70fbebf5014a82dd6d897ebccddd610c2fc8ba98cdfe3ae",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888413,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51978,
"message_url": "https://xray.tilda.pizza/alerts/messages/51978"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51976 2026-11-17 04:05:22 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=cfaf1c8ba6d68db114230cbbd0f63418&at=1794888317&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "cfaf1c8ba6d68db114230cbbd0f63418",
"log": "stdout",
"ts": "2026-11-17T04:05:17.163376105Z"
},
"log_key": "520c14762a99a5be7aefae37d3d55b28e8805375174fc9968943ebb288bd249c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794888352,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51976,
"message_url": "https://xray.tilda.pizza/alerts/messages/51976"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51972 2026-11-17 03:14:13 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nevalResult is not a bool",
"message": "evalResult is not a bool",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=fc357f2a406be83e41fbedda9c2ce3bd&at=1794885249&msg=evalResult+is+not+a+bool",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "evalResult is not a bool",
"trace": "fc357f2a406be83e41fbedda9c2ce3bd",
"log": "stdout",
"ts": "2026-11-17T03:14:09.199526776Z"
},
"log_key": "5fe49d3829c318f25ee6cc600a4c5170e367760f7a03b10d87afef9e46d17198",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794885283,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51972,
"message_url": "https://xray.tilda.pizza/alerts/messages/51972"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51970 2026-11-17 03:14:09 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nJob completed",
"message": "Job completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=fc357f2a406be83e41fbedda9c2ce3bd&at=1794885246&msg=Job+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Job completed",
"trace": "fc357f2a406be83e41fbedda9c2ce3bd",
"log": "stdout",
"ts": "2026-11-17T03:14:06.673260834Z"
},
"log_key": "c74deab9b0ebaca0dfd6164704293554f866fed15541770acc539e74f689ff56",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794885279,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794885249
},
"message_id": 51970,
"message_url": "/alerts/messages/51970"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #51968 2026-11-17 03:14:08 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nauto query re-evaluation failed",
"message": "auto query re-evaluation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=fc357f2a406be83e41fbedda9c2ce3bd&at=1794885246&msg=auto+query+re-evaluation+failed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "auto query re-evaluation failed",
"trace": "fc357f2a406be83e41fbedda9c2ce3bd",
"log": "stdout",
"ts": "2026-11-17T03:14:06.609148877Z"
},
"log_key": "ab591fe230c52e574850a2b26a9eb3dcbbec2251f9d65aa2c21a908c4a887519",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794885341,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51968,
"message_url": "https://xray.tilda.pizza/alerts/messages/51968"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51966 2026-11-17 03:14:07 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to evaluate and raise auto query for participant question",
"message": "failed to evaluate and raise auto query for participant question",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=fc357f2a406be83e41fbedda9c2ce3bd&at=1794885246&msg=failed+to+evaluate+and+raise+auto+query+for+participant+question",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to evaluate and raise auto query for participant question",
"trace": "fc357f2a406be83e41fbedda9c2ce3bd",
"log": "stdout",
"ts": "2026-11-17T03:14:06.970864733Z"
},
"log_key": "6bd3587402488be63611acc5599dc01571e825e5157878dd4c0a52de2538572f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794885342,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51966,
"message_url": "https://xray.tilda.pizza/alerts/messages/51966"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51964 2026-11-17 03:14:07 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nno result from branch evaluation",
"message": "no result from branch evaluation",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=fc357f2a406be83e41fbedda9c2ce3bd&at=1794885247&msg=no+result+from+branch+evaluation",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "no result from branch evaluation",
"trace": "fc357f2a406be83e41fbedda9c2ce3bd",
"log": "stdout",
"ts": "2026-11-17T03:14:07.002262246Z"
},
"log_key": "99389d22a81127e2d8a7b9897343d56e8b9ff07cd2a5c2eaedff03af61df8f87",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794885277,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51964,
"message_url": "https://xray.tilda.pizza/alerts/messages/51964"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51962 2026-11-17 02:54:44 UTC |
channel test | warning Error in feedsvc (tilda) on master
show the payload{
"title": "Error in feedsvc (tilda) on master",
"severity": "warning",
"headline": "Error in feedsvc (tilda) on master",
"text": "Error in feedsvc (tilda) on master\nUnable to retrieve routing information",
"message": "Unable to retrieve routing information",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=feedsvc&namespace=tilda&at=1794884081&msg=Unable+to+retrieve+routing+information",
"log": {
"severity": "ERROR",
"service": "feedsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Unable to retrieve routing information",
"trace": null,
"log": "stderr",
"ts": "2026-11-17T02:54:41.060762195Z"
},
"log_key": "ca6ac523f76aab819c1c237fc1c67a1faf8bc774f638402d88720fcfe0038c3f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794884114,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51962,
"message_url": "https://xray.tilda.pizza/alerts/messages/51962"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51960 2026-11-17 02:11:22 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=256a620e1cdecbc15b24bfd01f0ef946&at=1794881479&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "256a620e1cdecbc15b24bfd01f0ef946",
"log": "stderr",
"ts": "2026-11-17T02:11:19.903538862Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794881512,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51960,
"message_url": "https://xray.tilda.pizza/alerts/messages/51960"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51958 2026-11-17 02:11:07 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=0a7d6ce5f30a62f317321327f306dde6&at=1794881464&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "0a7d6ce5f30a62f317321327f306dde6",
"log": "stderr",
"ts": "2026-11-17T02:11:04.866499943Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794881497,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51958,
"message_url": "https://xray.tilda.pizza/alerts/messages/51958"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51956 2026-11-17 02:10:57 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=0a7d6ce5f30a62f317321327f306dde6&at=1794881452&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "0a7d6ce5f30a62f317321327f306dde6",
"log": "stderr",
"ts": "2026-11-17T02:10:52.338628368Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794881487,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51956,
"message_url": "https://xray.tilda.pizza/alerts/messages/51956"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51950 2026-11-17 00:40:46 UTC |
channel test | warning Error in studiosvc (tilda) on master
show the payload{
"title": "Error in studiosvc (tilda) on master",
"severity": "warning",
"headline": "Error in studiosvc (tilda) on master",
"text": "Error in studiosvc (tilda) on master\nfailed to fetch form summary",
"message": "failed to fetch form summary",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studiosvc&namespace=tilda&trace=4123ff6a5664f4a68abad51029866263&at=1794876033&msg=failed+to+fetch+form+summary",
"log": {
"severity": "ERROR",
"service": "studiosvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to fetch form summary",
"trace": "4123ff6a5664f4a68abad51029866263",
"log": "stdout",
"ts": "2026-11-17T00:40:33.134870915Z"
},
"log_key": "498c5646a871bf7f705b88d5bdace4ce0762f6a7b7617542b86b30461edd6658",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794876076,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51950,
"message_url": "https://xray.tilda.pizza/alerts/messages/51950"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51946 2026-11-17 00:11:30 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=2d80460f20b14e463dd0db16a932e73c&at=1794874282&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "2d80460f20b14e463dd0db16a932e73c",
"log": "stderr",
"ts": "2026-11-17T00:11:22.558701841Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794874320,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51946,
"message_url": "https://xray.tilda.pizza/alerts/messages/51946"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51944 2026-11-17 00:11:13 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=bafd0c10b451471cdb9de34ca843af13&at=1794874268&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "bafd0c10b451471cdb9de34ca843af13",
"log": "stderr",
"ts": "2026-11-17T00:11:08.140588896Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794874304,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51944,
"message_url": "https://xray.tilda.pizza/alerts/messages/51944"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51942 2026-11-17 00:10:52 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=bafd0c10b451471cdb9de34ca843af13&at=1794874249&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "bafd0c10b451471cdb9de34ca843af13",
"log": "stderr",
"ts": "2026-11-17T00:10:49.409286549Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794874283,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51942,
"message_url": "https://xray.tilda.pizza/alerts/messages/51942"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51932 2026-11-16 23:17:37 UTC |
channel test | warning Error in chat-agent-postgres (tilda) on master
show the payload{
"title": "Error in chat-agent-postgres (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent-postgres (tilda) on master",
"text": "Error in chat-agent-postgres (tilda) on master\ntuple concurrently updated",
"message": "tuple concurrently updated",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent-postgres&namespace=tilda&at=1794871053&msg=tuple+concurrently+updated",
"log": {
"severity": "ERROR",
"service": "chat-agent-postgres",
"cluster": "master",
"namespace": "tilda",
"message": "tuple concurrently updated",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T23:17:33.496942805Z"
},
"log_key": "10657908ba30c930168a4f2d453c85a28d05a86adbcb245d77442a403f3f5320",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794871087,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51932,
"message_url": "https://xray.tilda.pizza/alerts/messages/51932"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51928 2026-11-16 23:01:16 UTC |
channel test | warning Error in chat-agent-postgres (tilda) on master
show the payload{
"title": "Error in chat-agent-postgres (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent-postgres (tilda) on master",
"text": "Error in chat-agent-postgres (tilda) on master\ntuple concurrently updated",
"message": "tuple concurrently updated",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent-postgres&namespace=tilda&at=1794870071&msg=tuple+concurrently+updated",
"log": {
"severity": "ERROR",
"service": "chat-agent-postgres",
"cluster": "master",
"namespace": "tilda",
"message": "tuple concurrently updated",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T23:01:11.327389736Z"
},
"log_key": "10657908ba30c930168a4f2d453c85a28d05a86adbcb245d77442a403f3f5320",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794870106,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51928,
"message_url": "https://xray.tilda.pizza/alerts/messages/51928"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51924 2026-11-16 22:36:16 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794868570",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T22:36:10.186483Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794868606,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51924,
"message_url": "https://xray.tilda.pizza/alerts/messages/51924"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51918 2026-11-16 22:11:13 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=6bb30d9cf0192db4ace8265917f20114&at=1794867068&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "6bb30d9cf0192db4ace8265917f20114",
"log": "stderr",
"ts": "2026-11-16T22:11:08.991663091Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794867103,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51918,
"message_url": "https://xray.tilda.pizza/alerts/messages/51918"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51916 2026-11-16 22:11:02 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f06c36beba3ca7c4f1e6637607886101&at=1794867052&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f06c36beba3ca7c4f1e6637607886101",
"log": "stderr",
"ts": "2026-11-16T22:10:52.647836464Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794867092,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51916,
"message_url": "https://xray.tilda.pizza/alerts/messages/51916"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51914 2026-11-16 22:10:42 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f06c36beba3ca7c4f1e6637607886101&at=1794867038&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f06c36beba3ca7c4f1e6637607886101",
"log": "stderr",
"ts": "2026-11-16T22:10:38.413020634Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794867072,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51914,
"message_url": "https://xray.tilda.pizza/alerts/messages/51914"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51905 2026-11-16 21:11:50 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nevalResult is not a bool",
"message": "evalResult is not a bool",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=33fd3228f56325e3c212700c062f61dc&at=1794863506&msg=evalResult+is+not+a+bool",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "evalResult is not a bool",
"trace": "33fd3228f56325e3c212700c062f61dc",
"log": "stdout",
"ts": "2026-11-16T21:11:46.386409483Z"
},
"log_key": "5fe49d3829c318f25ee6cc600a4c5170e367760f7a03b10d87afef9e46d17198",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794863540,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51905,
"message_url": "https://xray.tilda.pizza/alerts/messages/51905"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51903 2026-11-16 21:11:48 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nJob completed",
"message": "Job completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=33fd3228f56325e3c212700c062f61dc&at=1794863503&msg=Job+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Job completed",
"trace": "33fd3228f56325e3c212700c062f61dc",
"log": "stdout",
"ts": "2026-11-16T21:11:43.315452949Z"
},
"log_key": "c74deab9b0ebaca0dfd6164704293554f866fed15541770acc539e74f689ff56",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794863538,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51903,
"message_url": "https://xray.tilda.pizza/alerts/messages/51903"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51901 2026-11-16 21:11:48 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nauto query re-evaluation failed",
"message": "auto query re-evaluation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=33fd3228f56325e3c212700c062f61dc&at=1794863503&msg=auto+query+re-evaluation+failed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "auto query re-evaluation failed",
"trace": "33fd3228f56325e3c212700c062f61dc",
"log": "stdout",
"ts": "2026-11-16T21:11:43.225829631Z"
},
"log_key": "ab591fe230c52e574850a2b26a9eb3dcbbec2251f9d65aa2c21a908c4a887519",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794863538,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51901,
"message_url": "https://xray.tilda.pizza/alerts/messages/51901"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51899 2026-11-16 21:11:45 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nno result from branch evaluation",
"message": "no result from branch evaluation",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=33fd3228f56325e3c212700c062f61dc&at=1794863502&msg=no+result+from+branch+evaluation",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "no result from branch evaluation",
"trace": "33fd3228f56325e3c212700c062f61dc",
"log": "stdout",
"ts": "2026-11-16T21:11:42.288211707Z"
},
"log_key": "99389d22a81127e2d8a7b9897343d56e8b9ff07cd2a5c2eaedff03af61df8f87",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794863537,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51899,
"message_url": "https://xray.tilda.pizza/alerts/messages/51899"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51897 2026-11-16 21:11:45 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to evaluate and raise auto query for participant question",
"message": "failed to evaluate and raise auto query for participant question",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=33fd3228f56325e3c212700c062f61dc&at=1794863502&msg=failed+to+evaluate+and+raise+auto+query+for+participant+question",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to evaluate and raise auto query for participant question",
"trace": "33fd3228f56325e3c212700c062f61dc",
"log": "stdout",
"ts": "2026-11-16T21:11:42.374088654Z"
},
"log_key": "6bd3587402488be63611acc5599dc01571e825e5157878dd4c0a52de2538572f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794863535,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51897,
"message_url": "https://xray.tilda.pizza/alerts/messages/51897"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51894 2026-11-16 20:45:45 UTC |
channel test | warning Error in studioai-master (tilda) on master
show the payload{
"title": "Error in studioai-master (tilda) on master",
"severity": "warning",
"headline": "Error in studioai-master (tilda) on master",
"text": "Error in studioai-master (tilda) on master\nError in message pull loop",
"message": "Error in message pull loop",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studioai-master&namespace=tilda&at=1794861940&msg=Error+in+message+pull+loop",
"log": {
"severity": "ERROR",
"service": "studioai-master",
"cluster": "master",
"namespace": "tilda",
"message": "Error in message pull loop",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T20:45:40.071354586Z"
},
"log_key": "2d5fe8ad0bc2e5e584ad67772d994c6f39c507060dd84b289a925ef49cf9c68c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794861975,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51894,
"message_url": "https://xray.tilda.pizza/alerts/messages/51894"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51892 2026-11-16 20:37:14 UTC |
channel test | warning Error in studioai-master (tilda) on master
show the payload{
"title": "Error in studioai-master (tilda) on master",
"severity": "warning",
"headline": "Error in studioai-master (tilda) on master",
"text": "Error in studioai-master (tilda) on master\nError in message pull loop",
"message": "Error in message pull loop",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=studioai-master&namespace=tilda&at=1794861433&msg=Error+in+message+pull+loop",
"log": {
"severity": "ERROR",
"service": "studioai-master",
"cluster": "master",
"namespace": "tilda",
"message": "Error in message pull loop",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T20:37:13.959090098Z"
},
"log_key": "2d5fe8ad0bc2e5e584ad67772d994c6f39c507060dd84b289a925ef49cf9c68c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794861464,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51892,
"message_url": "https://xray.tilda.pizza/alerts/messages/51892"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51885 2026-11-16 20:11:18 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=8153c99ed0d0c06ffe79a0599d198398&at=1794859873&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "8153c99ed0d0c06ffe79a0599d198398",
"log": "stderr",
"ts": "2026-11-16T20:11:13.670649369Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794859908,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51885,
"message_url": "https://xray.tilda.pizza/alerts/messages/51885"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51883 2026-11-16 20:11:00 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=330908ecc90be3826310c669130291c9&at=1794859856&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "330908ecc90be3826310c669130291c9",
"log": "stderr",
"ts": "2026-11-16T20:10:56.640260199Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794859890,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51883,
"message_url": "https://xray.tilda.pizza/alerts/messages/51883"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51881 2026-11-16 20:10:42 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=330908ecc90be3826310c669130291c9&at=1794859840&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "330908ecc90be3826310c669130291c9",
"log": "stderr",
"ts": "2026-11-16T20:10:40.027811582Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794859872,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51881,
"message_url": "https://xray.tilda.pizza/alerts/messages/51881"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51875 2026-11-16 18:34:30 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794854062",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T18:34:22.300412Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794854100,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51875,
"message_url": "https://xray.tilda.pizza/alerts/messages/51875"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51873 2026-11-16 18:34:27 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&at=1794854047&msg=Traceback+%28most+recent+call+last%29%3A%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fstarlette%2Frouting.py%22%2C+line+645%2C+in+lifespan%0A++++await+receive%28%29%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fuvicorn%2Flifespan%2Fon.py%22%2C+line+137%2C+in+receive%0A++++return+await+self.receive_queue.get%28%29%0A+++++++++++%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fasyncio%2Fqueues.py%22%2C+line+158%2C+in+get%0A+++",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-16T18:34:07.970904728Z"
},
"log_key": "07c70b2564d01b69c3cb76ec353977c171c0a9520a4757e2fb1931781a092a22",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794854097,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51873,
"message_url": "https://xray.tilda.pizza/alerts/messages/51873"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51869 2026-11-16 18:20:24 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794853220",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T18:20:20.181919Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794853254,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51869,
"message_url": "https://xray.tilda.pizza/alerts/messages/51869"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51867 2026-11-16 18:11:42 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=93b5742f-1fda-4057-a59b-3503c3c13a5b greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Buenos d\u00edas, equipo de {{country_name}},'. The draft opens 'Buenos d\u00edas, equipo de United States,'.) \u2014 sending the best draft anyway",
"message": "[reminder] wo=93b5742f-1fda-4057-a59b-3503c3c13a5b greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Buenos d\u00edas, equipo de {{country_name}},'. The draft opens 'Buenos d\u00edas, equipo de United States,'.) \u2014 sending the best draft anyway",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f7444e7d52e7549a1c1b6267439e781f&at=1794852699&msg=%5Breminder%5D+wo%3D93b5742f-1fda-4057-a59b-3503c3c13a5b+greeting+still+wrong+after+2+attempts+%28The+greeting+must+reproduce+the+reviewer%27s+own+line+exactly%3A+%27Buenos+d%C3%ADas%2C+equipo+de+%7B%7Bcountry_name%7D%7D%2C%27.+The+draft+opens+%27Buenos+d%C3%ADas%2C+equipo+de+United+States%2C%27.%29+%E2%80%94+sending+the+best+draft+anyway",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=93b5742f-1fda-4057-a59b-3503c3c13a5b greeting still wrong after 2 attempts (The greeting must reproduce the reviewer's own line exactly: 'Buenos d\u00edas, equipo de {{country_name}},'. The draft opens 'Buenos d\u00edas, equipo de United States,'.) \u2014 sending the best draft anyway",
"trace": "f7444e7d52e7549a1c1b6267439e781f",
"log": "stderr",
"ts": "2026-11-16T18:11:39.580593179Z"
},
"log_key": "2bda67268084c01f705274adddcb3469642ca8819e77e6e1e546f428f7b5cd37",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852732,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51867,
"message_url": "https://xray.tilda.pizza/alerts/messages/51867"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51865 2026-11-16 18:11:18 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f7444e7d52e7549a1c1b6267439e781f&at=1794852674&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f7444e7d52e7549a1c1b6267439e781f",
"log": "stderr",
"ts": "2026-11-16T18:11:14.222343695Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852708,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51865,
"message_url": "https://xray.tilda.pizza/alerts/messages/51865"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51863 2026-11-16 18:11:02 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=c7abc117575a6784d76c2214319ff23b&at=1794852658&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "c7abc117575a6784d76c2214319ff23b",
"log": "stderr",
"ts": "2026-11-16T18:10:58.304246321Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852692,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51863,
"message_url": "https://xray.tilda.pizza/alerts/messages/51863"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51861 2026-11-16 18:09:43 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794852559",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T18:09:19.679075Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852613,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51861,
"message_url": "https://xray.tilda.pizza/alerts/messages/51861"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51859 2026-11-16 18:09:19 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=d02255cbfc545b892075d60ae276d707&at=1794852554&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "d02255cbfc545b892075d60ae276d707",
"log": "stderr",
"ts": "2026-11-16T18:09:14.943139694Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852589,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51859,
"message_url": "https://xray.tilda.pizza/alerts/messages/51859"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51857 2026-11-16 18:06:03 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&at=1794852342&msg=Traceback+%28most+recent+call+last%29%3A%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fstarlette%2Frouting.py%22%2C+line+645%2C+in+lifespan%0A++++await+receive%28%29%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fuvicorn%2Flifespan%2Fon.py%22%2C+line+137%2C+in+receive%0A++++return+await+self.receive_queue.get%28%29%0A+++++++++++%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fasyncio%2Fqueues.py%22%2C+line+158%2C+in+get%0A+++",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-16T18:05:42.904550819Z"
},
"log_key": "07c70b2564d01b69c3cb76ec353977c171c0a9520a4757e2fb1931781a092a22",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852393,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51857,
"message_url": "https://xray.tilda.pizza/alerts/messages/51857"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51855 2026-11-16 18:05:27 UTC |
channel test | warning Error in ontologysvc (tilda) on master
show the payload{
"title": "Error in ontologysvc (tilda) on master",
"severity": "warning",
"headline": "Error in ontologysvc (tilda) on master",
"text": "Error in ontologysvc (tilda) on master\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=ontologysvc&namespace=tilda&at=1794852310&msg=Traceback+%28most+recent+call+last%29%3A%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fstarlette%2Frouting.py%22%2C+line+645%2C+in+lifespan%0A++++await+receive%28%29%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fuvicorn%2Flifespan%2Fon.py%22%2C+line+137%2C+in+receive%0A++++return+await+self.receive_queue.get%28%29%0A+++++++++++%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fasyncio%2Fqueues.py%22%2C+line+158%2C+in+get%0A+++",
"log": {
"severity": "ERROR",
"service": "ontologysvc",
"cluster": "master",
"namespace": "tilda",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-16T18:05:10.479391997Z"
},
"log_key": "dd6ce7080e0ba08fc33cc4fa310eee58d6aa5126784bad9e725eabfc1a5608e6",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794852357,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51855,
"message_url": "https://xray.tilda.pizza/alerts/messag |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51851 2026-11-16 17:35:25 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nhitl_tool: body raised after user approval \u2014 write outcome may be unconfirmed",
"message": "hitl_tool: body raised after user approval \u2014 write outcome may be unconfirmed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=f1c150aa732284dc559944c8afb10331&at=1794850521&msg=hitl_tool%3A+body+raised+after+user+approval+%E2%80%94+write+outcome+may+be+unconfirmed",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "hitl_tool: body raised after user approval \u2014 write outcome may be unconfirmed",
"trace": "f1c150aa732284dc559944c8afb10331",
"log": "stdout",
"ts": "2026-11-16T17:35:21.06099679Z"
},
"log_key": "bd91eede7d9cb35d4dc2cf2befee4168395d1b2a34c035082dc8cb29f8a65e9b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794850555,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51851,
"message_url": "https://xray.tilda.pizza/alerts/messages/51851"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51849 2026-11-16 17:35:23 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=f1c150aa732284dc559944c8afb10331&at=1794850520&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "f1c150aa732284dc559944c8afb10331",
"log": "stdout",
"ts": "2026-11-16T17:35:20.973466522Z"
},
"log_key": "77db170144f3de14233f801b01d071ae079a710620d377007912f895aa3c5c3b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794850556,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51849,
"message_url": "https://xray.tilda.pizza/alerts/messages/51849"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51847 2026-11-16 17:35:23 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ndeliverable '02.01.02.01 Protocol': Could not find the specified TMF location (with identifier '5a814a00-c154-4911-bd40-cd6d75cdf6a7') in study",
"message": "deliverable '02.01.02.01 Protocol': Could not find the specified TMF location (with identifier '5a814a00-c154-4911-bd40-cd6d75cdf6a7') in study",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=f1c150aa732284dc559944c8afb10331&at=1794850520&msg=deliverable+%2702.01.02.01+Protocol%27%3A+Could+not+find+the+specified+TMF+location+%28with+identifier+%275a814a00-c154-4911-bd40-cd6d75cdf6a7%27%29+in+study",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "deliverable '02.01.02.01 Protocol': Could not find the specified TMF location (with identifier '5a814a00-c154-4911-bd40-cd6d75cdf6a7') in study",
"trace": "f1c150aa732284dc559944c8afb10331",
"log": "stdout",
"ts": "2026-11-16T17:35:20.973257588Z"
},
"log_key": "8d0533dd19973d5f9c8a7fde152b58579227020799feb7d31280f0e903efc004",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794850553,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51847,
"message_url": "https://xray.tilda.pizza/alerts/messages/51847"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51843 2026-11-16 17:24:03 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=8fc2c3a4d5353f8fe1f44322b62df049&at=1794849837&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "8fc2c3a4d5353f8fe1f44322b62df049",
"log": "stderr",
"ts": "2026-11-16T17:23:57.994745492Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849873,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51843,
"message_url": "https://xray.tilda.pizza/alerts/messages/51843"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51841 2026-11-16 17:23:48 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=29f7ee1e3a1a6f44fa5aabd0290064bb&at=1794849822&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "29f7ee1e3a1a6f44fa5aabd0290064bb",
"log": "stderr",
"ts": "2026-11-16T17:23:42.399948659Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849858,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51841,
"message_url": "https://xray.tilda.pizza/alerts/messages/51841"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51839 2026-11-16 17:23:42 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=de09cc5dc623d6dfa6b6a48ff267e31c&at=1794849818&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "de09cc5dc623d6dfa6b6a48ff267e31c",
"log": "stdout",
"ts": "2026-11-16T17:23:38.682379936Z"
},
"log_key": "77db170144f3de14233f801b01d071ae079a710620d377007912f895aa3c5c3b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849853,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51839,
"message_url": "https://xray.tilda.pizza/alerts/messages/51839"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51837 2026-11-16 17:23:42 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\ndeliverable '05.02.02 Protocol Signature Page': Could not find SubArtifact name 'Protocol Signature Page' in study TMF structure",
"message": "deliverable '05.02.02 Protocol Signature Page': Could not find SubArtifact name 'Protocol Signature Page' in study TMF structure",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=de09cc5dc623d6dfa6b6a48ff267e31c&at=1794849818&msg=deliverable+%2705.02.02+Protocol+Signature+Page%27%3A+Could+not+find+SubArtifact+name+%27Protocol+Signature+Page%27+in+study+TMF+structure",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "deliverable '05.02.02 Protocol Signature Page': Could not find SubArtifact name 'Protocol Signature Page' in study TMF structure",
"trace": "de09cc5dc623d6dfa6b6a48ff267e31c",
"log": "stdout",
"ts": "2026-11-16T17:23:38.68207435Z"
},
"log_key": "5fdd382302b68700e8706d6f5d2516eee3fb138db1d8041340b363b9c21b5546",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849852,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51837,
"message_url": "https://xray.tilda.pizza/alerts/messages/51837"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51835 2026-11-16 17:23:33 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=29f7ee1e3a1a6f44fa5aabd0290064bb&at=1794849804&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "29f7ee1e3a1a6f44fa5aabd0290064bb",
"log": "stderr",
"ts": "2026-11-16T17:23:24.655256879Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849843,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51835,
"message_url": "https://xray.tilda.pizza/alerts/messages/51835"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51828 2026-11-16 17:11:32 UTC |
channel test | warning Error in chat-agent-postgres (tilda) on master
show the payload{
"title": "Error in chat-agent-postgres (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent-postgres (tilda) on master",
"text": "Error in chat-agent-postgres (tilda) on master\ntuple concurrently updated",
"message": "tuple concurrently updated",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent-postgres&namespace=tilda&at=1794849083&msg=tuple+concurrently+updated",
"log": {
"severity": "ERROR",
"service": "chat-agent-postgres",
"cluster": "master",
"namespace": "tilda",
"message": "tuple concurrently updated",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T17:11:23.535325768Z"
},
"log_key": "10657908ba30c930168a4f2d453c85a28d05a86adbcb245d77442a403f3f5320",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794849122,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51828,
"message_url": "https://xray.tilda.pizza/alerts/messages/51828"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51821 2026-11-16 16:49:29 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ntraining reconcile skipped user",
"message": "training reconcile skipped user",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847767&msg=training+reconcile+skipped+user",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "training reconcile skipped user",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:27.739419784Z"
},
"log_key": "b2622cff0dfdd5cfae74928aac1f5eddaf87b2bef1520d64661433f2f46edd21",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847800,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51821,
"message_url": "https://xray.tilda.pizza/alerts/messages/51821"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51819 2026-11-16 16:49:29 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ntraining reconcile completed with skipped users",
"message": "training reconcile completed with skipped users",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847767&msg=training+reconcile+completed+with+skipped+users",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "training reconcile completed with skipped users",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:27.739425954Z"
},
"log_key": "1cc512a62d10be4d2abeb6afefdda20b8af0bd8613ca30cd31199a973b99fd5c",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847799,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51819,
"message_url": "https://xray.tilda.pizza/alerts/messages/51819"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51817 2026-11-16 16:49:20 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to process db result",
"message": "failed to process db result",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847757&msg=failed+to+process+db+result",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to process db result",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:17.438444033Z"
},
"log_key": "335fe921f6bfd8809ae3faafb4e3a9c00c98be7820b32020511ea2e5fe1370c2",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847791,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51817,
"message_url": "https://xray.tilda.pizza/alerts/messages/51817"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51815 2026-11-16 16:49:20 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to bulk-write document nodes",
"message": "failed to bulk-write document nodes",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847757&msg=failed+to+bulk-write+document+nodes",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to bulk-write document nodes",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:17.438527243Z"
},
"log_key": "fc4b3b2ed72066969f4d34eefd8aa998743b28cda3ffc213904d5d25709471cf",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847792,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51815,
"message_url": "https://xray.tilda.pizza/alerts/messages/51815"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51813 2026-11-16 16:49:06 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to run cypher in transaction",
"message": "failed to run cypher in transaction",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847742&msg=failed+to+run+cypher+in+transaction",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to run cypher in transaction",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:02.222400679Z"
},
"log_key": "cdff4a7588b1873f0c0e472b4badfec98e9d3d6c50d0bdac125d51ad68656091",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847776,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": [],
"reserved_at": 1794847746
},
"message_id": 51813,
"message_url": "/alerts/messages/51813"
} |
#in-app delivered#dev-notifications sent |
opened | in-app: 1 inbox row(s) for [email protected] ok |
| #51811 2026-11-16 16:49:05 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to create document copy",
"message": "failed to create document copy",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847742&msg=failed+to+create+document+copy",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to create document copy",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:02.222465859Z"
},
"log_key": "b104153ad6e72398884577da63df8ef86e798f7fb0599051c4b7e2df8bc330c4",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847836,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51811,
"message_url": "https://xray.tilda.pizza/alerts/messages/51811"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51809 2026-11-16 16:49:05 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to add document to scope binders",
"message": "failed to add document to scope binders",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=f91346437409440b0cc533d78b9297f2&at=1794847742&msg=failed+to+add+document+to+scope+binders",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to add document to scope binders",
"trace": "f91346437409440b0cc533d78b9297f2",
"log": "stdout",
"ts": "2026-11-16T16:49:02.222454879Z"
},
"log_key": "ff188c0bb9937c347d6fad6568d12d3768eeea697e3edb793beaf797a76ef53b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847775,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51809,
"message_url": "https://xray.tilda.pizza/alerts/messages/51809"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51805 2026-11-16 16:38:49 UTC |
channel test | warning Error in chat-agent-postgres (tilda) on master
show the payload{
"title": "Error in chat-agent-postgres (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent-postgres (tilda) on master",
"text": "Error in chat-agent-postgres (tilda) on master\ntuple concurrently updated",
"message": "tuple concurrently updated",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent-postgres&namespace=tilda&at=1794847121&msg=tuple+concurrently+updated",
"log": {
"severity": "ERROR",
"service": "chat-agent-postgres",
"cluster": "master",
"namespace": "tilda",
"message": "tuple concurrently updated",
"trace": null,
"log": "stdout",
"ts": "2026-11-16T16:38:41.189144169Z"
},
"log_key": "10657908ba30c930168a4f2d453c85a28d05a86adbcb245d77442a403f3f5320",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794847159,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51805,
"message_url": "https://xray.tilda.pizza/alerts/messages/51805"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51802 2026-11-16 16:16:16 UTC |
channel test | warning Any error → notify me (master): 5 more messages held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 5 more messages held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 5 more messages held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in virtual-assistant (tilda) on master; Error in virtual-assistant (tilda) on master; Error in virtual-assistant (tilda) on master; Error in docproc (tilda) on master; Error in tilda on master",
"body": "The alert allows 20 an hour. Most recent held: Error in virtual-assistant (tilda) on master; Error in virtual-assistant (tilda) on master; Error in virtual-assistant (tilda) on master; Error in docproc (tilda) on master; Error in tilda on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794845806,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51802,
"message_url": "https://xray.tilda.pizza/alerts/messages/51802"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #51798 2026-11-16 15:37:56 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794843470",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T15:37:50.244302Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794843506,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51798,
"message_url": "https://xray.tilda.pizza/alerts/messages/51798"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51793 2026-11-16 15:23:48 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=f90dda10fb3522c0d397b4a227c17363&at=1794842625&msg=%5Breminder%5D+wo%3D97bcca47-6e42-49be-bcf8-715ee3a73d53+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ko%27-script+text+%28%27%ED%8C%80%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=97bcca47-6e42-49be-bcf8-715ee3a73d53 draft's English content failed validation (content (meant to stay English) contains 'ko'-script text ('\ud300') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "f90dda10fb3522c0d397b4a227c17363",
"log": "stderr",
"ts": "2026-11-16T15:23:45.85330047Z"
},
"log_key": "a21b1a376c74f5195c8af08090b36f6360bb500c051ade1192a1ebd1eec933bd",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794842628,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51793,
"message_url": "https://xray.tilda.pizza/alerts/messages/51793"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51791 2026-11-16 15:23:35 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=af3f6f3a82a2022995615de8fd39da6a&at=1794842609&msg=%5Breminder%5D+wo%3D48275f5e-57af-4670-bd15-2b08e179b69b+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27ja%27-script+text+%28%27%E6%A7%98%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=48275f5e-57af-4670-bd15-2b08e179b69b draft's English content failed validation (content (meant to stay English) contains 'ja'-script text ('\u69d8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "af3f6f3a82a2022995615de8fd39da6a",
"log": "stderr",
"ts": "2026-11-16T15:23:29.03636552Z"
},
"log_key": "db8c99fb601ac7cbb971b15f6bd7a430dcf42331e2810535259637448bfb7406",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794842615,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51791,
"message_url": "https://xray.tilda.pizza/alerts/messages/51791"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51789 2026-11-16 15:23:18 UTC |
channel test | warning Error in virtual-assistant (tilda) on master
show the payload{
"title": "Error in virtual-assistant (tilda) on master",
"severity": "warning",
"headline": "Error in virtual-assistant (tilda) on master",
"text": "Error in virtual-assistant (tilda) on master\n[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=virtual-assistant&namespace=tilda&trace=af3f6f3a82a2022995615de8fd39da6a&at=1794842592&msg=%5Breminder%5D+wo%3D850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4+draft%27s+English+content+failed+validation+%28content+%28meant+to+stay+English%29+contains+%27zh-tw%27-script+text+%28%27%E6%82%A8%27%29+%E2%80%94+likely+bled+in+from+a+goal%2Finstructions+language+directive+that+should+only+govern+translated_content%29+%E2%80%94+skipping+this+cycle",
"log": {
"severity": "ERROR",
"service": "virtual-assistant",
"cluster": "master",
"namespace": "tilda",
"message": "[reminder] wo=850dceb3-7f32-45e9-9c6a-d0ed81b0b6d4 draft's English content failed validation (content (meant to stay English) contains 'zh-tw'-script text ('\u60a8') \u2014 likely bled in from a goal/instructions language directive that should only govern translated_content) \u2014 skipping this cycle",
"trace": "af3f6f3a82a2022995615de8fd39da6a",
"log": "stderr",
"ts": "2026-11-16T15:23:12.849396465Z"
},
"log_key": "df998a0f4c524270d6d64977e8c1d486629224b4953daef9c122c60f8cfc74ad",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794842598,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51789,
"message_url": "https://xray.tilda.pizza/alerts/messages/51789"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51787 2026-11-16 15:20:53 UTC |
channel test | warning Error in docproc (tilda) on master
show the payload{
"title": "Error in docproc (tilda) on master",
"severity": "warning",
"headline": "Error in docproc (tilda) on master",
"text": "Error in docproc (tilda) on master\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docproc&namespace=tilda&at=1794842420&msg=Traceback+%28most+recent+call+last%29%3A%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fstarlette%2Frouting.py%22%2C+line+645%2C+in+lifespan%0A++++await+receive%28%29%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fsite-packages%2Fuvicorn%2Flifespan%2Fon.py%22%2C+line+137%2C+in+receive%0A++++return+await+self.receive_queue.get%28%29%0A+++++++++++%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%5E%0A++File+%22%2Fusr%2Flocal%2Flib%2Fpython3.12%2Fasyncio%2Fqueues.py%22%2C+line+158%2C+in+get%0A+++",
"log": {
"severity": "ERROR",
"service": "docproc",
"cluster": "master",
"namespace": "tilda",
"message": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.12/site-packages/starlette/routing.py\", line 645, in lifespan\n await receive()\n File \"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py\", line 137, in receive\n return await self.receive_queue.get()\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.12/asyncio/queues.py\", line 158, in get\n await getter\nasyncio.exceptions.CancelledError\n",
"trace": null,
"log": "stderr",
"ts": "2026-11-16T15:20:20.196500642Z"
},
"log_key": "07c70b2564d01b69c3cb76ec353977c171c0a9520a4757e2fb1931781a092a22",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794842453,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51787,
"message_url": "https://xray.tilda.pizza/alerts/messages/51787"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51785 2026-11-16 15:20:22 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794842419",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T15:20:19.888297Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794842422,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51785,
"message_url": "https://xray.tilda.pizza/alerts/messages/51785"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51783 2026-11-16 15:16:08 UTC |
channel test | warning Any error → notify me (master): 2 more messages held by the rate limit in the last hour
show the payload{
"title": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour",
"text": "Any error \u2192 notify me (master): 2 more messages held by the rate limit in the last hour\nThe alert allows 20 an hour. Most recent held: Error in chat-agent (tilda) on master; Error in docsvc (tilda) on master",
"body": "The alert allows 20 an hour. Most recent held: Error in chat-agent (tilda) on master; Error in docsvc (tilda) on master",
"url": "https://xray.tilda.pizza/alerts/8",
"severity": "warning",
"_scope": {},
"_job": {
"finding": null,
"force": true,
"retry_at": 1794842198,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51783,
"message_url": "https://xray.tilda.pizza/alerts/messages/51783"
} |
#dev-notifications sent#in-app delivered |
rollup | ok in-app: 1 inbox row(s) for [email protected] |
| #51781 2026-11-16 15:12:08 UTC |
channel test | warning Error in chat-agent (tilda) on master
show the payload{
"title": "Error in chat-agent (tilda) on master",
"severity": "warning",
"headline": "Error in chat-agent (tilda) on master",
"text": "Error in chat-agent (tilda) on master\nstatus_tool: body raised, surfaced to the LLM as a tool error",
"message": "status_tool: body raised, surfaced to the LLM as a tool error",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=chat-agent&namespace=tilda&trace=ecc42651b4fb2173c96064c23d967ded&at=1794841925&msg=status_tool%3A+body+raised%2C+surfaced+to+the+LLM+as+a+tool+error",
"log": {
"severity": "ERROR",
"service": "chat-agent",
"cluster": "master",
"namespace": "tilda",
"message": "status_tool: body raised, surfaced to the LLM as a tool error",
"trace": "ecc42651b4fb2173c96064c23d967ded",
"log": "stdout",
"ts": "2026-11-16T15:12:05.668268303Z"
},
"log_key": "9aef469f9857e748a2707df51313a5a76a51361e5944593fe4b87fbe22ce401b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841928,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51781,
"message_url": "https://xray.tilda.pizza/alerts/messages/51781"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51779 2026-11-16 15:11:13 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nduplicate subfolder name 'Chat uploads' found in the parent folder(asas22)",
"message": "duplicate subfolder name 'Chat uploads' found in the parent folder(asas22)",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=77e99418fab53fe50e3919c29f4a9939&at=1794841868&msg=duplicate+subfolder+name+%27Chat+uploads%27+found+in+the+parent+folder%28asas22%29",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "duplicate subfolder name 'Chat uploads' found in the parent folder(asas22)",
"trace": "77e99418fab53fe50e3919c29f4a9939",
"log": "stdout",
"ts": "2026-11-16T15:11:08.512140261Z"
},
"log_key": "37ca8fdd5dbbe6a9c5470e7b82efb8334cee26ae5cb7c00d1d8b4ab95b5f460f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841873,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51779,
"message_url": "https://xray.tilda.pizza/alerts/messages/51779"
} |
#dev-notifications capped#in-app capped |
opened | over this alert's limit of 20 messages an hour — counted in its hourly roll-up over this alert's limit of 20 messages an hour — counted in its hourly roll-up |
| #51777 2026-11-16 15:11:13 UTC |
channel test | warning Error in docsvc (tilda) on master
show the payload{
"title": "Error in docsvc (tilda) on master",
"severity": "warning",
"headline": "Error in docsvc (tilda) on master",
"text": "Error in docsvc (tilda) on master\nRequest completed",
"message": "Request completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=docsvc&namespace=tilda&trace=77e99418fab53fe50e3919c29f4a9939&at=1794841868&msg=Request+completed",
"log": {
"severity": "ERROR",
"service": "docsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Request completed",
"trace": "77e99418fab53fe50e3919c29f4a9939",
"log": "stdout",
"ts": "2026-11-16T15:11:08.51265401Z"
},
"log_key": "77db170144f3de14233f801b01d071ae079a710620d377007912f895aa3c5c3b",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841903,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51777,
"message_url": "https://xray.tilda.pizza/alerts/messages/51777"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51773 2026-11-16 15:06:21 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nevalResult is not a bool",
"message": "evalResult is not a bool",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=68953fcafc57fed747bf195db337bec4&at=1794841576&msg=evalResult+is+not+a+bool",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "evalResult is not a bool",
"trace": "68953fcafc57fed747bf195db337bec4",
"log": "stdout",
"ts": "2026-11-16T15:06:16.400986351Z"
},
"log_key": "5fe49d3829c318f25ee6cc600a4c5170e367760f7a03b10d87afef9e46d17198",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841612,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51773,
"message_url": "https://xray.tilda.pizza/alerts/messages/51773"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51771 2026-11-16 15:06:21 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nJob completed",
"message": "Job completed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=68953fcafc57fed747bf195db337bec4&at=1794841576&msg=Job+completed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "Job completed",
"trace": "68953fcafc57fed747bf195db337bec4",
"log": "stdout",
"ts": "2026-11-16T15:06:16.450968937Z"
},
"log_key": "c74deab9b0ebaca0dfd6164704293554f866fed15541770acc539e74f689ff56",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841611,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51771,
"message_url": "https://xray.tilda.pizza/alerts/messages/51771"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51769 2026-11-16 15:06:16 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nfailed to evaluate and raise auto query for participant question",
"message": "failed to evaluate and raise auto query for participant question",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=68953fcafc57fed747bf195db337bec4&at=1794841575&msg=failed+to+evaluate+and+raise+auto+query+for+participant+question",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to evaluate and raise auto query for participant question",
"trace": "68953fcafc57fed747bf195db337bec4",
"log": "stdout",
"ts": "2026-11-16T15:06:15.982033245Z"
},
"log_key": "6bd3587402488be63611acc5599dc01571e825e5157878dd4c0a52de2538572f",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841669,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51769,
"message_url": "https://xray.tilda.pizza/alerts/messages/51769"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51767 2026-11-16 15:06:16 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nauto query re-evaluation failed",
"message": "auto query re-evaluation failed",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=68953fcafc57fed747bf195db337bec4&at=1794841576&msg=auto+query+re-evaluation+failed",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "auto query re-evaluation failed",
"trace": "68953fcafc57fed747bf195db337bec4",
"log": "stdout",
"ts": "2026-11-16T15:06:16.014284998Z"
},
"log_key": "ab591fe230c52e574850a2b26a9eb3dcbbec2251f9d65aa2c21a908c4a887519",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841607,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51767,
"message_url": "https://xray.tilda.pizza/alerts/messages/51767"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51765 2026-11-16 15:06:16 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\nno result from branch evaluation",
"message": "no result from branch evaluation",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=68953fcafc57fed747bf195db337bec4&at=1794841575&msg=no+result+from+branch+evaluation",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "no result from branch evaluation",
"trace": "68953fcafc57fed747bf195db337bec4",
"log": "stdout",
"ts": "2026-11-16T15:06:15.983116004Z"
},
"log_key": "99389d22a81127e2d8a7b9897343d56e8b9ff07cd2a5c2eaedff03af61df8f87",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841606,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51765,
"message_url": "https://xray.tilda.pizza/alerts/messages/51765"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51763 2026-11-16 15:03:45 UTC |
channel test | warning Error in tilda on master
show the payload{
"title": "Error in tilda on master",
"severity": "warning",
"headline": "Error in tilda on master",
"text": "Error in tilda on master\n",
"message": "",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&namespace=tilda&at=1794841418",
"log": {
"severity": "ERROR",
"service": "",
"cluster": "master",
"namespace": "tilda",
"message": "",
"trace": null,
"log": "client-accesslog-stackdriver",
"ts": "2026-11-16T15:03:38.847149Z"
},
"log_key": "1f4f069e1a171983e24b16c6dedd7d5e20d66231f957c9e8d6284b833a9f6773",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794841455,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51763,
"message_url": "https://xray.tilda.pizza/alerts/messages/51763"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51760 2026-11-16 14:40:40 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\nfailed to get organization",
"message": "failed to get organization",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&trace=93488f93b8dd7a3caadee5f7ebc6e387&at=1794840036&msg=failed+to+get+organization",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "failed to get organization",
"trace": "93488f93b8dd7a3caadee5f7ebc6e387",
"log": "stdout",
"ts": "2026-11-16T14:40:36.302581354Z"
},
"log_key": "b2d281d29449178cf763946734aa51c22d0bbd67382f6bb448313f36860dd2da",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794840072,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51760,
"message_url": "https://xray.tilda.pizza/alerts/messages/51760"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51758 2026-11-16 14:40:40 UTC |
channel test | warning Error in regsvc (tilda) on master
show the payload{
"title": "Error in regsvc (tilda) on master",
"severity": "warning",
"headline": "Error in regsvc (tilda) on master",
"text": "Error in regsvc (tilda) on master\nworkflow not found",
"message": "workflow not found",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=regsvc&namespace=tilda&trace=93488f93b8dd7a3caadee5f7ebc6e387&at=1794840036&msg=workflow+not+found",
"log": {
"severity": "ERROR",
"service": "regsvc",
"cluster": "master",
"namespace": "tilda",
"message": "workflow not found",
"trace": "93488f93b8dd7a3caadee5f7ebc6e387",
"log": "stdout",
"ts": "2026-11-16T14:40:36.744237801Z"
},
"log_key": "4e91e1e230dceefc863a41b3059f37f351f0e1d1ef7c36c972b566616e1124cf",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794840071,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51758,
"message_url": "https://xray.tilda.pizza/alerts/messages/51758"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |
| #51756 2026-11-16 14:38:27 UTC |
channel test | warning Error in execsvc (tilda) on master
show the payload{
"title": "Error in execsvc (tilda) on master",
"severity": "warning",
"headline": "Error in execsvc (tilda) on master",
"text": "Error in execsvc (tilda) on master\ncould not list DutyPlans for Study cf10408f-05d3-441e-b7c6-8b59a1139b56: passed Study was not found",
"message": "could not list DutyPlans for Study cf10408f-05d3-441e-b7c6-8b59a1139b56: passed Study was not found",
"recipients": [
"[email protected]"
],
"url": "/logs/one?cluster=master&sev=ERROR&service=execsvc&namespace=tilda&trace=58cd1f309158f891410b19026c39da5e&at=1794839902&msg=could+not+list+DutyPlans+for+Study+cf10408f-05d3-441e-b7c6-8b59a1139b56%3A+passed+Study+was+not+found",
"log": {
"severity": "ERROR",
"service": "execsvc",
"cluster": "master",
"namespace": "tilda",
"message": "could not list DutyPlans for Study cf10408f-05d3-441e-b7c6-8b59a1139b56: passed Study was not found",
"trace": "58cd1f309158f891410b19026c39da5e",
"log": "stdout",
"ts": "2026-11-16T14:38:22.648242824Z"
},
"log_key": "c7bf0839ab9ad96050b7b15ad41c2b58e8df7a0a9481bc783ae61aa89e0c9f1d",
"_scope": {
"cluster:master": true
},
"_job": {
"finding": null,
"force": false,
"retry_at": 1794839937,
"route_signature": "{\"clusters\": [\"master\"], \"kind\": \"log\", \"kinds\": [], \"min_severity\": \"warning\", \"mode\": \"immediate\", \"services\": \"*\", \"spec\": {\"namespaces\": [\"tilda\"]}}",
"digest_items": []
},
"message_id": 51756,
"message_url": "https://xray.tilda.pizza/alerts/messages/51756"
} |
#dev-notifications sent#in-app delivered |
opened | ok in-app: 1 inbox row(s) for [email protected] |