25b - 监控与告警

┌─────────────────────────────────────────────────────────┐
│                    告警通知层                              │
│   Alertmanager → Slack / Telegram / Email / PagerDuty    │
├─────────────────────────────────────────────────────────┤
│                    可视化层                                │
│   Grafana 仪表板（基础设施 + Agent 业务指标）              │
├─────────────────────────────────────────────────────────┤
│                    指标存储层                              │
│   Prometheus（时序数据库，拉取模式）                       │
├─────────────────────────────────────────────────────────┤
│                    指标采集层                              │
│   Node Exporter    cAdvisor    自定义 /metrics 端点       │
│   (主机指标)      (容器指标)   (Agent 业务指标)           │
├─────────────────────────────────────────────────────────┤
│                    被监控目标                              │
│   VPS 主机    Docker 容器    OpenClaw Gateway + Agents    │
└─────────────────────────────────────────────────────────┘

# docker-compose.monitoring.yml — 监控栈
# 与主 docker-compose.yml 配合使用：
# docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d
 
version: '3.8'
 
services:
  # ============================================
  # Prometheus — 指标采集与存储
  # ============================================
  prometheus:
    image: prom/prometheus:v3.4.0
    container_name: openclaw-prometheus
    restart: unless-stopped
    user: "65534:65534"  # nobody 用户
    ports:
      - "127.0.0.1:9090:9090"
    volumes:
      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./monitoring/prometheus/rules/:/etc/prometheus/rules/:ro
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'          # 支持热重载配置
      - '--web.enable-admin-api'
    networks:
      - openclaw-net
    logging:
      driver: json-file
      options:
        max-size: "20m"
        max-file: "3"
 
  # ============================================
  # Grafana — 可视化仪表板
  # ============================================
  grafana:
    image: grafana/grafana:11.6.0
    container_name: openclaw-grafana
    restart: unless-stopped
    ports:
      - "127.0.0.1:3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_SERVER_ROOT_URL=https://grafana.yourdomain.com
    volumes:
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:ro
      - ./monitoring/grafana/dashboards/:/var/lib/grafana/dashboards/:ro
    networks:
      - openclaw-net
    depends_on:
      - prometheus
    logging:
      driver: json-file
      options:
        max-size: "20m"
        max-file: "3"
 
  # ============================================
  # Alertmanager — 告警路由与通知
  # ============================================
  alertmanager:
    image: prom/alertmanager:v0.28.1
    container_name: openclaw-alertmanager
    restart: unless-stopped
    ports:
      - "127.0.0.1:9093:9093"
    volumes:
      - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
      - alertmanager_data:/alertmanager
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
    networks:
      - openclaw-net
    logging:
      driver: json-file
      options:
        max-size: "10m"
        max-file: "3"
 
  # ============================================
  # Node Exporter — 主机指标
  # ============================================
  node-exporter:
    image: prom/node-exporter:v1.9.1
    container_name: openclaw-node-exporter
    restart: unless-stopped
    pid: host
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    networks:
      - openclaw-net
 
  # ============================================
  # cAdvisor — 容器指标
  # ============================================
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.52.1
    container_name: openclaw-cadvisor
    restart: unless-stopped
    privileged: true
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    networks:
      - openclaw-net
 
volumes:
  prometheus_data:
  grafana_data:
  alertmanager_data:

openclaw-deploy/
├── docker-compose.yml                    # 主服务
├── docker-compose.monitoring.yml         # 监控栈
├── monitoring/
│   ├── prometheus/
│   │   ├── prometheus.yml                # Prometheus 主配置
│   │   └── rules/
│   │       ├── infrastructure.yml        # 基础设施告警规则
│   │       └── openclaw-agent.yml        # Agent 业务告警规则
│   ├── alertmanager/
│   │   └── alertmanager.yml              # 告警路由配置
│   └── grafana/
│       ├── provisioning/
│       │   ├── datasources/
│       │   │   └── prometheus.yml        # 自动配置数据源
│       │   └── dashboards/
│       │       └── dashboards.yml        # 仪表板自动发现
│       └── dashboards/
│           ├── openclaw-overview.json     # OpenClaw 总览仪表板
│           └── infrastructure.json        # 基础设施仪表板
└── .env

# monitoring/prometheus/prometheus.yml
global:
  scrape_interval: 15s          # 默认采集间隔
  evaluation_interval: 15s      # 规则评估间隔
  scrape_timeout: 10s
 
# 告警规则文件
rule_files:
  - /etc/prometheus/rules/*.yml
 
# Alertmanager 配置
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093
 
# 采集目标
scrape_configs:
  # Prometheus 自身指标
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
 
  # Node Exporter — 主机指标
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        replacement: 'openclaw-vps'
 
  # cAdvisor — 容器指标
  - job_name: 'cadvisor'
    static_configs:
      - targets: ['cadvisor:8080']
 
  # OpenClaw Gateway — 应用指标
  # 如果 OpenClaw 暴露 /metrics 端点
  - job_name: 'openclaw-gateway'
    static_configs:
      - targets: ['openclaw-gateway:18789']
    metrics_path: /metrics
    scrape_interval: 30s
    # 如果需要认证
    # authorization:
    #   credentials: '${OPENCLAW_GATEWAY_TOKEN}'
 
  # OpenClaw 健康检查探针
  - job_name: 'openclaw-health'
    metrics_path: /health
    static_configs:
      - targets: ['openclaw-gateway:18789']
    scrape_interval: 30s

# monitoring/grafana/provisioning/datasources/prometheus.yml
apiVersion: 1
 
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false
    jsonData:
      timeInterval: '15s'
      httpMethod: POST

# monitoring/grafana/provisioning/dashboards/dashboards.yml
apiVersion: 1
 
providers:
  - name: 'default'
    orgId: 1
    folder: 'OpenClaw'
    type: file
    disableDeletion: false
    editable: true
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: false

# 创建目录结构
mkdir -p monitoring/{prometheus/rules,alertmanager,grafana/provisioning/{datasources,dashboards},grafana/dashboards}
 
# 启动（与主服务一起）
docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d
 
# 验证各组件
curl -s http://127.0.0.1:9090/-/healthy     # Prometheus
curl -s http://127.0.0.1:3000/api/health     # Grafana
curl -s http://127.0.0.1:9093/-/healthy      # Alertmanager
 
# 检查 Prometheus 目标状态
curl -s http://127.0.0.1:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health: .health}'

#!/usr/bin/env python3
"""
openclaw_exporter.py — OpenClaw Prometheus Exporter
采集 Agent 业务指标并暴露为 Prometheus 格式
"""
 
import os
import time
import json
import requests
from prometheus_client import start_http_server, Gauge, Counter, Histogram
 
# ============================================
# 指标定义
# ============================================
 
# 会话指标
ACTIVE_SESSIONS = Gauge(
    'openclaw_active_sessions',
    'Number of currently active agent sessions'
)
 
# 任务指标
TASKS_TOTAL = Counter(
    'openclaw_tasks_total',
    'Total number of tasks executed',
    ['status']  # success, failure, timeout
)
 
TASK_DURATION = Histogram(
    'openclaw_task_duration_seconds',
    'Task execution duration in seconds',
    buckets=[1, 5, 10, 30, 60, 120, 300, 600]
)
 
# LLM 指标
LLM_REQUESTS = Counter(
    'openclaw_llm_requests_total',
    'Total LLM API requests',
    ['provider', 'model']
)
 
LLM_TOKENS = Counter(
    'openclaw_llm_tokens_total',
    'Total tokens consumed',
    ['provider', 'model', 'type']  # type: input/output
)
 
LLM_COST = Counter(
    'openclaw_llm_cost_dollars',
    'Estimated LLM API cost in USD',
    ['provider', 'model']
)
 
LLM_LATENCY = Histogram(
    'openclaw_llm_request_duration_seconds',
    'LLM API request duration',
    ['provider', 'model'],
    buckets=[0.5, 1, 2, 5, 10, 30, 60]
)
 
# 工具调用指标
TOOL_CALLS = Counter(
    'openclaw_tool_calls_total',
    'Total tool invocations',
    ['tool_name']
)
 
TOOL_ERRORS = Counter(
    'openclaw_tool_errors_total',
    'Total tool invocation errors',
    ['tool_name', 'error_type']
)
 
# 健康状态
HEALTH_STATUS = Gauge(
    'openclaw_health_status',
    'Gateway health status (1=healthy, 0=unhealthy)'
)
 
GATEWAY_UPTIME = Gauge(
    'openclaw_gateway_uptime_seconds',
    'Gateway uptime in seconds'
)
 
# ============================================
# 采集逻辑
# ============================================
 
GATEWAY_URL = os.getenv('OPENCLAW_GATEWAY_URL', 'http://127.0.0.1:18789')
GATEWAY_TOKEN = os.getenv('OPENCLAW_GATEWAY_TOKEN', '')
 
def fetch_api(endpoint):
    """从 OpenClaw API 获取数据"""
    headers = {}
    if GATEWAY_TOKEN:
        headers['Authorization'] = f'Bearer {GATEWAY_TOKEN}'
    try:
        resp = requests.get(
            f'{GATEWAY_URL}{endpoint}',
            headers=headers,
            timeout=10
        )
        resp.raise_for_status()
        return resp.json()
    except Exception as e:
        print(f"[WARN] Failed to fetch {endpoint}: {e}")
        return None
 
def collect_health():
    """采集健康状态"""
    data = fetch_api('/health')
    if data and data.get('status') == 'ok':
        HEALTH_STATUS.set(1)
        if 'uptime' in data:
            GATEWAY_UPTIME.set(data['uptime'])
    else:
        HEALTH_STATUS.set(0)
 
def collect_sessions():
    """采集会话指标"""
    data = fetch_api('/api/status')
    if data:
        sessions = data.get('activeSessions', 0)
        ACTIVE_SESSIONS.set(sessions)
 
def collect_metrics():
    """主采集循环"""
    collect_health()
    collect_sessions()
    # 根据 OpenClaw API 的实际响应结构
    # 扩展更多指标采集逻辑
 
if __name__ == '__main__':
    port = int(os.getenv('EXPORTER_PORT', '9101'))
    interval = int(os.getenv('COLLECT_INTERVAL', '30'))
 
    print(f"[INFO] Starting OpenClaw Exporter on :{port}")
    start_http_server(port)
 
    while True:
        collect_metrics()
        time.sleep(interval)

# monitoring/exporter/Dockerfile
FROM python:3.12-slim
WORKDIR /app
RUN pip install --no-cache-dir prometheus-client requests
COPY openclaw_exporter.py .
EXPOSE 9101
CMD ["python", "openclaw_exporter.py"]

  openclaw-exporter:
    build: ./monitoring/exporter
    container_name: openclaw-exporter
    restart: unless-stopped
    environment:
      - OPENCLAW_GATEWAY_URL=http://openclaw-gateway:18789
      - OPENCLAW_GATEWAY_TOKEN=${OPENCLAW_GATEWAY_TOKEN}
      - EXPORTER_PORT=9101
      - COLLECT_INTERVAL=30
    networks:
      - openclaw-net
    depends_on:
      - openclaw-gateway

  - job_name: 'openclaw-exporter'
    static_configs:
      - targets: ['openclaw-exporter:9101']
    scrape_interval: 30s

# monitoring/prometheus/rules/infrastructure.yml
groups:
  - name: infrastructure
    rules:
      # ---- 主机级告警 ----
 
      - alert: HighCpuUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
          component: infrastructure
        annotations:
          summary: "CPU 使用率过高 ({{ $labels.instance }})"
          description: "CPU 使用率已超过 80%，当前值 {{ $value | printf \"%.1f\" }}%。持续 5 分钟。"
          runbook: "检查是否有失控的 Agent 会话，考虑增加资源限制或升级 VPS。"
 
      - alert: CriticalCpuUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
        for: 2m
        labels:
          severity: critical
          component: infrastructure
        annotations:
          summary: "🔴 CPU 使用率危急 ({{ $labels.instance }})"
          description: "CPU 使用率超过 95%，当前值 {{ $value | printf \"%.1f\" }}%。系统可能无响应。"
 
      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85
        for: 5m
        labels:
          severity: warning
          component: infrastructure
        annotations:
          summary: "内存使用率过高 ({{ $labels.instance }})"
          description: "内存使用率超过 85%，当前值 {{ $value | printf \"%.1f\" }}%。"
          runbook: "检查 Agent 会话数量，考虑限制并发会话或增加内存。"
 
      - alert: DiskSpaceRunningLow
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 80
        for: 10m
        labels:
          severity: warning
          component: infrastructure
        annotations:
          summary: "磁盘空间不足 ({{ $labels.instance }})"
          description: "根分区使用率超过 80%，当前值 {{ $value | printf \"%.1f\" }}%。"
          runbook: "清理旧日志和备份，检查 Docker 镜像缓存：docker system prune"
 
      - alert: DiskSpaceCritical
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 95
        for: 5m
        labels:
          severity: critical
          component: infrastructure
        annotations:
          summary: "🔴 磁盘空间即将耗尽 ({{ $labels.instance }})"
          description: "根分区使用率超过 95%。OpenClaw 可能无法写入数据。"
 
      # ---- 容器级告警 ----
 
      - alert: ContainerHighMemory
        expr: container_memory_usage_bytes{name="openclaw-gateway"} / container_spec_memory_limit_bytes{name="openclaw-gateway"} * 100 > 85
        for: 5m
        labels:
          severity: warning
          component: container
        annotations:
          summary: "OpenClaw 容器内存接近上限"
          description: "Gateway 容器内存使用率 {{ $value | printf \"%.1f\" }}%，接近 deploy.resources.limits。"
 
      - alert: ContainerRestarted
        expr: increase(container_restart_count{name="openclaw-gateway"}[1h]) > 2
        labels:
          severity: critical
          component: container
        annotations:
          summary: "🔴 OpenClaw 容器频繁重启"
          description: "Gateway 容器在过去 1 小时内重启了 {{ $value }} 次。"
          runbook: "检查容器日志：docker logs openclaw-gateway --tail 100"

# monitoring/prometheus/rules/openclaw-agent.yml
groups:
  - name: openclaw-agent
    rules:
      # ---- Gateway 健康 ----
 
      - alert: OpenClawGatewayDown
        expr: openclaw_health_status == 0
        for: 1m
        labels:
          severity: critical
          component: openclaw
        annotations:
          summary: "🔴 OpenClaw Gateway 不可用"
          description: "Gateway 健康检查失败超过 1 分钟。所有 Agent 会话已中断。"
          runbook: |
            1. 检查容器状态：docker compose ps
            2. 查看日志：docker compose logs --tail 50 openclaw-gateway
            3. 尝试重启：docker compose restart openclaw-gateway
            4. 检查 API 密钥是否过期
 
      - alert: OpenClawHealthCheckTimeout
        expr: up{job="openclaw-gateway"} == 0
        for: 2m
        labels:
          severity: critical
          component: openclaw
        annotations:
          summary: "🔴 OpenClaw 健康检查端点无响应"
          description: "Prometheus 无法连接到 OpenClaw Gateway 的 /metrics 端点。"
 
      # ---- 任务执行 ----
 
      - alert: HighTaskFailureRate
        expr: |
          rate(openclaw_tasks_total{status="failure"}[15m])
          / rate(openclaw_tasks_total[15m]) > 0.3
        for: 10m
        labels:
          severity: warning
          component: openclaw
        annotations:
          summary: "Agent 任务失败率过高"
          description: "过去 15 分钟任务失败率超过 30%，当前值 {{ $value | printf \"%.1f\" }}%。"
          runbook: "检查 Agent 日志中的错误模式，可能是 API 限流或工具故障。"
 
      - alert: TaskExecutionSlow
        expr: histogram_quantile(0.95, rate(openclaw_task_duration_seconds_bucket[15m])) > 300
        for: 10m
        labels:
          severity: warning
          component: openclaw
        annotations:
          summary: "Agent 任务执行缓慢"
          description: "P95 任务执行时间超过 5 分钟（{{ $value | printf \"%.0f\" }}s）。"
 
      # ---- LLM API ----
 
      - alert: LLMApiHighLatency
        expr: histogram_quantile(0.95, rate(openclaw_llm_request_duration_seconds_bucket[10m])) > 30
        for: 5m
        labels:
          severity: warning
          component: llm
        annotations:
          summary: "LLM API 响应缓慢"
          description: "P95 LLM 请求延迟超过 30 秒（{{ $value | printf \"%.1f\" }}s）。可能是模型提供商限流。"
 
      - alert: LLMApiErrors
        expr: rate(openclaw_llm_requests_total{status="error"}[10m]) > 0.1
        for: 5m
        labels:
          severity: warning
          component: llm
        annotations:
          summary: "LLM API 错误率上升"
          description: "LLM API 错误率超过 0.1 req/s。检查 API 密钥和配额。"
 
      - alert: HighTokenBurn
        expr: rate(openclaw_llm_tokens_total[1h]) * 3600 > 500000
        for: 30m
        labels:
          severity: warning
          component: cost
        annotations:
          summary: "⚠️ Token 消耗速率异常"
          description: "过去 1 小时 Token 消耗速率超过 50 万/小时。可能有失控的 Agent 循环。"
          runbook: "检查活跃会话，终止异常 Agent：通过 Control UI 或 API 停止可疑会话。"
 
      - alert: DailyCostExceeded
        expr: increase(openclaw_llm_cost_dollars[24h]) > 50
        labels:
          severity: critical
          component: cost
        annotations:
          summary: "🔴 日 API 成本超过预算"
          description: "过去 24 小时 LLM API 成本超过 $50（当前 ${{ $value | printf \"%.2f\" }}）。"
 
      # ---- 工具调用 ----
 
      - alert: ToolCallFailureSpike
        expr: |
          rate(openclaw_tool_errors_total[10m])
          / rate(openclaw_tool_calls_total[10m]) > 0.5
        for: 5m
        labels:
          severity: warning
          component: openclaw
        annotations:
          summary: "工具调用失败率飙升"
          description: "工具调用失败率超过 50%。可能是外部服务故障。"
 
      # ---- 会话健康 ----
 
      - alert: NoActiveSessions
        expr: openclaw_active_sessions == 0
        for: 15m
        labels:
          severity: info
          component: openclaw
        annotations:
          summary: "无活跃 Agent 会话"
          description: "过去 15 分钟没有活跃的 Agent 会话。如果预期 24/7 运行，请检查。"

# monitoring/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  # SMTP 配置（邮件通知）
  smtp_smarthost: 'smtp.gmail.com:587'
  smtp_from: 'openclaw-alerts@yourdomain.com'
  smtp_auth_username: 'openclaw-alerts@yourdomain.com'
  smtp_auth_password: 'your-app-password'
  smtp_require_tls: true
 
# 告警模板
templates:
  - '/etc/alertmanager/templates/*.tmpl'
 
# 路由规则
route:
  # 默认接收者
  receiver: 'slack-default'
  # 分组策略：按 alertname + component 分组
  group_by: ['alertname', 'component']
  group_wait: 30s        # 等待 30 秒收集同组告警
  group_interval: 5m     # 同组告警间隔 5 分钟
  repeat_interval: 4h    # 未恢复告警每 4 小时重复
 
  routes:
    # Critical 告警 → Telegram + Email（立即通知）
    - match:
        severity: critical
      receiver: 'critical-multi'
      group_wait: 10s
      repeat_interval: 1h
 
    # 成本告警 → 专用通道
    - match:
        component: cost
      receiver: 'cost-alerts'
      group_wait: 1m
      repeat_interval: 6h
 
    # Info 级别 → 仅 Slack
    - match:
        severity: info
      receiver: 'slack-default'
      repeat_interval: 24h
 
# 抑制规则：Critical 触发时抑制同组件的 Warning
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['component']
 
# 接收者配置
receivers:
  # Slack 默认通道
  - name: 'slack-default'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
        channel: '#openclaw-alerts'
        send_resolved: true
        title: '{{ if eq .Status "firing" }}🔥{{ else }}✅{{ end }} {{ .CommonLabels.alertname }}'
        text: >-
          {{ range .Alerts }}
          *{{ .Annotations.summary }}*
          {{ .Annotations.description }}
          {{ if .Annotations.runbook }}📋 *处理步骤:* {{ .Annotations.runbook }}{{ end }}
          {{ end }}
 
  # Critical 多通道（Telegram + Email）
  - name: 'critical-multi'
    telegram_configs:
      - bot_token: 'YOUR_TELEGRAM_BOT_TOKEN'
        chat_id: -1001234567890
        parse_mode: 'HTML'
        message: |
          {{ if eq .Status "firing" }}🚨 <b>CRITICAL ALERT</b>{{ else }}✅ <b>RESOLVED</b>{{ end }}
 
          {{ range .Alerts }}
          <b>{{ .Annotations.summary }}</b>
          {{ .Annotations.description }}
          {{ end }}
    email_configs:
      - to: 'oncall@yourdomain.com'
        send_resolved: true
        headers:
          Subject: '{{ if eq .Status "firing" }}🔴 CRITICAL{{ else }}✅ Resolved{{ end }}: {{ .CommonLabels.alertname }}'
 
  # 成本告警
  - name: 'cost-alerts'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
        channel: '#openclaw-costs'
        send_resolved: true
        title: '💰 {{ .CommonLabels.alertname }}'
        text: >-
          {{ range .Alerts }}
          {{ .Annotations.description }}
          {{ end }}

你是一个 DevOps 专家。请为我的 [平台名称] 生成 Alertmanager 配置。

要求：
- 通知通道：[Slack/Telegram/Email/PagerDuty/Discord]
- Critical 告警需要 [即时通知/电话升级]
- Warning 告警发送到 [通道名称]
- 告警分组策略：按 [alertname/component/severity] 分组
- 重复间隔：Critical [1小时]，Warning [4小时]
- 需要抑制规则：Critical 触发时抑制同组件的 Warning
- 工作时间：[时区]，非工作时间仅发送 Critical

请生成完整的 alertmanager.yml 配置文件，包含注释说明。

{
  "dashboard": {
    "title": "OpenClaw Agent 监控总览",
    "tags": ["openclaw", "agent", "monitoring"],
    "timezone": "browser",
    "refresh": "30s",
    "panels": [
      {
        "title": "Gateway 状态",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
        "targets": [{
          "expr": "openclaw_health_status",
          "legendFormat": "Health"
        }],
        "fieldConfig": {
          "defaults": {
            "mappings": [
              { "type": "value", "options": { "1": { "text": "✅ 健康", "color": "green" } } },
              { "type": "value", "options": { "0": { "text": "🔴 异常", "color": "red" } } }
            ]
          }
        }
      },
      {
        "title": "Gateway 运行时间",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
        "targets": [{
          "expr": "openclaw_gateway_uptime_seconds / 3600",
          "legendFormat": "Uptime"
        }],
        "fieldConfig": {
          "defaults": { "unit": "h", "decimals": 1 }
        }
      },
      {
        "title": "活跃会话数",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 },
        "targets": [{
          "expr": "openclaw_active_sessions",
          "legendFormat": "Sessions"
        }]
      },
      {
        "title": "今日 Token 消耗",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 },
        "targets": [{
          "expr": "increase(openclaw_llm_tokens_total[24h])",
          "legendFormat": "Tokens"
        }],
        "fieldConfig": {
          "defaults": { "unit": "short", "decimals": 0 }
        }
      },
      {
        "title": "今日估算成本",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 },
        "targets": [{
          "expr": "increase(openclaw_llm_cost_dollars[24h])",
          "legendFormat": "Cost"
        }],
        "fieldConfig": {
          "defaults": { "unit": "currencyUSD", "decimals": 2 }
        }
      },
      {
        "title": "当前告警数",
        "type": "stat",
        "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 },
        "targets": [{
          "expr": "count(ALERTS{alertstate=\"firing\"})",
          "legendFormat": "Firing"
        }],
        "fieldConfig": {
          "defaults": {
            "thresholds": {
              "steps": [
                { "value": 0, "color": "green" },
                { "value": 1, "color": "yellow" },
                { "value": 3, "color": "red" }
              ]
            }
          }
        }
      },
      {
        "title": "LLM 请求延迟 (P50 / P95 / P99)",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
        "targets": [
          {
            "expr": "histogram_quantile(0.50, rate(openclaw_llm_request_duration_seconds_bucket[5m]))",
            "legendFormat": "P50"
          },
          {
            "expr": "histogram_quantile(0.95, rate(openclaw_llm_request_duration_seconds_bucket[5m]))",
            "legendFormat": "P95"
          },
          {
            "expr": "histogram_quantile(0.99, rate(openclaw_llm_request_duration_seconds_bucket[5m]))",
            "legendFormat": "P99"
          }
        ],
        "fieldConfig": {
          "defaults": { "unit": "s" }
        }
      },
      {
        "title": "Token 消耗趋势（按模型）",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
        "targets": [{
          "expr": "rate(openclaw_llm_tokens_total[1h]) * 3600",
          "legendFormat": "{{ provider }}/{{ model }} ({{ type }})"
        }],
        "fieldConfig": {
          "defaults": { "unit": "short" }
        }
      },
      {
        "title": "任务成功率",
        "type": "gauge",
        "gridPos": { "h": 6, "w": 6, "x": 0, "y": 12 },
        "targets": [{
          "expr": "rate(openclaw_tasks_total{status=\"success\"}[1h]) / rate(openclaw_tasks_total[1h]) * 100",
          "legendFormat": "Success Rate"
        }],
        "fieldConfig": {
          "defaults": {
            "unit": "percent",
            "min": 0, "max": 100,
            "thresholds": {
              "steps": [
                { "value": 0, "color": "red" },
                { "value": 80, "color": "yellow" },
                { "value": 95, "color": "green" }
              ]
            }
          }
        }
      },
      {
        "title": "CPU 使用率",
        "type": "timeseries",
        "gridPos": { "h": 6, "w": 9, "x": 6, "y": 12 },
        "targets": [{
          "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
          "legendFormat": "CPU %"
        }],
        "fieldConfig": {
          "defaults": { "unit": "percent", "max": 100 }
        }
      },
      {
        "title": "内存使用率",
        "type": "timeseries",
        "gridPos": { "h": 6, "w": 9, "x": 15, "y": 12 },
        "targets": [{
          "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
          "legendFormat": "Memory %"
        }],
        "fieldConfig": {
          "defaults": { "unit": "percent", "max": 100 }
        }
      },
      {
        "title": "工具调用统计（Top 10）",
        "type": "barchart",
        "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
        "targets": [{
          "expr": "topk(10, increase(openclaw_tool_calls_total[24h]))",
          "legendFormat": "{{ tool_name }}"
        }]
      },
      {
        "title": "累计 API 成本趋势",
        "type": "timeseries",
        "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
        "targets": [{
          "expr": "increase(openclaw_llm_cost_dollars[24h])",
          "legendFormat": "{{ provider }}/{{ model }}"
        }],
        "fieldConfig": {
          "defaults": { "unit": "currencyUSD" }
        }
      }
    ]
  }
}

# 在 docker-compose.monitoring.yml 中添加
  uptime-kuma:
    image: louislam/uptime-kuma:1
    container_name: openclaw-uptime-kuma
    restart: unless-stopped
    ports:
      - "127.0.0.1:3001:3001"
    volumes:
      - uptime_kuma_data:/app/data
    networks:
      - openclaw-net

# 月度可用性计算
SLA = (1 - 总停机分钟数 / 总分钟数) × 100%

# 目标 SLA 对照表
99.0% = 每月最多 7.3 小时停机
99.5% = 每月最多 3.65 小时停机
99.9% = 每月最多 43.8 分钟停机

# PromQL：过去 30 天可用性百分比
(1 - (
  count_over_time((openclaw_health_status == 0)[30d:1m])
  / count_over_time(openclaw_health_status[30d:1m])
)) * 100

# 添加 Helm 仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
 
# 安装 kube-prometheus-stack
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi

# servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: openclaw-monitor
  namespace: openclaw
  labels:
    release: monitoring  # 匹配 kube-prometheus-stack 的 label selector
spec:
  selector:
    matchLabels:
      app: openclaw
  endpoints:
    - port: http
      path: /metrics
      interval: 30s
      scrapeTimeout: 10s
  namespaceSelector:
    matchNames:
      - openclaw

# prometheusrule.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: openclaw-alerts
  namespace: openclaw
  labels:
    release: monitoring
spec:
  groups:
    - name: openclaw-agent
      rules:
        - alert: OpenClawGatewayDown
          expr: openclaw_health_status == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "OpenClaw Gateway 不可用"
        # ... 其他规则同上

# 在 openclaw-deploy 目录下
mkdir -p monitoring/{prometheus/rules,alertmanager,grafana/provisioning/{datasources,dashboards},grafana/dashboards}
 
# 创建 Prometheus 配置（使用上方模板）
vim monitoring/prometheus/prometheus.yml
 
# 创建告警规则
vim monitoring/prometheus/rules/infrastructure.yml
vim monitoring/prometheus/rules/openclaw-agent.yml
 
# 创建 Alertmanager 配置
vim monitoring/alertmanager/alertmanager.yml
 
# 创建 Grafana 数据源配置
vim monitoring/grafana/provisioning/datasources/prometheus.yml
 
# 创建仪表板配置
vim monitoring/grafana/provisioning/dashboards/dashboards.yml
 
# 复制仪表板 JSON
vim monitoring/grafana/dashboards/openclaw-overview.json

# 1. 在 Telegram 中找到 @BotFather，创建新 Bot
# 2. 获取 Bot Token
# 3. 创建一个群组，将 Bot 加入
# 4. 获取 Chat ID：
curl -s "https://api.telegram.org/bot<YOUR_BOT_TOKEN>/getUpdates" | jq '.result[0].message.chat.id'
 
# 5. 将 Token 和 Chat ID 填入 alertmanager.yml

# 在 .env 中添加 Grafana 密码
echo "GRAFANA_ADMIN_PASSWORD=$(openssl rand -base64 16)" >> .env
 
# 启动所有服务
docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d
 
# 检查所有容器状态
docker compose -f docker-compose.yml -f docker-compose.monitoring.yml ps

grafana.yourdomain.com {
    reverse_proxy localhost:3000
    header {
        X-Content-Type-Options nosniff
        X-Frame-Options DENY
        -Server
    }
}

# 重载 Caddy 配置
docker compose restart caddy

# 检查 Prometheus 目标
curl -s http://127.0.0.1:9090/api/v1/targets | \
  jq '.data.activeTargets[] | {job: .labels.job, health: .health}'
 
# 预期输出：
# {"job": "prometheus", "health": "up"}
# {"job": "node-exporter", "health": "up"}
# {"job": "cadvisor", "health": "up"}
# {"job": "openclaw-gateway", "health": "up"}
 
# 检查告警规则
curl -s http://127.0.0.1:9090/api/v1/rules | jq '.data.groups[].rules[] | {name: .name, state: .state}'
 
# 测试 Alertmanager 通知
# 手动触发测试告警
curl -X POST http://127.0.0.1:9093/api/v2/alerts \
  -H "Content-Type: application/json" \
  -d '[{
    "labels": {
      "alertname": "TestAlert",
      "severity": "warning",
      "component": "test"
    },
    "annotations": {
      "summary": "这是一条测试告警",
      "description": "验证 Alertmanager 通知通道是否正常工作。"
    }
  }]'