04d - Prompt 版本管理

prompts/
├── code-review/
│   ├── v1.0.0.md          # 版本化的 prompt 文件
│   ├── v1.1.0.md
│   ├── v2.0.0.md
│   ├── current.md          # 符号链接到当前版本
│   ├── test-cases.json     # 测试用例
│   └── CHANGELOG.md        # 变更日志
├── bug-analysis/
│   ├── v1.0.0.md
│   ├── current.md
│   └── test-cases.json
└── prompt-config.yaml       # 全局配置

prompts:
  code-review:
    current_version: "2.0.0"
    model: "claude-sonnet-4-20250514"
    max_tokens: 2048
    temperature: 0
    tags: ["development", "quality"]
    owner: "backend-team"
 
  bug-analysis:
    current_version: "1.0.0"
    model: "claude-sonnet-4-20250514"
    max_tokens: 1024
    temperature: 0
    tags: ["development", "debugging"]
    owner: "backend-team"

用户请求
    │
    ▼
┌──────────────┐
│  流量分配器   │  50% / 50%（或其他比例）
└──────┬───────┘
       │
  ┌────┴────┐
  ▼         ▼
Prompt A   Prompt B
(v1.2.0)   (v2.0.0-beta)
  │         │
  ▼         ▼
输出 A     输出 B
  │         │
  └────┬────┘
       ▼
┌──────────────┐
│  指标收集器   │  记录质量、延迟、成本、用户反馈
└──────────────┘

interface PromptVersion {
  id: string;
  version: string;
  content: string;
  model: string;
  weight: number; // 流量权重，0-100
}
 
interface ABTestResult {
  versionId: string;
  input: string;
  output: string;
  latencyMs: number;
  tokenCount: number;
  timestamp: Date;
}
 
class PromptABTester {
  private versions: PromptVersion[];
  private results: ABTestResult[] = [];
 
  constructor(versions: PromptVersion[]) {
    this.versions = versions;
  }
 
  selectVersion(): PromptVersion {
    const totalWeight = this.versions.reduce((sum, v) => sum + v.weight, 0);
    let random = Math.random() * totalWeight;
 
    for (const version of this.versions) {
      random -= version.weight;
      if (random <= 0) return version;
    }
    return this.versions[0];
  }
 
  async execute(input: string): Promise<ABTestResult> {
    const version = this.selectVersion();
    const start = Date.now();
 
    const response = await client.messages.create({
      model: version.model,
      max_tokens: 2048,
      messages: [{
        role: "user",
        content: version.content.replace("[INPUT]", input)
      }]
    });
 
    const result: ABTestResult = {
      versionId: version.id,
      input,
      output: response.content[0].type === "text"
        ? response.content[0].text : "",
      latencyMs: Date.now() - start,
      tokenCount: response.usage.input_tokens + response.usage.output_tokens,
      timestamp: new Date()
    };
 
    this.results.push(result);
    return result;
  }
 
  getStats() {
    const grouped = new Map<string, ABTestResult[]>();
    for (const r of this.results) {
      const list = grouped.get(r.versionId) || [];
      list.push(r);
      grouped.set(r.versionId, list);
    }
 
    return Array.from(grouped.entries()).map(([id, results]) => ({
      versionId: id,
      count: results.length,
      avgLatencyMs: results.reduce((s, r) => s + r.latencyMs, 0) / results.length,
      avgTokens: results.reduce((s, r) => s + r.tokenCount, 0) / results.length,
    }));
  }
}

发现问题
    │
    ▼
┌──────────────┐
│ 严重性评估    │
└──────┬───────┘
       │
  ┌────┼────┐
  ▼    ▼    ▼
 高    中    低
  │    │    │
  ▼    ▼    ▼
立即   计划   记录
回滚   回滚   观察

class PromptVersionManager {
  private currentVersion: string;
  private versionHistory: Map<string, string>; // version -> prompt content
  private rollbackStack: string[] = [];
 
  async deploy(version: string): Promise<void> {
    // 保存当前版本到回滚栈
    this.rollbackStack.push(this.currentVersion);
 
    // 部署新版本
    this.currentVersion = version;
 
    // 记录部署事件
    console.log(`Deployed prompt version ${version} at ${new Date().toISOString()}`);
  }
 
  async rollback(): Promise<string> {
    const previousVersion = this.rollbackStack.pop();
    if (!previousVersion) {
      throw new Error("No version to rollback to");
    }
 
    this.currentVersion = previousVersion;
    console.log(`Rolled back to version ${previousVersion}`);
    return previousVersion;
  }
 
  getCurrentPrompt(): string {
    return this.versionHistory.get(this.currentVersion) || "";
  }
}

┌─────────────────────────────────────────────┐
│              Prompt 性能仪表板               │
│                                             │
│  ┌──────────┐  ┌──────────┐  ┌──────────┐  │
│  │ 质量指标  │  │ 成本指标  │  │ 速度指标  │  │
│  │          │  │          │  │          │  │
│  │ 完成率    │  │ Token/次  │  │ P50 延迟  │  │
│  │ 准确率    │  │ 美元/次   │  │ P95 延迟  │  │
│  │ 满意度    │  │ 日成本    │  │ P99 延迟  │  │
│  └──────────┘  └──────────┘  └──────────┘  │
│                                             │
│  ┌──────────┐  ┌──────────┐                 │
│  │ 使用指标  │  │ 异常指标  │                 │
│  │          │  │          │                 │
│  │ 调用次数  │  │ 错误率    │                 │
│  │ 活跃用户  │  │ 超时率    │                 │
│  │ 峰值时段  │  │ 拒绝率    │                 │
│  └──────────┘  └──────────┘                 │
└─────────────────────────────────────────────┘

interface PromptMetrics {
  promptId: string;
  version: string;
  timestamp: Date;
  latencyMs: number;
  inputTokens: number;
  outputTokens: number;
  costUsd: number;
  success: boolean;
  userRating?: "positive" | "negative";
  errorType?: string;
}
 
class PromptTracker {
  private metrics: PromptMetrics[] = [];
 
  record(metric: PromptMetrics): void {
    this.metrics.push(metric);
  }
 
  getVersionStats(promptId: string, version: string) {
    const versionMetrics = this.metrics.filter(
      m => m.promptId === promptId && m.version === version
    );
 
    const total = versionMetrics.length;
    if (total === 0) return null;
 
    const successful = versionMetrics.filter(m => m.success);
    const rated = versionMetrics.filter(m => m.userRating);
    const positive = rated.filter(m => m.userRating === "positive");
 
    return {
      totalCalls: total,
      successRate: successful.length / total,
      avgLatencyMs: versionMetrics.reduce((s, m) => s + m.latencyMs, 0) / total,
      avgCostUsd: versionMetrics.reduce((s, m) => s + m.costUsd, 0) / total,
      satisfactionRate: rated.length > 0 ? positive.length / rated.length : null,
      p95LatencyMs: percentile(versionMetrics.map(m => m.latencyMs), 95),
    };
  }
}
 
function percentile(values: number[], p: number): number {
  const sorted = [...values].sort((a, b) => a - b);
  const index = Math.ceil((p / 100) * sorted.length) - 1;
  return sorted[index];
}

┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
│  开发     │    │  测试     │    │  预发布   │    │  生产     │
│          │    │          │    │          │    │          │
│ 编辑      │ →  │ 自动评估  │ →  │ A/B 测试  │ →  │ 全量部署  │
│ prompt    │    │ promptfoo │    │ 10% 流量  │    │ 100%     │
│          │    │          │    │          │    │          │
│ 本地测试  │    │ 回归测试  │    │ 指标监控  │    │ 持续监控  │
└──────────┘    └──────────┘    └──────────┘    └──────────┘

name: Prompt CI/CD
 
on:
  push:
    paths:
      - 'prompts/**'
 
jobs:
  test-prompts:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
 
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
 
      - name: Install promptfoo
        run: npm install -g promptfoo
 
      - name: Run prompt evaluation
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          promptfoo eval \
            --config prompts/promptfoo-config.yaml \
            --output results.json
 
      - name: Check quality gate
        run: |
          node scripts/check-quality-gate.js results.json
 
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: prompt-eval-results
          path: results.json
 
  deploy-prompts:
    needs: test-prompts
    if: github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
 
      - name: Deploy prompts
        run: |
          node scripts/deploy-prompts.js

# prompts/promptfoo-config.yaml
description: "Code Review Prompt Evaluation"
 
prompts:
  - file://code-review/v1.2.0.md
  - file://code-review/v2.0.0.md
 
providers:
  - id: anthropic:messages:claude-sonnet-4-20250514
    config:
      max_tokens: 2048
      temperature: 0
 
tests:
  - vars:
      code: "def login(user, pwd): return db.query(f'SELECT * FROM users WHERE name={user}')"
      language: "Python"
    assert:
      - type: contains
        value: "SQL injection"
      - type: llm-rubric
        value: "The response identifies the SQL injection vulnerability and provides a parameterized query fix"
 
  - vars:
      code: "const data = JSON.parse(req.body); eval(data.expression);"
      language: "JavaScript"
    assert:
      - type: contains
        value: "eval"
      - type: llm-rubric
        value: "The response identifies the eval() security risk and suggests a safe alternative"
 
  - vars:
      code: "func handler(w http.ResponseWriter, r *http.Request) { w.Write([]byte(r.URL.Query().Get('name'))) }"
      language: "Go"
    assert:
      - type: contains
        value: "XSS"
      - type: llm-rubric
        value: "The response identifies the XSS vulnerability and suggests proper output encoding"
 
# 质量门：所有测试通过率 >= 90%
defaultTest:
  options:
    threshold: 0.9

// scripts/check-quality-gate.js
const fs = require("fs");
 
const results = JSON.parse(fs.readFileSync(process.argv[2], "utf-8"));
 
const totalTests = results.results.length;
const passedTests = results.results.filter(r => r.success).length;
const passRate = passedTests / totalTests;
 
console.log(`Pass rate: ${(passRate * 100).toFixed(1)}% (${passedTests}/${totalTests})`);
 
if (passRate < 0.9) {
  console.error("Quality gate FAILED: pass rate below 90%");
  process.exit(1);
}
 
console.log("Quality gate PASSED");

1. 创建分支 → 2. 修改 prompt → 3. 本地测试 → 4. 提交 PR
                                                    │
                                                    ▼
8. 合并部署 ← 7. 审批通过 ← 6. 团队审查 ← 5. CI 自动测试

## Prompt 变更
 
**Prompt**: [prompt 名称]
**版本**: [旧版本] → [新版本]
**变更类型**: [Major / Minor / Patch]
 
## 变更原因
 
[为什么要改这个 prompt]
 
## 变更内容
 
[具体改了什么]
 
## 测试结果
 
- 测试用例数：[N]
- 通过率：[X%]
- 与旧版本对比：[质量提升/持平/下降]
 
## 回滚计划
 
[如果出问题，如何回滚]