Latency Budgets

import os
import statistics
import time

from promptguard import GuardClient

client = GuardClient(api_key=os.environ["PROMPTGUARD_API_KEY"])
samples = []

for i in range(200):
    # Vary the input so you measure detectors, not the policy cache.
    messages = [{"role": "user", "content": f"Summarize support ticket {i}"}]
    start = time.perf_counter()
    client.scan(messages, direction="input")
    samples.append((time.perf_counter() - start) * 1000)

samples.sort()
p = lambda q: samples[int(len(samples) * q) - 1]
print(f"n={len(samples)}")
print(f"p50 {p(0.50):6.1f} ms")
print(f"p95 {p(0.95):6.1f} ms")
print(f"p99 {p(0.99):6.1f} ms")
print(f"max {samples[-1]:6.1f} ms   mean {statistics.mean(samples):.1f} ms")

import { GuardClient } from 'promptguard-sdk/guard'

const guard = new GuardClient({ apiKey: process.env.PROMPTGUARD_API_KEY! })
const samples: number[] = []

for (let i = 0; i < 200; i++) {
  // Vary the input so you measure detectors, not the policy cache.
  const messages = [{ role: 'user' as const, content: `Summarize support ticket ${i}` }]
  const start = performance.now()
  await guard.scan(messages, 'input')
  samples.push(performance.now() - start)
}

samples.sort((a, b) => a - b)
const p = (q: number) => samples[Math.ceil(samples.length * q) - 1]
console.log(`n=${samples.length}`)
console.log(`p50 ${p(0.5).toFixed(1)} ms`)
console.log(`p95 ${p(0.95).toFixed(1)} ms`)
console.log(`p99 ${p(0.99).toFixed(1)} ms`)