347 lines
11 KiB
Go
347 lines
11 KiB
Go
package report
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"html/template"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/go-echarts/go-echarts/v2/charts"
|
|
"github.com/go-echarts/go-echarts/v2/opts"
|
|
"github.com/go-echarts/go-echarts/v2/render"
|
|
"llm-api-benchmark-tool/pkg/stats"
|
|
)
|
|
|
|
// ReportData holds the data passed to the HTML template.
|
|
type ReportData struct {
|
|
Stats stats.FinalStats
|
|
LatencyChart template.HTML // Already safe HTML
|
|
TTFTChart template.HTML // Already safe HTML
|
|
// Add other charts here (QPSChart, etc.)
|
|
}
|
|
|
|
// Basic HTML template string - No |safeHTML filter needed
|
|
const htmlTemplate = `<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>LLM API Benchmark Report</title>
|
|
<!-- Include ECharts JS directly for simplicity in this example -->
|
|
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/echarts@5.4.1/dist/echarts.min.js"></script>
|
|
<style> body { font-family: sans-serif; margin: 20px; } table { border-collapse: collapse; margin-bottom: 20px; } th, td { border: 1px solid #ddd; padding: 8px; text-align: left; } th { background-color: #f2f2f2; } .chart-container { width: 80%; min-height: 400px; margin: auto; margin-bottom: 30px; border: 1px solid #eee; padding: 15px; } </style>
|
|
</head>
|
|
<body>
|
|
<h1>LLM API Benchmark Report</h1>
|
|
|
|
<h2>Summary</h2>
|
|
<table>
|
|
<tr><th>Metric</th><th>Value</th></tr>
|
|
<tr><td>Total Duration:</td><td>{{ .Stats.TotalDuration }}</td></tr>
|
|
<tr><td>Total Requests:</td><td>{{ .Stats.TotalRequests }}</td></tr>
|
|
<tr><td>Successful Requests:</td><td>{{ .Stats.SuccessfulRequests }}</td></tr>
|
|
<tr><td>Failed Requests:</td><td>{{ .Stats.FailedRequests }}</td></tr>
|
|
<tr><td>Avg QPS:</td><td>{{ printf "%.2f" .Stats.AvgQPS }}</td></tr>
|
|
<tr><td>Avg Latency:</td><td>{{ .Stats.AvgLatency }}</td></tr>
|
|
<tr><td>Min Latency:</td><td>{{ .Stats.MinLatency }}</td></tr>
|
|
<tr><td>Max Latency:</td><td>{{ .Stats.MaxLatency }}</td></tr>
|
|
<tr><td>P90 Latency:</td><td>{{ .Stats.P90Latency }}</td></tr>
|
|
<tr><td>P95 Latency:</td><td>{{ .Stats.P95Latency }}</td></tr>
|
|
<tr><td>P99 Latency:</td><td>{{ .Stats.P99Latency }}</td></tr>
|
|
<tr><td>Avg TTFT:</td><td>{{ .Stats.AvgTimeToFirstToken }}</td></tr>
|
|
<tr><td>Min TTFT:</td><td>{{ .Stats.MinTimeToFirstToken }}</td></tr>
|
|
<tr><td>Max TTFT:</td><td>{{ .Stats.MaxTimeToFirstToken }}</td></tr>
|
|
<tr><td>P90 TTFT:</td><td>{{ .Stats.P90TimeToFirstToken }}</td></tr>
|
|
<tr><td>P95 TTFT:</td><td>{{ .Stats.P95TimeToFirstToken }}</td></tr>
|
|
<tr><td>P99 TTFT:</td><td>{{ .Stats.P99TimeToFirstToken }}</td></tr>
|
|
<tr><td>Avg Tokens/Second:</td><td>{{ printf "%.2f" .Stats.AvgTokensPerSecond }}</td></tr>
|
|
<!-- Add more stats as needed -->
|
|
</table>
|
|
|
|
<h2>Charts</h2>
|
|
<!-- Latency Histogram Chart -->
|
|
<div class="chart-container">
|
|
{{ .LatencyChart }}
|
|
</div>
|
|
|
|
<!-- TTFT Histogram Chart -->
|
|
<div class="chart-container">
|
|
{{ .TTFTChart }}
|
|
</div>
|
|
|
|
<!-- Add more chart containers here -->
|
|
|
|
</body>
|
|
</html>`
|
|
|
|
// GenerateHTMLReport generates an HTML report from the benchmark statistics.
|
|
func GenerateHTMLReport(finalStats stats.FinalStats, outputFilePath string) error {
|
|
// --- Create Charts ---
|
|
latencyChart := createLatencyHistogram(finalStats)
|
|
ttftChart := createTTFTHistogram(finalStats)
|
|
|
|
// --- Prepare Template Data ---
|
|
// Render charts to temporary buffers to get HTML content
|
|
latencyChartHTML, err := renderChartToHTMLFragment(latencyChart)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to render latency chart: %w", err)
|
|
}
|
|
ttftChartHTML, err := renderChartToHTMLFragment(ttftChart)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to render TTFT chart: %w", err)
|
|
}
|
|
|
|
summaryStr := fmt.Sprintf(`
|
|
Total Requests: %d
|
|
Successful Requests: %d
|
|
Failed Requests: %d
|
|
Total Duration: %s
|
|
Average Latency: %s
|
|
Min Latency: %s
|
|
Max Latency: %s
|
|
P90 Latency: %s
|
|
P95 Latency: %s
|
|
P99 Latency: %s
|
|
Average TTFT: %s
|
|
Min TTFT: %s
|
|
Max TTFT: %s
|
|
P90 TTFT: %s
|
|
P95 TTFT: %s
|
|
P99 TTFT: %s
|
|
Average QPS: %.2f
|
|
Average Tokens/Second: %.2f
|
|
`,
|
|
finalStats.TotalRequests,
|
|
finalStats.SuccessfulRequests,
|
|
finalStats.FailedRequests,
|
|
finalStats.TotalDuration,
|
|
finalStats.AvgLatency,
|
|
finalStats.MinLatency,
|
|
finalStats.MaxLatency,
|
|
finalStats.P90Latency,
|
|
finalStats.P95Latency,
|
|
finalStats.P99Latency,
|
|
finalStats.AvgTimeToFirstToken,
|
|
finalStats.MinTimeToFirstToken,
|
|
finalStats.MaxTimeToFirstToken,
|
|
finalStats.P90TimeToFirstToken,
|
|
finalStats.P95TimeToFirstToken,
|
|
finalStats.P99TimeToFirstToken,
|
|
finalStats.AvgQPS,
|
|
finalStats.AvgTokensPerSecond,
|
|
)
|
|
|
|
templateData := map[string]interface{}{
|
|
"Stats": finalStats,
|
|
"Summary": summaryStr,
|
|
"LatencyChart": latencyChartHTML,
|
|
"TTFTChart": ttftChartHTML,
|
|
}
|
|
|
|
// --- Parse and Execute Template ---
|
|
// No Funcs map needed as renderChartToHTMLFragment returns template.HTML
|
|
tmpl, err := template.New("report").Parse(htmlTemplate)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse HTML template: %w", err)
|
|
}
|
|
|
|
f, err := os.Create(outputFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create report file '%s': %w", outputFilePath, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
// log.Printf("DEBUG: Template Data Before Execution: %+v", templateData)
|
|
|
|
// Execute the template
|
|
err = tmpl.Execute(f, templateData)
|
|
if err != nil {
|
|
// Include the underlying template execution error for better debugging
|
|
return fmt.Errorf("failed to execute HTML template: %w", err)
|
|
}
|
|
|
|
fmt.Printf("HTML report generated successfully: %s\n", outputFilePath)
|
|
return nil
|
|
}
|
|
|
|
// getChartBaseConfig safely extracts the BaseConfiguration from a render.Renderer.
|
|
func getChartBaseConfig(chart render.Renderer) (*charts.BaseConfiguration, string, error) {
|
|
var baseConfig *charts.BaseConfiguration
|
|
var chartID string
|
|
|
|
switch c := chart.(type) {
|
|
case *charts.Bar:
|
|
baseConfig = &c.RectChart.RectConfiguration.BaseConfiguration
|
|
chartID = c.Initialization.ChartID // Access ChartID from embedded Initialization
|
|
case *charts.Line: // Add cases for other chart types if needed
|
|
baseConfig = &c.RectChart.RectConfiguration.BaseConfiguration
|
|
chartID = c.Initialization.ChartID
|
|
// Add other chart types here (e.g., Pie, Scatter, etc.)
|
|
default:
|
|
return nil, "", fmt.Errorf("unsupported chart type: %T", chart)
|
|
}
|
|
|
|
if chartID == "" {
|
|
// Generate a fallback ID if not set in Initialization
|
|
chartID = fmt.Sprintf("chart%d", time.Now().UnixNano())
|
|
log.Printf("Warning: ChartID not set for %T, using generated ID: %s", chart, chartID)
|
|
}
|
|
|
|
return baseConfig, chartID, nil
|
|
}
|
|
|
|
// renderChartToHTMLFragment renders a chart configuration to an HTML fragment.
|
|
func renderChartToHTMLFragment(chart render.Renderer) (template.HTML, error) {
|
|
baseConfig, chartID, err := getChartBaseConfig(chart)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// Serialize the BaseConfiguration which holds the options.
|
|
optJSONBytes, err := json.Marshal(baseConfig)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to marshal chart options for %s (%T): %w", chartID, chart, err)
|
|
}
|
|
optJSON := string(optJSONBytes)
|
|
|
|
// Construct the HTML fragment (div and script)
|
|
script := fmt.Sprintf(`
|
|
<div id="%s" style="width:100%%; height:400px;"></div>
|
|
<script type="text/javascript">
|
|
"use strict";
|
|
// Check if element exists before initializing
|
|
if (document.getElementById('%s')) {
|
|
let chart_%s = echarts.init(document.getElementById('%s'), null, {renderer: '%s'});
|
|
let option_%s = %s;
|
|
chart_%s.setOption(option_%s);
|
|
} else {
|
|
console.error("Chart element not found: %s");
|
|
}
|
|
</script>`,
|
|
chartID, // div id
|
|
chartID, // Check element id
|
|
chartID, chartID, // script variable and element id
|
|
baseConfig.Initialization.Renderer, // Use renderer from initialization options
|
|
chartID, optJSON, // options variable and json content
|
|
chartID, chartID, // setOption call
|
|
chartID, // Error log element id
|
|
)
|
|
|
|
return template.HTML(script), nil
|
|
}
|
|
|
|
// calculateHistogram calculates histogram data (categories and counts) from a slice of durations.
|
|
func calculateHistogram(data []time.Duration, numBins int) ([]string, []opts.BarData) {
|
|
if len(data) == 0 || numBins <= 0 {
|
|
return []string{"No Data"}, []opts.BarData{{Value: 0}}
|
|
}
|
|
|
|
// Find min and max values
|
|
minVal, maxVal := data[0], data[0]
|
|
for _, d := range data {
|
|
if d < minVal {
|
|
minVal = d
|
|
}
|
|
if d > maxVal {
|
|
maxVal = d
|
|
}
|
|
}
|
|
|
|
// Handle case where all data points are the same
|
|
if minVal == maxVal {
|
|
return []string{fmt.Sprintf("%.2f ms", float64(minVal)/float64(time.Millisecond))},
|
|
[]opts.BarData{{Value: len(data)}}
|
|
}
|
|
|
|
binWidth := (maxVal - minVal) / time.Duration(numBins)
|
|
if binWidth == 0 {
|
|
binWidth = 1 // Avoid division by zero if range is very small relative to numBins
|
|
}
|
|
|
|
bins := make([]int, numBins) // Stores counts for each bin
|
|
categories := make([]string, numBins)
|
|
|
|
for i := 0; i < numBins; i++ {
|
|
binStart := minVal + time.Duration(i)*binWidth
|
|
binEnd := binStart + binWidth
|
|
// Ensure the last bin's end includes the max value exactly
|
|
if i == numBins-1 {
|
|
binEnd = maxVal
|
|
}
|
|
|
|
// Format category label (e.g., "50.00-100.00 ms")
|
|
// Use slightly different formatting for clarity: [start, end) or [start, end] for the last bin
|
|
labelFormat := "[%.2f, %.2f) ms"
|
|
if i == numBins-1 {
|
|
labelFormat = "[%.2f, %.2f] ms"
|
|
}
|
|
categories[i] = fmt.Sprintf(labelFormat,
|
|
float64(binStart)/float64(time.Millisecond),
|
|
float64(binEnd)/float64(time.Millisecond),
|
|
)
|
|
}
|
|
|
|
for _, d := range data {
|
|
// Determine the correct bin index
|
|
var binIndex int
|
|
if d == maxVal {
|
|
// Special case: max value goes into the last bin
|
|
binIndex = numBins - 1
|
|
} else {
|
|
// Calculate bin index based on width
|
|
binIndex = int((d - minVal) / binWidth)
|
|
// Clamp index to valid range (should not happen with maxVal check, but defensive)
|
|
if binIndex < 0 {
|
|
binIndex = 0
|
|
} else if binIndex >= numBins {
|
|
binIndex = numBins - 1
|
|
}
|
|
}
|
|
bins[binIndex]++
|
|
}
|
|
|
|
barData := make([]opts.BarData, numBins)
|
|
for i := 0; i < numBins; i++ {
|
|
barData[i] = opts.BarData{Value: bins[i]}
|
|
}
|
|
|
|
return categories, barData
|
|
}
|
|
|
|
// createLatencyHistogram creates a Bar chart for latency distribution.
|
|
func createLatencyHistogram(finalStats stats.FinalStats) *charts.Bar {
|
|
bar := charts.NewBar()
|
|
bar.SetGlobalOptions(
|
|
charts.WithTitleOpts(opts.Title{Title: "Latency Distribution"}),
|
|
// Ensure ID is set correctly via InitializationOpts
|
|
charts.WithInitializationOpts(opts.Initialization{ChartID: "latencyHistogram"}),
|
|
// Add other global options if needed, e.g., Toolbox
|
|
charts.WithToolboxOpts(opts.Toolbox{Show: opts.Bool(true)}),
|
|
)
|
|
|
|
// Calculate and set histogram data
|
|
categories, data := calculateHistogram(finalStats.LatencyData, 10) // Use 10 bins for example
|
|
bar.SetXAxis(categories).AddSeries("Latency", data)
|
|
|
|
return bar
|
|
}
|
|
|
|
// createTTFTHistogram creates a Bar chart for Time To First Token distribution.
|
|
func createTTFTHistogram(finalStats stats.FinalStats) *charts.Bar {
|
|
bar := charts.NewBar()
|
|
bar.SetGlobalOptions(
|
|
charts.WithTitleOpts(opts.Title{Title: "TTFT Distribution"}),
|
|
// Ensure ID is set correctly via InitializationOpts
|
|
charts.WithInitializationOpts(opts.Initialization{ChartID: "ttftHistogram"}),
|
|
// Add other global options if needed
|
|
charts.WithToolboxOpts(opts.Toolbox{Show: opts.Bool(true)}),
|
|
)
|
|
|
|
// Calculate and set histogram data (assuming TTFTData exists in FinalStats)
|
|
categories, data := calculateHistogram(finalStats.TTFTData, 10) // Use actual TTFT data
|
|
bar.SetXAxis(categories).AddSeries("TTFT", data)
|
|
|
|
return bar
|
|
}
|