openai_ws_forwarder_benchmark_test.go 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. package service
  2. import (
  3. "fmt"
  4. "testing"
  5. "github.com/Wei-Shaw/sub2api/internal/config"
  6. )
  7. var (
  8. benchmarkOpenAIWSPayloadJSONSink string
  9. benchmarkOpenAIWSStringSink string
  10. benchmarkOpenAIWSBoolSink bool
  11. benchmarkOpenAIWSBytesSink []byte
  12. )
  13. func BenchmarkOpenAIWSForwarderHotPath(b *testing.B) {
  14. cfg := &config.Config{}
  15. svc := &OpenAIGatewayService{cfg: cfg}
  16. account := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
  17. reqBody := benchmarkOpenAIWSHotPathRequest()
  18. b.ReportAllocs()
  19. b.ResetTimer()
  20. for i := 0; i < b.N; i++ {
  21. payload := svc.buildOpenAIWSCreatePayload(reqBody, account)
  22. _, _ = applyOpenAIWSRetryPayloadStrategy(payload, 2)
  23. setOpenAIWSTurnMetadata(payload, `{"trace":"bench","turn":"1"}`)
  24. benchmarkOpenAIWSStringSink = openAIWSPayloadString(payload, "previous_response_id")
  25. benchmarkOpenAIWSBoolSink = payload["tools"] != nil
  26. benchmarkOpenAIWSStringSink = summarizeOpenAIWSPayloadKeySizes(payload, openAIWSPayloadKeySizeTopN)
  27. benchmarkOpenAIWSStringSink = summarizeOpenAIWSInput(payload["input"])
  28. benchmarkOpenAIWSPayloadJSONSink = payloadAsJSON(payload)
  29. }
  30. }
  31. func benchmarkOpenAIWSHotPathRequest() map[string]any {
  32. tools := make([]map[string]any, 0, 24)
  33. for i := 0; i < 24; i++ {
  34. tools = append(tools, map[string]any{
  35. "type": "function",
  36. "name": fmt.Sprintf("tool_%02d", i),
  37. "description": "benchmark tool schema",
  38. "parameters": map[string]any{
  39. "type": "object",
  40. "properties": map[string]any{
  41. "query": map[string]any{"type": "string"},
  42. "limit": map[string]any{"type": "number"},
  43. },
  44. "required": []string{"query"},
  45. },
  46. })
  47. }
  48. input := make([]map[string]any, 0, 16)
  49. for i := 0; i < 16; i++ {
  50. input = append(input, map[string]any{
  51. "role": "user",
  52. "type": "message",
  53. "content": fmt.Sprintf("benchmark message %d", i),
  54. })
  55. }
  56. return map[string]any{
  57. "type": "response.create",
  58. "model": "gpt-5.3-codex",
  59. "input": input,
  60. "tools": tools,
  61. "parallel_tool_calls": true,
  62. "previous_response_id": "resp_benchmark_prev",
  63. "prompt_cache_key": "bench-cache-key",
  64. "reasoning": map[string]any{"effort": "medium"},
  65. "instructions": "benchmark instructions",
  66. "store": false,
  67. }
  68. }
  69. func BenchmarkOpenAIWSEventEnvelopeParse(b *testing.B) {
  70. event := []byte(`{"type":"response.completed","response":{"id":"resp_bench_1","model":"gpt-5.1","usage":{"input_tokens":12,"output_tokens":8}}}`)
  71. b.ReportAllocs()
  72. b.ResetTimer()
  73. for i := 0; i < b.N; i++ {
  74. eventType, responseID, response := parseOpenAIWSEventEnvelope(event)
  75. benchmarkOpenAIWSStringSink = eventType
  76. benchmarkOpenAIWSStringSink = responseID
  77. benchmarkOpenAIWSBoolSink = response.Exists()
  78. }
  79. }
  80. func BenchmarkOpenAIWSErrorEventFieldReuse(b *testing.B) {
  81. event := []byte(`{"type":"error","error":{"type":"invalid_request_error","code":"invalid_request","message":"invalid input"}}`)
  82. b.ReportAllocs()
  83. b.ResetTimer()
  84. for i := 0; i < b.N; i++ {
  85. codeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(event)
  86. benchmarkOpenAIWSStringSink, benchmarkOpenAIWSBoolSink = classifyOpenAIWSErrorEventFromRaw(codeRaw, errTypeRaw, errMsgRaw)
  87. code, errType, errMsg := summarizeOpenAIWSErrorEventFieldsFromRaw(codeRaw, errTypeRaw, errMsgRaw)
  88. benchmarkOpenAIWSStringSink = code
  89. benchmarkOpenAIWSStringSink = errType
  90. benchmarkOpenAIWSStringSink = errMsg
  91. benchmarkOpenAIWSBoolSink = openAIWSErrorHTTPStatusFromRaw(codeRaw, errTypeRaw) > 0
  92. }
  93. }
  94. func BenchmarkReplaceOpenAIWSMessageModel_NoMatchFastPath(b *testing.B) {
  95. event := []byte(`{"type":"response.output_text.delta","delta":"hello world"}`)
  96. b.ReportAllocs()
  97. b.ResetTimer()
  98. for i := 0; i < b.N; i++ {
  99. benchmarkOpenAIWSBytesSink = replaceOpenAIWSMessageModel(event, "gpt-5.1", "custom-model")
  100. }
  101. }
  102. func BenchmarkReplaceOpenAIWSMessageModel_DualReplace(b *testing.B) {
  103. event := []byte(`{"type":"response.completed","model":"gpt-5.1","response":{"id":"resp_1","model":"gpt-5.1"}}`)
  104. b.ReportAllocs()
  105. b.ResetTimer()
  106. for i := 0; i < b.N; i++ {
  107. benchmarkOpenAIWSBytesSink = replaceOpenAIWSMessageModel(event, "gpt-5.1", "custom-model")
  108. }
  109. }