mirror of
https://github.com/githubhjs/CLIProxyAPIPlus.git
synced 2026-07-13 18:15:24 +00:00
feat: add Kiro OAuth web, rate limiter, metrics, fingerprint, background refresh and model converter
This commit is contained in:
781456868@qq.com
@@ -0,0 +1,97 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type UTF8StreamParser struct {
|
||||
buffer []byte
|
||||
}
|
||||
|
||||
func NewUTF8StreamParser() *UTF8StreamParser {
|
||||
return &UTF8StreamParser{
|
||||
buffer: make([]byte, 0, 64),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) Write(data []byte) {
|
||||
p.buffer = append(p.buffer, data...)
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) Read() (string, bool) {
|
||||
if len(p.buffer) == 0 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
validLen := p.findValidUTF8End(p.buffer)
|
||||
if validLen == 0 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
result := string(p.buffer[:validLen])
|
||||
p.buffer = p.buffer[validLen:]
|
||||
|
||||
return result, true
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) Flush() string {
|
||||
if len(p.buffer) == 0 {
|
||||
return ""
|
||||
}
|
||||
result := string(p.buffer)
|
||||
p.buffer = p.buffer[:0]
|
||||
return result
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) Reset() {
|
||||
p.buffer = p.buffer[:0]
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) findValidUTF8End(data []byte) int {
|
||||
if len(data) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
end := len(data)
|
||||
for i := 1; i <= 3 && i <= len(data); i++ {
|
||||
b := data[len(data)-i]
|
||||
if b&0x80 == 0 {
|
||||
break
|
||||
}
|
||||
if b&0xC0 == 0xC0 {
|
||||
size := p.utf8CharSize(b)
|
||||
available := i
|
||||
if size > available {
|
||||
end = len(data) - i
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if end > 0 && !utf8.Valid(data[:end]) {
|
||||
for i := end - 1; i >= 0; i-- {
|
||||
if utf8.Valid(data[:i+1]) {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
return end
|
||||
}
|
||||
|
||||
func (p *UTF8StreamParser) utf8CharSize(b byte) int {
|
||||
if b&0x80 == 0 {
|
||||
return 1
|
||||
}
|
||||
if b&0xE0 == 0xC0 {
|
||||
return 2
|
||||
}
|
||||
if b&0xF0 == 0xE0 {
|
||||
return 3
|
||||
}
|
||||
if b&0xF8 == 0xF0 {
|
||||
return 4
|
||||
}
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestNewUTF8StreamParser(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
if p == nil {
|
||||
t.Fatal("expected non-nil UTF8StreamParser")
|
||||
}
|
||||
if p.buffer == nil {
|
||||
t.Error("expected non-nil buffer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrite(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("hello"))
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true")
|
||||
}
|
||||
if result != "hello" {
|
||||
t.Errorf("expected 'hello', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrite_MultipleWrites(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("hel"))
|
||||
p.Write([]byte("lo"))
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true")
|
||||
}
|
||||
if result != "hello" {
|
||||
t.Errorf("expected 'hello', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead_EmptyBuffer(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
result, ok := p.Read()
|
||||
if ok {
|
||||
t.Error("expected ok to be false for empty buffer")
|
||||
}
|
||||
if result != "" {
|
||||
t.Errorf("expected empty string, got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead_IncompleteUTF8(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
// Write incomplete multi-byte UTF-8 character
|
||||
// 中 (U+4E2D) = E4 B8 AD
|
||||
p.Write([]byte{0xE4, 0xB8})
|
||||
|
||||
result, ok := p.Read()
|
||||
if ok {
|
||||
t.Error("expected ok to be false for incomplete UTF-8")
|
||||
}
|
||||
if result != "" {
|
||||
t.Errorf("expected empty string, got '%s'", result)
|
||||
}
|
||||
|
||||
// Complete the character
|
||||
p.Write([]byte{0xAD})
|
||||
result, ok = p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true after completing UTF-8")
|
||||
}
|
||||
if result != "中" {
|
||||
t.Errorf("expected '中', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead_MixedASCIIAndUTF8(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("Hello 世界"))
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true")
|
||||
}
|
||||
if result != "Hello 世界" {
|
||||
t.Errorf("expected 'Hello 世界', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead_PartialMultibyteAtEnd(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
// "Hello" + partial "世" (E4 B8 96)
|
||||
p.Write([]byte("Hello"))
|
||||
p.Write([]byte{0xE4, 0xB8})
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true for valid portion")
|
||||
}
|
||||
if result != "Hello" {
|
||||
t.Errorf("expected 'Hello', got '%s'", result)
|
||||
}
|
||||
|
||||
// Complete the character
|
||||
p.Write([]byte{0x96})
|
||||
result, ok = p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok to be true after completing")
|
||||
}
|
||||
if result != "世" {
|
||||
t.Errorf("expected '世', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlush(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("hello"))
|
||||
|
||||
result := p.Flush()
|
||||
if result != "hello" {
|
||||
t.Errorf("expected 'hello', got '%s'", result)
|
||||
}
|
||||
|
||||
// Verify buffer is cleared
|
||||
result2, ok := p.Read()
|
||||
if ok {
|
||||
t.Error("expected ok to be false after flush")
|
||||
}
|
||||
if result2 != "" {
|
||||
t.Errorf("expected empty string after flush, got '%s'", result2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlush_EmptyBuffer(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
result := p.Flush()
|
||||
if result != "" {
|
||||
t.Errorf("expected empty string, got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlush_IncompleteUTF8(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte{0xE4, 0xB8})
|
||||
|
||||
result := p.Flush()
|
||||
// Flush returns everything including incomplete bytes
|
||||
if len(result) != 2 {
|
||||
t.Errorf("expected 2 bytes flushed, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReset(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("hello"))
|
||||
p.Reset()
|
||||
|
||||
result, ok := p.Read()
|
||||
if ok {
|
||||
t.Error("expected ok to be false after reset")
|
||||
}
|
||||
if result != "" {
|
||||
t.Errorf("expected empty string after reset, got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUtf8CharSize(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
testCases := []struct {
|
||||
b byte
|
||||
expected int
|
||||
}{
|
||||
{0x00, 1}, // ASCII
|
||||
{0x7F, 1}, // ASCII max
|
||||
{0xC0, 2}, // 2-byte start
|
||||
{0xDF, 2}, // 2-byte start
|
||||
{0xE0, 3}, // 3-byte start
|
||||
{0xEF, 3}, // 3-byte start
|
||||
{0xF0, 4}, // 4-byte start
|
||||
{0xF7, 4}, // 4-byte start
|
||||
{0x80, 1}, // Continuation byte (fallback)
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
size := p.utf8CharSize(tc.b)
|
||||
if size != tc.expected {
|
||||
t.Errorf("utf8CharSize(0x%X) = %d, expected %d", tc.b, size, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingScenario(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
// Simulate streaming: "Hello, 世界! 🌍"
|
||||
chunks := [][]byte{
|
||||
[]byte("Hello, "),
|
||||
{0xE4, 0xB8}, // partial 世
|
||||
{0x96, 0xE7}, // complete 世, partial 界
|
||||
{0x95, 0x8C}, // complete 界
|
||||
[]byte("! "),
|
||||
{0xF0, 0x9F}, // partial 🌍
|
||||
{0x8C, 0x8D}, // complete 🌍
|
||||
}
|
||||
|
||||
var results []string
|
||||
for _, chunk := range chunks {
|
||||
p.Write(chunk)
|
||||
if result, ok := p.Read(); ok {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
combined := strings.Join(results, "")
|
||||
if combined != "Hello, 世界! 🌍" {
|
||||
t.Errorf("expected 'Hello, 世界! 🌍', got '%s'", combined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidUTF8Output(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
testStrings := []string{
|
||||
"Hello World",
|
||||
"你好世界",
|
||||
"こんにちは",
|
||||
"🎉🎊🎁",
|
||||
"Mixed 混合 Текст ტექსტი",
|
||||
}
|
||||
|
||||
for _, s := range testStrings {
|
||||
p.Reset()
|
||||
p.Write([]byte(s))
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Errorf("expected ok for '%s'", s)
|
||||
}
|
||||
if !utf8.ValidString(result) {
|
||||
t.Errorf("invalid UTF-8 output for input '%s'", s)
|
||||
}
|
||||
if result != s {
|
||||
t.Errorf("expected '%s', got '%s'", s, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLargeData(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
// Generate large UTF-8 string
|
||||
var builder strings.Builder
|
||||
for i := 0; i < 1000; i++ {
|
||||
builder.WriteString("Hello 世界! ")
|
||||
}
|
||||
largeString := builder.String()
|
||||
|
||||
p.Write([]byte(largeString))
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok for large data")
|
||||
}
|
||||
if result != largeString {
|
||||
t.Error("large data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestByteByByteWriting(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
input := "Hello 世界"
|
||||
inputBytes := []byte(input)
|
||||
|
||||
var results []string
|
||||
for _, b := range inputBytes {
|
||||
p.Write([]byte{b})
|
||||
if result, ok := p.Read(); ok {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
combined := strings.Join(results, "")
|
||||
if combined != input {
|
||||
t.Errorf("expected '%s', got '%s'", input, combined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmoji4ByteUTF8(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
// 🎉 = F0 9F 8E 89
|
||||
emoji := "🎉"
|
||||
emojiBytes := []byte(emoji)
|
||||
|
||||
for i := 0; i < len(emojiBytes)-1; i++ {
|
||||
p.Write(emojiBytes[i : i+1])
|
||||
result, ok := p.Read()
|
||||
if ok && result != "" {
|
||||
t.Errorf("unexpected output before emoji complete: '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
p.Write(emojiBytes[len(emojiBytes)-1:])
|
||||
result, ok := p.Read()
|
||||
if !ok {
|
||||
t.Error("expected ok after completing emoji")
|
||||
}
|
||||
if result != emoji {
|
||||
t.Errorf("expected '%s', got '%s'", emoji, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuationBytesOnly(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
|
||||
// Write only continuation bytes (invalid UTF-8)
|
||||
p.Write([]byte{0x80, 0x80, 0x80})
|
||||
|
||||
result, ok := p.Read()
|
||||
// Should handle gracefully - either return nothing or return the bytes
|
||||
_ = result
|
||||
_ = ok
|
||||
}
|
||||
|
||||
func TestUTF8StreamParser_ConcurrentSafety(t *testing.T) {
|
||||
// Note: UTF8StreamParser doesn't have built-in locks,
|
||||
// so this test verifies it works with external synchronization
|
||||
p := NewUTF8StreamParser()
|
||||
var mu sync.Mutex
|
||||
const numGoroutines = 10
|
||||
const numOperations = 100
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numGoroutines)
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numOperations; j++ {
|
||||
mu.Lock()
|
||||
switch j % 4 {
|
||||
case 0:
|
||||
p.Write([]byte("test"))
|
||||
case 1:
|
||||
p.Read()
|
||||
case 2:
|
||||
p.Flush()
|
||||
case 3:
|
||||
p.Reset()
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestConsecutiveReads(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("hello"))
|
||||
|
||||
result1, ok1 := p.Read()
|
||||
if !ok1 || result1 != "hello" {
|
||||
t.Error("first read failed")
|
||||
}
|
||||
|
||||
result2, ok2 := p.Read()
|
||||
if ok2 || result2 != "" {
|
||||
t.Error("second read should return empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushThenWrite(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("first"))
|
||||
p.Flush()
|
||||
p.Write([]byte("second"))
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok || result != "second" {
|
||||
t.Errorf("expected 'second', got '%s'", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetThenWrite(t *testing.T) {
|
||||
p := NewUTF8StreamParser()
|
||||
p.Write([]byte("first"))
|
||||
p.Reset()
|
||||
p.Write([]byte("second"))
|
||||
|
||||
result, ok := p.Read()
|
||||
if !ok || result != "second" {
|
||||
t.Errorf("expected 'second', got '%s'", result)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user