Update app.py

app.py

@@ -3,8 +3,10 @@ from json import dumps, loads
 from re import sub
 from urllib.parse import urlparse, urlunparse
 
+from cchardet import detect
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, StreamingResponse, HTMLResponse
+from ftfy import fix_text
 from httpx import AsyncClient, Limits, Timeout
 
 app = FastAPI(title='PROXI-API')
@@ -70,35 +72,41 @@ def normalize_headers(request: Request, original_url: str) -> dict:
 
 def decode(chunk: bytes) -> str:
     try:
-        content = chunk.decode('utf-8')
+        return fix_text(chunk.decode('utf-8'))
     except UnicodeDecodeError:
+        pass
+    detected = detect(chunk)
+    if detected['encoding'] and detected['confidence'] > 0.75:
         try:
-            content = chunk.decode('latin-1')
+            return fix_text(chunk.decode(detected['encoding'].lower()))
+        except (UnicodeDecodeError, LookupError):
+            pass
+    for encoding in [
+        'cp1251', 'cp1252', 'iso-8859-5', 'iso-8859-1', 'windows-1252',
+        'gb18030', 'big5', 'gb2312', 'shift_jis', 'euc-kr', 'cp1256',
+        'iso-8859-7', 'koi8-r', 'cp866', 'mac_cyrillic'
+    ]:
+        try:
+            return fix_text(chunk.decode(encoding))
         except UnicodeDecodeError:
-            try:
-                content = chunk.decode('iso-8859-1')
-            except UnicodeDecodeError:
-                content = chunk.decode(errors='replace')
-    return content
+            continue
+    return fix_text(chunk.decode('latin-1', errors='replace'))
 
 
-def format_chunk(chunk: bytes, model: str) -> bytes | str | dict:
-    chunk_id = 'chatcmpl-AQ8Lzxlg8eSCB1lgVmboiXwZiexqE'
-    timestamp = int(datetime.now().timestamp())
-    
+def format_chunk(chunk: bytes, model: str) -> str:
     data = {
-        "id": chunk_id,
-        "object": "chat.completion.chunk",
-        "created": timestamp,
-        "model": model,
-        "system_fingerprint": "fp_67802d9a6d",
-        "choices": [
+        'id': 'chatcmpl-AQ8Lzxlg8eSCB1lgVmboiXwZiexqE',
+        'object': 'chat.completion.chunk',
+        'created': int(datetime.now().timestamp()),
+        'model': model,
+        'system_fingerprint': 'fp_67802d9a6d',
+        'choices': [
             {
-                "index": 0,
-                "delta": {
-                    "content": decode(chunk)
+                'index': 0,
+                'delta': {
+                    'content': decode(chunk)
                 },
-                "finish_reason": None
+                'finish_reason': None
             }
         ]
     }