fix fastapi, nginx, and entrypoint for sse

Author: ahmedkhaleel2004

backend/app/routers/generate.py

@@ -15,6 +15,7 @@
 import re
 import json
 import asyncio
+from fastapi.responses import JSONResponse
 
 # from app.services.claude_service import ClaudeService
 # from app.core.limiter import limiter
@@ -276,6 +277,12 @@ def replace_path(match):
 
 @router.post("/stream")
 async def generate_stream(request: Request, body: ApiRequest):
+    headers = {
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "X-Accel-Buffering": "no",  # Important for Nginx
+    }
+
     try:
         # Initial validation checks
         if len(body.instructions) > 1000:
@@ -415,6 +422,8 @@ async def event_generator():
             except Exception as e:
                 yield f"data: {json.dumps({'error': str(e)})}\n\n"
 
-        return StreamingResponse(event_generator(), media_type="text/event-stream")
+        return StreamingResponse(
+            event_generator(), media_type="text/event-stream", headers=headers
+        )
     except Exception as e:
-        return {"error": str(e)}
+        return JSONResponse(status_code=500, content={"error": str(e)})

backend/entrypoint.sh

@@ -7,7 +7,14 @@ if [ "$ENVIRONMENT" = "development" ]; then
     exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
 elif [ "$ENVIRONMENT" = "production" ]; then
     echo "Starting in production mode with multiple workers..."
-    exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --timeout-keep-alive 300 --workers 2
+    exec uvicorn app.main:app \
+        --host 0.0.0.0 \
+        --port 8000 \
+        --timeout-keep-alive 300 \
+        --workers 2 \
+        --backlog 2048 \
+        --limit-concurrency 500 \
+        --no-access-log
 else
     echo "ENVIRONMENT must be set to either 'development' or 'production'"
     exit 1

backend/nginx/api.conf

@@ -16,27 +16,28 @@ server {
         include proxy_params;
         proxy_redirect off;
 
-        # Add these new settings
+        # SSE specific settings
         proxy_buffering off;
+        proxy_cache off;
         proxy_http_version 1.1;
         proxy_set_header Connection '';
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
 
-        # Increase timeouts
-        proxy_connect_timeout 300;
-        proxy_send_timeout 300;
-        proxy_read_timeout 300;
-        send_timeout 300;
+        # Increase timeouts significantly
+        proxy_connect_timeout 300s;
+        proxy_send_timeout 300s;
+        proxy_read_timeout 300s;
+        send_timeout 300s;
 
-        # Add error handling
-        proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
-        proxy_next_upstream_tries 2;
+        # Important for SSE
+        chunked_transfer_encoding on;
+        proxy_set_header X-Accel-Buffering no;
 
         # Keep connection alive
-        keepalive_timeout 300;
-        keepalive_requests 100;
+        keepalive_timeout 300s;
+        keepalive_requests 1000;
     }
 
     # Return 444 for everything else (no response, just close connection)