useChat() streaming working in localhost but not docker deployment with nginx

[apensotti]

Description

AI Page

const { messages, input, handleInputChange, handleSubmit } = useChat();

/api/chat/route.ts

 import { LangChainAdapter, Message } from 'ai';
 import { unstable_noStore as noStore } from 'next/cache';


 // Allow streaming responses up to 30 seconds
 export const dynamic = 'force-dynamic';
 export const maxDuration = 30;

 noStore();
 export async function POST(req: Request) {
   const { messages }: { messages: Message[] } = await req.json();

   // Forward the request to your FastAPI backend
   const response = await fetch("https://www.moviewizard.com/wizapi/chat/stream", {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
     },
     body: JSON.stringify({
       messages,  // Forward the messages directly
       email: "youremail",  // Add email or other necessary fields
       session_id: ""  // Add the session_id if required
     }),
   });

   // Read the streaming response from FastAPI
   const reader = response.body?.getReader();
   const decoder = new TextDecoder();

   const stream = new ReadableStream({
     async start(controller) {
       if (!reader) {
         controller.close();
         return;
       }

       while (true) {
         const { done, value } = await reader.read();
         if (done) {
           controller.close();
           break;
         }

         // Decode and send chunks to the stream
         const chunk = decoder.decode(value);
        controller.enqueue(chunk);
       }
     }
   });

   // Use LangChainAdapter to convert the stream to a data stream response
   return LangChainAdapter.toDataStreamResponse(stream);
 }

Fast API endpoint code

class Chat:
    def __init__(self):
        self.session_id = ""
        self.messages = []
        self.full_message = ""
        self.query = ""

    async def handle_chat_stream(self, req):
        self.query = get_query(req)

        if req.session_id == "":
            self.messages = []
        else:
            self.messages = get_messages(req.session_id)

        chain = rag_chain()

        async def generator():
            async for chunk in chain.astream({"input": self.query, "chat_history": self.messages}):
                try:
                    answer = chunk.get("answer", "")
                    self.full_message += answer
                    yield str(answer)
                except Exception as e:
                    print(e)

        # Add the user query and the system's response to the chat history
        req.messages.append(HumanMessage(content=self.query))
        req.messages.append(SystemMessage(content=self.full_message))

        if req.session_id == "":
            await create_session(req)
        else:
            await update_session(req)

        # Return a StreamingResponse with properly formatted JSON chunks
        return StreamingResponse(generator(), media_type='text/plain')

Code example

No response

Additional context

https://youtu.be/nK7fAOeQWeE

first one is local host, works fine. The second is the deployed website, which doesn't work.

I tried https://sdk.vercel.ai/docs/troubleshooting/common-issues/streaming-not-working-on-vercel

That solution didn't work.