Add out-of-band responses + controlled response timing #1320
Conversation
|
| @@ -287,9 +288,6 @@ def __init__( | |||
| ValueError: If the API key is not provided and cannot be found in environment variables. | |||
| """ | |||
| super().__init__() | |||
| self._capabilities = Capabilities( | |||
There was a problem hiding this comment.
We should keep the capabilities property
There was a problem hiding this comment.
Was that a new addition? I just updated with the differences
| @@ -880,7 +938,7 @@ def session_update( | |||
| input_audio_format: api_proto.AudioFormat | None = None, | |||
| output_audio_format: api_proto.AudioFormat | None = None, | |||
| input_audio_transcription: InputTranscriptionOptions | None = None, | |||
| turn_detection: ServerVadOptions | None = None, | |||
| turn_detection: ServerVadOptions | Literal["None"] | None = None, | |||
There was a problem hiding this comment.
I didn't see this PR was also adding None to the turn_detection.
It has been done inside this PR: #1347
We're still interested to have this out-of-band responses tho! :)
There was a problem hiding this comment.
I integrated the changes! It should all work now :D
| if instructions is not None: | ||
| self._sess._queue_msg( | ||
| { | ||
| "type": "response.create", | ||
| "response": { | ||
| "conversation": conversation, | ||
| "metadata": metadata, | ||
| "instructions": instructions, | ||
| "modalities": modalities, | ||
| }, | ||
| } | ||
| ) | ||
| else: | ||
| self._sess._queue_msg( | ||
| { | ||
| "type": "response.create", | ||
| "response": { | ||
| "conversation": conversation, | ||
| "metadata": metadata, | ||
| "modalities": modalities, | ||
| }, | ||
| } | ||
| ) |
There was a problem hiding this comment.
Is the reason for this if-else condition the fact that OAI will returns an error if instructions is set to None/null
There was a problem hiding this comment.
I forgot but that sounds about right, it works without it now though
I recently came across new additions that OpenAI developed and would love for them to be included in Livekit!
I worked on out-of-band responses and an extra VAD option where responses are not automatic. I also added an option to turn server VAD off by
turn_detection="None"insession.update().Please let me know your thoughts! :-)