From 7cc945aded8fa90c32f1f4229c767400e991e191 Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Wed, 31 Jan 2024 16:45:31 +0100 Subject: [PATCH 1/6] self.client_uid accessed without being defined --- whisper_live/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/whisper_live/server.py b/whisper_live/server.py index be0e9598..5a5b08ec 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -136,6 +136,7 @@ def recv_audio(self, ) logging.info(f"Running TensorRT backend.") except Exception as e: + self.client_uid = options["uid"] websocket.send( json.dumps( { From 82002075304b920ca708b8fcae26b9342cbee77f Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Thu, 1 Feb 2024 13:16:23 +0100 Subject: [PATCH 2/6] Center transcription div --- Audio-Transcription-Chrome/content.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Audio-Transcription-Chrome/content.js b/Audio-Transcription-Chrome/content.js index 1c89a2a8..3c911e48 100644 --- a/Audio-Transcription-Chrome/content.js +++ b/Audio-Transcription-Chrome/content.js @@ -59,7 +59,7 @@ function init_element() { elem_container = document.createElement('div'); elem_container.id = "transcription"; - elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;'; + elem_container.style.cssText = 'padding-top:16px;font-size:18px;position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);z-index: 9999;line-height:18px;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;'; for (var i = 0; i < 4; i++) { elem_text = document.createElement('span'); From e1f531eccfa2f8342fd81da5bdc157619a53f9f9 Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Thu, 1 Feb 2024 13:56:09 +0100 Subject: [PATCH 3/6] Remove duplicate assignment --- whisper_live/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/whisper_live/client.py b/whisper_live/client.py index 96971c96..34fadd4b 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -78,8 +78,6 @@ def __init__( self.rate = 16000 self.record_seconds = 60000 self.recording = False - self.multilingual = False - self.language = None self.task = "transcribe" self.uid = str(uuid.uuid4()) self.waiting = False From 5b28ddefbdee16f8a5edfcccd4341e46beb0422a Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Thu, 1 Feb 2024 14:44:07 +0100 Subject: [PATCH 4/6] Chrome extension - QOL. Remove multilingual part. --- Audio-Transcription-Chrome/options.js | 4 - Audio-Transcription-Chrome/popup.html | 185 +++++++++++++------------- Audio-Transcription-Chrome/popup.js | 23 ---- 3 files changed, 93 insertions(+), 119 deletions(-) diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index 7e8ae5b2..435d75b2 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -93,14 +93,10 @@ async function startRecord(option) { const socket = new WebSocket(`ws://${option.host}:${option.port}/`); let isServerReady = false; let language = option.language; - if (language === null && !option.multilingual) { - language = 'en'; - } socket.onopen = function(e) { socket.send( JSON.stringify({ uid: uuid, - multilingual: option.multilingual, language: option.language, task: option.task, model: option.modelSize diff --git a/Audio-Transcription-Chrome/popup.html b/Audio-Transcription-Chrome/popup.html index 3fae5484..8d45bcb2 100644 --- a/Audio-Transcription-Chrome/popup.html +++ b/Audio-Transcription-Chrome/popup.html @@ -15,112 +15,109 @@ -
- - -
- + + + + + + + + + + + + + + + + + + + - - - - - + + + - - - + + + + - - - - + + + + + + - - - - - - - + + + + + + + + + + - - - - - - - - - - - - + + + - - - - - - - - - - - + + + + + - - - - - - - + + + + + - - - - - - - - - - - - - - + + + + + + - - - + + + + + + + + + + + - - + + - - - - - - + + + + + + + + + + + +
@@ -134,11 +131,15 @@
diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js index 9a708d12..e71c62d5 100644 --- a/Audio-Transcription-Chrome/popup.js +++ b/Audio-Transcription-Chrome/popup.js @@ -4,7 +4,6 @@ document.addEventListener("DOMContentLoaded", function () { const stopButton = document.getElementById("stopCapture"); const useServerCheckbox = document.getElementById("useServerCheckbox"); - const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox'); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); const modelSizeDropdown = document.getElementById('modelSizeDropdown'); @@ -32,14 +31,6 @@ document.addEventListener("DOMContentLoaded", function () { } }); - chrome.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => { - if (useMultilingualModelState !== undefined) { - useMultilingualCheckbox.checked = useMultilingualModelState; - languageDropdown.disabled = !useMultilingualModelState; - taskDropdown.disabled = !useMultilingualModelState; - } - }); - chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { if (storedLanguage !== undefined) { languageDropdown.value = storedLanguage; @@ -86,7 +77,6 @@ document.addEventListener("DOMContentLoaded", function () { tabId: currentTab.id, host: host, port: port, - useMultilingual: useMultilingualCheckbox.checked, language: selectedLanguage, task: selectedTask, modelSize: selectedModelSize @@ -129,7 +119,6 @@ document.addEventListener("DOMContentLoaded", function () { startButton.disabled = isCapturing; stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; - useMultilingualCheckbox.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing; startButton.classList.toggle("disabled", isCapturing); @@ -142,18 +131,6 @@ document.addEventListener("DOMContentLoaded", function () { chrome.storage.local.set({ useServerState }); }); - useMultilingualCheckbox.addEventListener('change', function() { - const useMultilingualModelState = useMultilingualCheckbox.checked; - if (useMultilingualModelState) { - languageDropdown.disabled = false; - taskDropdown.disabled = false; - } else { - languageDropdown.disabled = true; - taskDropdown.disabled = true; - } - chrome.storage.local.set({ useMultilingualModelState }); - }); - languageDropdown.addEventListener('change', function() { if (languageDropdown.value === "") { selectedLanguage = null; From 32ed089a76efdd477406456697b1abbb4eadbaf9 Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Thu, 1 Feb 2024 14:54:29 +0100 Subject: [PATCH 5/6] Change faster whisper to work with new extension --- whisper_live/server.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index f2746084..e3498a60 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -155,7 +155,7 @@ def recv_audio(self, options["model"] = faster_whisper_custom_model_path client = ServeClientFasterWhisper( websocket, - multilingual=options["multilingual"], + multilingual=False, language=options["language"], task=options["task"], client_uid=options["uid"], @@ -585,13 +585,11 @@ def __init__( "tiny", "tiny.en", "base", "base.en", "small", "small.en", "medium", "medium.en", "large-v2", "large-v3", ] - - self.multilingual = multilingual if not os.path.exists(model): - self.model_size_or_path = self.get_model_size(model) + self.model_size_or_path = self.check_valid_model(model) else: self.model_size_or_path = model - self.language = language if self.multilingual else "en" + self.language = "en" if self.model_size_or_path.endswith("en") else language self.task = task self.initial_prompt = initial_prompt self.vad_parameters = vad_parameters or {"threshold": 0.5} @@ -600,7 +598,7 @@ def __init__( if self.model_size_or_path == None: return - + self.transcriber = WhisperModel( self.model_size_or_path, device=device, @@ -620,9 +618,15 @@ def __init__( ) ) - def get_model_size(self, model_size): + def check_valid_model(self, model_size): """ - Returns the whisper model size based on multilingual. + Check if it's a valid whisper model size. + + Args: + model_size (str): The name of the model size to check. + + Returns: + str: The model size if valid, None otherwise. """ if model_size not in self.model_sizes: self.websocket.send( @@ -635,15 +639,6 @@ def get_model_size(self, model_size): ) ) return None - - if model_size.endswith("en") and self.multilingual: - logging.info(f"Setting multilingual to false with {model_size} which is english only model.") - self.multilingual = False - - if not model_size.endswith("en") and not self.multilingual: - logging.info(f"Setting multilingual to true with multilingual model {model_size}.") - self.multilingual = True - return model_size def speech_to_text(self): From e69757487043b04df5a2d85d11c8f799eb3a4054 Mon Sep 17 00:00:00 2001 From: Sasa Trivic Date: Fri, 2 Feb 2024 13:56:26 +0100 Subject: [PATCH 6/6] Remove multilingual from client. Remove multilingual from faster whisper backend. Disable task dropdown when capturing in chrome extension. --- Audio-Transcription-Chrome/popup.js | 2 +- whisper_live/client.py | 16 +++++----------- whisper_live/server.py | 3 --- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js index e71c62d5..3c40aa8b 100644 --- a/Audio-Transcription-Chrome/popup.js +++ b/Audio-Transcription-Chrome/popup.js @@ -120,7 +120,7 @@ document.addEventListener("DOMContentLoaded", function () { stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing; - + taskDropdown.disabled = isCapturing; startButton.classList.toggle("disabled", isCapturing); stopButton.classList.toggle("disabled", !isCapturing); } diff --git a/whisper_live/client.py b/whisper_live/client.py index 09e06c53..46d52300 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -76,7 +76,6 @@ def __init__( self, host=None, port=None, - is_multilingual=False, lang=None, translate=False, model="small", @@ -92,8 +91,7 @@ def __init__( Args: host (str): The hostname or IP address of the server. port (int): The port number for the WebSocket server. - is_multilingual (bool, optional): Specifies if multilingual transcription is enabled. Default is False. - lang (str, optional): The selected language for transcription when multilingual is disabled. Default is None. + lang (str, optional): The selected language for transcription. Default is None. translate (bool, optional): Specifies if the task is translation. Default is False. """ self.chunk = 4096 @@ -107,7 +105,6 @@ def __init__( self.waiting = False self.last_response_recieved = None self.disconnect_if_no_response_for = 15 - self.multilingual = is_multilingual self.language = lang self.model = model self.server_error = False @@ -245,7 +242,7 @@ def on_open(self, ws): """ Callback function called when the WebSocket connection is successfully opened. - Sends an initial configuration message to the server, including client UID, multilingual mode, + Sends an initial configuration message to the server, including client UID, language selection, and task type. Args: @@ -259,7 +256,6 @@ def on_open(self, ws): json.dumps( { "uid": self.uid, - "multilingual": self.multilingual, "language": self.language, "task": self.task, "model": self.model, @@ -546,8 +542,7 @@ class TranscriptionClient: Args: host (str): The hostname or IP address of the server. port (int): The port number to connect to on the server. - is_multilingual (bool, optional): Indicates whether the transcription should support multiple languages (default is False). - lang (str, optional): The primary language for transcription (used if `is_multilingual` is False). Default is None, which defaults to English ('en'). + lang (str, optional): The primary language for transcription. Default is None, which defaults to English ('en'). translate (bool, optional): Indicates whether translation tasks are required (default is False). Attributes: @@ -556,19 +551,18 @@ class TranscriptionClient: Example: To create a TranscriptionClient and start transcription on microphone audio: ```python - transcription_client = TranscriptionClient(host="localhost", port=9090, is_multilingual=True) + transcription_client = TranscriptionClient(host="localhost", port=9090) transcription_client() ``` """ def __init__(self, host, port, - is_multilingual=False, lang=None, translate=False, model="small", ): - self.client = Client(host, port, is_multilingual, lang, translate, model) + self.client = Client(host, port, lang, translate, model) def __call__(self, audio=None, hls_url=None): """ diff --git a/whisper_live/server.py b/whisper_live/server.py index c8b7ca72..4a8a1d8d 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -155,7 +155,6 @@ def recv_audio(self, options["model"] = faster_whisper_custom_model_path client = ServeClientFasterWhisper( websocket, - multilingual=False, language=options["language"], task=options["task"], client_uid=options["uid"], @@ -559,7 +558,6 @@ def __init__( websocket, task="transcribe", device=None, - multilingual=False, language=None, client_uid=None, model="small.en", @@ -576,7 +574,6 @@ def __init__( websocket (WebSocket): The WebSocket connection for the client. task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe". device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None. - multilingual (bool, optional): Whether the client supports multilingual transcription. Defaults to False. language (str, optional): The language for transcription. Defaults to None. client_uid (str, optional): A unique identifier for the client. Defaults to None.