From 75813087a69cd2b5ad63f506dd519eb93ac3a00d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Piotr=20C=C5=82apa?= Date: Tue, 9 Apr 2024 09:48:45 +0000 Subject: [PATCH] Added a long-form inference example --- Long-form inference.ipynb | 441 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100644 Long-form inference.ipynb diff --git a/Long-form inference.ipynb b/Long-form inference.ipynb new file mode 100644 index 0000000..923dccf --- /dev/null +++ b/Long-form inference.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "61fe7942", + "metadata": {}, + "source": [ + "# Inference examples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdac8407", + "metadata": {}, + "outputs": [], + "source": [ + "# Do not forget to install all dependencies first:\n", + "!pip install -Uqq WhisperSpeech" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "264f56be", + "metadata": {}, + "outputs": [], + "source": [ + "def is_colab():\n", + " try: import google.colab; return True\n", + " except: return False\n", + "\n", + "import torch\n", + "if not torch.cuda.is_available():\n", + " if is_colab(): raise BaseException(\"Please change the runtime type to GPU. In the menu: Runtime -> Change runtime type (the free T4 instance is enough)\")\n", + " else: raise BaseException(\"Currently the example notebook requires CUDA, make sure you are running this on a machine with a GPU.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7885be1", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f32e8d5f", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "\n", + "from IPython.display import Markdown, HTML" + ] + }, + { + "cell_type": "markdown", + "id": "6f7e6e3b", + "metadata": {}, + "source": [ + "## The whole pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93525c56", + "metadata": {}, + "outputs": [], + "source": [ + "from whisperspeech.pipeline import Pipeline\n", + "import torchaudio\n", + "import nltk\n", + "from nltk.tokenize import sent_tokenize\n", + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1500ed40", + "metadata": {}, + "outputs": [], + "source": [ + "pipe = Pipeline(torch_compile=True)\n", + "nltk.download('punkt')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "517e4e78", + "metadata": {}, + "outputs": [], + "source": [ + "sample_text = \"\"\"Electromagnetism is a fundamental force of nature that encompasses the interaction between\n", + "electrically charged particles. It is described by Maxwell's equations, which unify electricity, magnetism,\n", + "and light into a single theory. In essence, electric charges produce electric fields that exert forces on\n", + "other charges, while moving charges (currents) generate magnetic fields. These magnetic fields, in turn,\n", + "can affect the motion of charges and currents. The interaction between electric and magnetic fields propagates\n", + "through space as electromagnetic waves, which include visible light, radio waves, and X-rays. Electromagnetic\n", + "forces are responsible for practically all the phenomena encountered in daily life, excluding gravity.\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bb5f998", + "metadata": {}, + "outputs": [], + "source": [ + "def split_and_prepare_text(text, cps=14):\n", + " chunks = []\n", + " sentences = sent_tokenize(text)\n", + " chunk = \"\"\n", + " for sentence in sentences:\n", + " # replace fancy punctuation that was unseen during training\n", + " sentence = re.sub('[()]', \",\", sentence).strip()\n", + " sentence = re.sub(\",+\", \",\", sentence)\n", + " sentence = re.sub('\"+', \"\", sentence)\n", + " sentence = re.sub(\"/\", \"\", sentence)\n", + " # merge until the result is < 20s\n", + " if len(chunk) + len(sentence) < 20*cps:\n", + " chunk += \" \" + sentence\n", + " else:\n", + " chunks.append(chunk)\n", + " chunk = sentence\n", + " if chunk: chunks.append(chunk)\n", + " return chunks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d65c1c3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[\" Electromagnetism is a fundamental force of nature that encompasses the interaction between\\nelectrically charged particles. It is described by Maxwell's equations, which unify electricity, magnetism,\\nand light into a single theory.\",\n", + " 'In essence, electric charges produce electric fields that exert forces on\\nother charges, while moving charges ,currents, generate magnetic fields. These magnetic fields, in turn,\\ncan affect the motion of charges and currents.',\n", + " 'The interaction between electric and magnetic fields propagates\\nthrough space as electromagnetic waves, which include visible light, radio waves, and X-rays. Electromagnetic\\nforces are responsible for practically all the phenomena encountered in daily life, excluding gravity.']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "split_and_prepare_text(sample_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "223f3542", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_long(text, cps=14, overlap=100, output=None, speaker=None):\n", + " global atoks, stoks\n", + " chunks = split_and_prepare_text(text)\n", + " speaker = pipe.default_speaker if speaker is None else speaker\n", + " r = []\n", + " old_stoks = None\n", + " old_atoks = None\n", + " for chunk in chunks:\n", + " print(chunk)\n", + " stoks = pipe.t2s.generate(chunk, cps=cps, show_progress_bar=False)[0]\n", + " stoks = stoks[stoks != 512]\n", + " if old_stoks is not None:\n", + " assert(len(stoks) < 750-overlap)\n", + " stoks = torch.cat([old_stoks[-overlap:], stoks])\n", + " atoks_prompt = old_atoks[:,:,-overlap*3:]\n", + " else:\n", + " atoks_prompt = None\n", + " atoks = pipe.s2a.generate(stoks, atoks_prompt=atoks_prompt, speakers=speaker.unsqueeze(0), show_progress_bar=False)\n", + " if atoks_prompt is not None: atoks = atoks[:,:,overlap*3+1:]\n", + " r.append(atoks)\n", + " old_stoks = stoks\n", + " old_atoks = atoks\n", + " pipe.vocoder.decode_to_notebook(atoks)\n", + " audios = []\n", + " for i,atoks in enumerate(r):\n", + " if i != 0: audios.append(torch.zeros((1, int(24000*0.5)), dtype=atoks.dtype, device=atoks.device))\n", + " audios.append(pipe.vocoder.decode(atoks))\n", + " if output:\n", + " torchaudio.save(output, torch.cat(audios, -1).cpu(), 24000)\n", + " from IPython.display import display, HTML, Audio\n", + " display(HTML(f'Listen to {output}'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18bff630", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Electromagnetism is a fundamental force of nature that encompasses the interaction between\n", + "electrically charged particles. It is described by Maxwell's equations, which unify electricity, magnetism,\n", + "and light into a single theory.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "In essence, electric charges produce electric fields that exert forces on\n", + "other charges, while moving charges ,currents, generate magnetic fields. These magnetic fields, in turn,\n", + "can affect the motion of charges and currents.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The interaction between electric and magnetic fields propagates\n", + "through space as electromagnetic waves, which include visible light, radio waves, and X-rays. Electromagnetic\n", + "forces are responsible for practically all the phenomena encountered in daily life, excluding gravity.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[torch.Size([1, 395200]), torch.Size([1, 12000]), torch.Size([1, 374720]), torch.Size([1, 12000]), torch.Size([1, 478400])]\n" + ] + }, + { + "data": { + "text/html": [ + "Listen to test.mp3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "generate_long(sample_text, output=\"test.mp3\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce7fe096", + "metadata": {}, + "outputs": [], + "source": [ + "speaker = pipe.extract_spk_emb('https://upload.wikimedia.org/wikipedia/commons/7/75/Winston_Churchill_-_Be_Ye_Men_of_Valour.ogg')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9738179", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Electromagnetism is a fundamental force of nature that encompasses the interaction between electrically charged particles. It is described by Maxwell's equations, which unify electricity, magnetism, and light into a single theory.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "In essence, electric charges produce electric fields that exert forces on other charges, while moving charges ,currents, generate magnetic fields. These magnetic fields, in turn, can affect the motion of charges and currents.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The interaction between electric and magnetic fields propagates through space as electromagnetic waves, which include visible light, radio waves, and X-rays. Electromagnetic forces are responsible for practically all the phenomena encountered in daily life, excluding gravity.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[torch.Size([1, 391360]), torch.Size([1, 12000]), torch.Size([1, 383360]), torch.Size([1, 12000]), torch.Size([1, 481280])]\n" + ] + }, + { + "data": { + "text/html": [ + "Listen to test-churchill.mp3" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "generate_long(sample_text, output=\"test-churchill.mp3\", speaker=speaker)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20123e46", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}