From a54a838e814880b248ae2f07d855113ddc95bd03 Mon Sep 17 00:00:00 2001 From: Amery2010 Date: Sat, 8 Jun 2024 22:28:06 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20Currently,=20you=20can=20use=20Gemini?= =?UTF-8?q?=E2=80=99s=20built-in=20audio=20recognition=20capabilities=20to?= =?UTF-8?q?=20achieve=20voice=20communication.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/globals.css | 13 +++++ app/page.tsx | 100 ++++++++++---------------------- components/AudioPlayer.tsx | 36 ++++++++++++ components/MessageItem.tsx | 36 ++++++++++-- components/ResponsiveDialog.tsx | 2 +- components/Setting.tsx | 11 ++++ components/ui/switch.tsx | 29 +++++++++ hooks/useAudio.ts | 53 +++++++++++++++++ hooks/useMediaQuery.ts | 4 +- locales/ar-SA.json | 5 +- locales/de-DE.json | 5 +- locales/en.json | 5 +- locales/es-ES.json | 5 +- locales/fr-FR.json | 5 +- locales/ja-JP.json | 5 +- locales/ko-KR.json | 5 +- locales/pt-BR.json | 5 +- locales/ru-RU.json | 5 +- locales/zh-TW.json | 5 +- locales/zh.json | 5 +- package.json | 4 +- pnpm-lock.yaml | 91 ++++++++++++++++++++++------- store/setting.ts | 7 +++ types.d.ts | 1 + utils/Recorder.ts | 35 +++-------- 25 files changed, 341 insertions(+), 136 deletions(-) create mode 100644 components/AudioPlayer.tsx create mode 100644 components/ui/switch.tsx create mode 100644 hooks/useAudio.ts diff --git a/app/globals.css b/app/globals.css index 8c92423..c74bcad 100644 --- a/app/globals.css +++ b/app/globals.css @@ -180,6 +180,19 @@ html.dark .hljs-warpper ::-webkit-scrollbar-thumb:hover { border-radius: 4px; margin-bottom: 8px; } +.yarl__root .yarl__container { + background-color: rgba(0, 0, 0, 0.8); +} +.audio-slider span[role='slider'] { + cursor: pointer; + width: 16px; + height: 16px; + transition: all 150ms linear; +} +.audio-slider span[role='slider']:hover { + width: 20px; + height: 20px; +} @media (max-width: 768px) { .hljs-warpper { diff --git a/app/page.tsx b/app/page.tsx index 68ba2d3..ef916a4 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,7 +1,7 @@ 'use client' import dynamic from 'next/dynamic' import { useRef, useState, useMemo, KeyboardEvent, useEffect, useCallback } from 'react' -import { EdgeSpeech, SpeechRecognition, getRecordMineType } from '@xiangfa/polly' +import { EdgeSpeech, getRecordMineType } from '@xiangfa/polly' import SiriWave from 'siriwave' import { MessageCircleHeart, @@ -61,7 +61,6 @@ export default function Home() { const audioStreamRef = useRef() const edgeSpeechRef = useRef() const audioRecordRef = useRef() - const speechRecognitionRef = useRef() const speechQueue = useRef() const messagesRef = useRef(useMessageStore.getState().messages) const messageStore = useMessageStore() @@ -72,12 +71,10 @@ export default function Home() { const [content, setContent] = useState('') const [subtitle, setSubtitle] = useState('') const [errorMessage, setErrorMessage] = useState('') - // const [isRecording, setIsRecording] = useState(false) - // const [recordTimer, setRecordTimer] = useState() const [recordTime, setRecordTime] = useState(0) const [settingOpen, setSetingOpen] = useState(false) const [speechSilence, setSpeechSilence] = useState(false) - const [disableSpeechRecognition, setDisableSpeechRecognition] = useState(false) + const [isRecording, setIsRecording] = useState(false) const [status, setStatus] = useState<'thinkng' | 'silence' | 'talking'>('silence') const statusText = useMemo(() => { switch (status) { @@ -95,6 +92,9 @@ export default function Home() { const supportAttachment = useMemo(() => { return !OldTextModel.includes(settingStore.model as Model) }, [settingStore.model]) + const supportSpeechRecognition = useMemo(() => { + return !OldTextModel.includes(settingStore.model as Model) && !OldVisionModel.includes(settingStore.model as Model) + }, [settingStore.model]) const isUploading = useMemo(() => { for (const file of attachmentStore.files) { if (file.status === 'PROCESSING') return true @@ -276,6 +276,9 @@ export default function Home() { } }, onFinish: async () => { + if (talkMode === 'voice') { + setStatus('silence') + } scrollToBottom() saveMessage() if (maxHistoryLength > 0) { @@ -432,47 +435,17 @@ export default function Home() { } }, []) - // const startRecordTime = useCallback(() => { - // const intervalTimer = setInterval(() => { - // setRecordTime((time) => time + 1) - // }, 1000) - // setRecordTimer(intervalTimer) - // }, []) - - // const endRecordTimer = useCallback(() => { - // clearInterval(recordTimer) - // }, [recordTimer]) - - // const handleRecorder = useCallback(() => { - // if (!checkAccessStatus()) return false - // if (!audioStreamRef.current) { - // audioStreamRef.current = new AudioStream() - // } - // if (speechRecognitionRef.current) { - // const { talkMode } = useSettingStore.getState() - // if (isRecording) { - // speechRecognitionRef.current.stop() - // endRecordTimer() - // setRecordTime(0) - // if (talkMode === 'voice') { - // handleSubmit(speechRecognitionRef.current.text) - // } - // setIsRecording(false) - // } else { - // speechRecognitionRef.current.start() - // setIsRecording(true) - // startRecordTime() - // } - // } - // }, [checkAccessStatus, handleSubmit, startRecordTime, endRecordTimer, isRecording]) - const handleRecorder = useCallback(() => { if (!checkAccessStatus()) return false if (!audioStreamRef.current) { audioStreamRef.current = new AudioStream() } - if (!audioRecordRef.current) { + if (!audioRecordRef.current || audioRecordRef.current.autoStop !== settingStore.autoStopRecord) { audioRecordRef.current = new AudioRecorder({ + autoStop: settingStore.autoStopRecord, + onStart: () => { + setIsRecording(true) + }, onTimeUpdate: (time) => { setRecordTime(time) }, @@ -481,15 +454,18 @@ export default function Home() { const file = new File([audioData], `${Date.now()}.${recordType.extension}`, { type: recordType.mineType }) const recordDataURL = await readFileAsDataURL(file) handleSubmit(recordDataURL) + setIsRecording(false) }, }) - } - if (audioRecordRef.current.isRecording) { - audioRecordRef.current.stop() - } else { audioRecordRef.current.start() + } else { + if (audioRecordRef.current.isRecording) { + audioRecordRef.current.stop() + } else { + audioRecordRef.current.start() + } } - }, [checkAccessStatus, handleSubmit]) + }, [settingStore.autoStopRecord, checkAccessStatus, handleSubmit]) const handleStopTalking = useCallback(() => { setSpeechSilence(true) @@ -500,14 +476,14 @@ export default function Home() { const handleKeyDown = useCallback( (ev: KeyboardEvent) => { - if (ev.key === 'Enter' && !ev.shiftKey && !audioRecordRef.current?.isRecording) { + if (ev.key === 'Enter' && !ev.shiftKey && !isRecording) { if (!checkAccessStatus()) return false // Prevent the default carriage return and line feed behavior ev.preventDefault() handleSubmit(content) } }, - [content, handleSubmit, checkAccessStatus], + [content, handleSubmit, checkAccessStatus, isRecording], ) const handleFileUpload = useCallback( @@ -569,20 +545,6 @@ export default function Home() { requestAnimationFrame(scrollToBottom) }, [messagesRef.current.length, scrollToBottom]) - useEffect(() => { - try { - speechRecognitionRef.current = new SpeechRecognition({ - locale: settingStore.sttLang, - onUpdate: (text) => { - setContent(text) - }, - }) - } catch (err) { - console.error(err) - setDisableSpeechRecognition(true) - } - }, [settingStore.sttLang]) - useEffect(() => { const setting = useSettingStore.getState() if (setting.ttsLang !== '') { @@ -688,7 +650,7 @@ export default function Home() { )}
- {!disableSpeechRecognition ? ( + {supportSpeechRecognition ? ( @@ -704,7 +666,7 @@ export default function Home() { autoFocus className={cn( 'h-auto max-h-[120px] w-full resize-none border-none bg-transparent px-2 text-sm leading-6 transition-[height] focus-visible:outline-none', - disableSpeechRecognition ? 'pr-9' : 'pr-[72px]', + !supportSpeechRecognition ? 'pr-9' : 'pr-[72px]', )} style={{ height: textareaHeight > 24 ? `${textareaHeight}px` : 'auto' }} value={content} @@ -731,15 +693,15 @@ export default function Home() { ) : null} - {!disableSpeechRecognition ? ( + {supportSpeechRecognition ? ( - +
handleRecorder()} > - +
handleSubmit(content)} > @@ -808,7 +770,7 @@ export default function Home() { disabled={status === 'thinkng'} onClick={() => handleRecorder()} > - {audioRecordRef.current?.isRecording ? formatTime(recordTime) : } + {isRecording ? formatTime(recordTime) : } )}
- setSetingOpen(false)} /> + setSetingOpen(false)} /> ) } diff --git a/components/AudioPlayer.tsx b/components/AudioPlayer.tsx new file mode 100644 index 0000000..8875733 --- /dev/null +++ b/components/AudioPlayer.tsx @@ -0,0 +1,36 @@ +import { memo } from 'react' +import { Play, Pause } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Slider } from '@/components/ui/slider' +import useAudio from '@/hooks/useAudio' +import { formatTime } from '@/utils/common' +import { cn } from '@/utils' + +type Props = { + className?: string + src: string +} + +function AudioPlayer({ src, className }: Props) { + const { playing, duration, current, toggle, onChange } = useAudio(src) + + return ( +
+ + onChange(current)} + /> +
+ {formatTime(current)}/{formatTime(duration)} +
+
+ ) +} + +export default memo(AudioPlayer) diff --git a/components/MessageItem.tsx b/components/MessageItem.tsx index 84b1156..2cb526a 100644 --- a/components/MessageItem.tsx +++ b/components/MessageItem.tsx @@ -1,17 +1,20 @@ 'use client' import { useEffect, useState, useCallback, useMemo, memo } from 'react' +import { useTranslation } from 'react-i18next' +import Lightbox from 'yet-another-react-lightbox' +import LightboxFullscreen from 'yet-another-react-lightbox/plugins/fullscreen' import MarkdownIt from 'markdown-it' import markdownHighlight from 'markdown-it-highlightjs' import highlight from 'highlight.js' import markdownKatex from '@traptitech/markdown-it-katex' import Clipboard from 'clipboard' -import { useTranslation } from 'react-i18next' -import { User, Bot, RotateCw, Sparkles, Copy, CopyCheck, PencilLine, Eraser, Volume2 } from 'lucide-react' +import { User, Bot, RotateCw, Sparkles, Copy, CopyCheck, PencilLine, Eraser, Volume2, Eye } from 'lucide-react' import { EdgeSpeech } from '@xiangfa/polly' import { Avatar, AvatarFallback } from '@/components/ui/avatar' import BubblesLoading from '@/components/BubblesLoading' import FileList from '@/components/FileList' import EditableArea from '@/components/EditableArea' +import AudioPlayer from '@/components/AudioPlayer' import IconButton from '@/components/IconButton' import { useMessageStore } from '@/store/chat' import { useSettingStore } from '@/store/setting' @@ -19,6 +22,8 @@ import AudioStream from '@/utils/AudioStream' import { sentenceSegmentation } from '@/utils/common' import { upperFirst, isFunction, find } from 'lodash-es' +import 'yet-another-react-lightbox/styles.css' + interface Props extends Message { onRegenerate?: (id: string) => void } @@ -68,6 +73,8 @@ function MessageItem({ id, role, parts, attachments, onRegenerate }: Props) { const [html, setHtml] = useState('') const [isEditing, setIsEditing] = useState(false) const [isCopyed, setIsCopyed] = useState(false) + const [showLightbox, setShowLightbox] = useState(false) + const [lightboxIndex, setLightboxIndex] = useState(0) const fileList = useMemo(() => { return attachments ? attachments.filter((item) => !item.metadata?.mimeType.startsWith('image/')) : [] }, [attachments]) @@ -160,6 +167,11 @@ function MessageItem({ id, role, parts, attachments, onRegenerate }: Props) { } }, [content]) + const openLightbox = useCallback((index: number) => { + setLightboxIndex(index) + setShowLightbox(true) + }, []) + const render = useCallback( (content: string) => { const md: MarkdownIt = MarkdownIt({ @@ -266,15 +278,22 @@ function MessageItem({ id, role, parts, attachments, onRegenerate }: Props) { {inlineAudioList.length > 0 ? (
{inlineAudioList.map((audio, idx) => { - return
) : null} {inlineImageList.length > 0 ? (
{inlineImageList.map((image, idx) => { - // eslint-disable-next-line - return inline-image + return ( +
openLightbox(idx)}> + { + // eslint-disable-next-line + inline-image + } + +
+ ) })}
) : null} @@ -319,6 +338,13 @@ function MessageItem({ id, role, parts, attachments, onRegenerate }: Props) { )} )} + setShowLightbox(false)} + slides={inlineImageList.map((item) => ({ src: item }))} + index={lightboxIndex} + plugins={[LightboxFullscreen]} + /> ) } diff --git a/components/ResponsiveDialog.tsx b/components/ResponsiveDialog.tsx index 56536c5..a985c17 100644 --- a/components/ResponsiveDialog.tsx +++ b/components/ResponsiveDialog.tsx @@ -21,7 +21,7 @@ import { DrawerTitle, DrawerTrigger, } from '@/components/ui/drawer' -import { useMediaQuery } from '@/hooks/useMediaQuery' +import useMediaQuery from '@/hooks/useMediaQuery' type Props = { open: boolean diff --git a/components/Setting.tsx b/components/Setting.tsx index d6f3df0..579fa63 100644 --- a/components/Setting.tsx +++ b/components/Setting.tsx @@ -10,6 +10,7 @@ import { Label } from '@/components/ui/label' import { Button } from '@/components/ui/button' import { Slider } from '@/components/ui/slider' import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group' +import { Switch } from '@/components/ui/switch' import ResponsiveDialog from '@/components/ResponsiveDialog' import i18n from '@/plugins/i18n' import locales from '@/constant/locales' @@ -45,6 +46,7 @@ function Setting({ open, hiddenTalkPanel, onClose }: SettingProps) { const [temperature, setTemperature] = useState(1) const [maxOutputTokens, setMaxOutputTokens] = useState(8192) const [safety, setSafety] = useState('low') + const [autoStopRecord, setAutoStopRecord] = useState(false) const isProtected = useMemo(() => { return settingStore.isProtected }, [settingStore.isProtected]) @@ -103,6 +105,7 @@ function Setting({ open, hiddenTalkPanel, onClose }: SettingProps) { if (temperature !== settingStore.temperature) settingStore.setTemperature(temperature) if (maxOutputTokens !== settingStore.maxOutputTokens) settingStore.setMaxOutputTokens(maxOutputTokens) if (safety !== settingStore.safety) settingStore.setSafety(safety) + if (autoStopRecord !== settingStore.autoStopRecord) settingStore.setAutoStopRecord(autoStopRecord) onClose() } @@ -149,6 +152,7 @@ function Setting({ open, hiddenTalkPanel, onClose }: SettingProps) { setTemperature(settingStore.temperature) setMaxOutputTokens(settingStore.maxOutputTokens) setSafety(settingStore.safety) + setAutoStopRecord(settingStore.autoStopRecord) }, [settingStore]) return ( @@ -453,6 +457,13 @@ function Setting({ open, hiddenTalkPanel, onClose }: SettingProps) { +
+ + + {autoStopRecord ? t('settingEnable') : t('settingDisable')} +
diff --git a/components/ui/switch.tsx b/components/ui/switch.tsx new file mode 100644 index 0000000..f51e2a0 --- /dev/null +++ b/components/ui/switch.tsx @@ -0,0 +1,29 @@ +"use client" + +import * as React from "react" +import * as SwitchPrimitives from "@radix-ui/react-switch" + +import { cn } from "@/utils" + +const Switch = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + +)) +Switch.displayName = SwitchPrimitives.Root.displayName + +export { Switch } diff --git a/hooks/useAudio.ts b/hooks/useAudio.ts new file mode 100644 index 0000000..5ffd251 --- /dev/null +++ b/hooks/useAudio.ts @@ -0,0 +1,53 @@ +import { useState, useEffect, useMemo } from 'react' + +function useAudio(url: string) { + const [playing, setPlaying] = useState(false) + const [duration, setDuration] = useState(0) + const [current, setCurrent] = useState(0) + const audio = useMemo(() => new Audio(url), [url]) + + const toggle = () => setPlaying(!playing) + + useEffect(() => { + playing ? audio.play() : audio.pause() + }, [audio, playing]) + + useEffect(() => { + let audioDuration = 0 + audio.addEventListener('ended', () => setPlaying(false)) + audio.addEventListener('loadeddata', () => { + if (audio.duration === Infinity) { + // HACK: Set a duration longer than the audio to get the actual duration of the audio + audio.currentTime = 1e1 + } + }) + audio.addEventListener('timeupdate', () => { + if (audioDuration === 0) { + audioDuration = audio.currentTime + setDuration(audioDuration) + setTimeout(() => { + audio.currentTime = 0 + }, 0) + } + setCurrent(audio.currentTime) + }) + return () => { + audioDuration = 0 + audio.removeEventListener('ended', () => setPlaying(false)) + audio.removeEventListener('loadeddata', () => setDuration(0)) + audio.removeEventListener('timeupdate', () => setCurrent(0)) + } + }, [audio]) + + return { + playing, + current, + duration, + toggle, + onChange: (value: number) => { + audio.currentTime = value + }, + } +} + +export default useAudio diff --git a/hooks/useMediaQuery.ts b/hooks/useMediaQuery.ts index 348c1a7..9733547 100644 --- a/hooks/useMediaQuery.ts +++ b/hooks/useMediaQuery.ts @@ -5,7 +5,7 @@ type UseMediaQueryOptions = { initializeWithValue?: boolean } -export function useMediaQuery( +function useMediaQuery( query: string, { defaultValue = false, initializeWithValue = true }: UseMediaQueryOptions = {}, ): boolean { @@ -55,3 +55,5 @@ export function useMediaQuery( return matches } + +export default useMediaQuery diff --git a/locales/ar-SA.json b/locales/ar-SA.json index feb4f15..15c42c8 100644 --- a/locales/ar-SA.json +++ b/locales/ar-SA.json @@ -60,5 +60,8 @@ "low": "منخفض", "middle": "متوسط", "high": "مرتفع", - "pwaInstall": "تثبيت تطبيق المتصفح(PWA)" + "pwaInstall": "تثبيت تطبيق المتصفح(PWA)", + "autoStopRecord": "إنهاء التسجيل تلقائيًا", + "settingEnable": "تم التفعيل", + "settingDisable": "غير مفعل" } diff --git a/locales/de-DE.json b/locales/de-DE.json index 23ee8c5..52cad50 100644 --- a/locales/de-DE.json +++ b/locales/de-DE.json @@ -60,5 +60,8 @@ "low": "Niedrig", "middle": "Mittel", "high": "Hoch", - "pwaInstall": "Browser-App (PWA) installieren" + "pwaInstall": "Browser-App (PWA) installieren", + "autoStopRecord": "Automatische Beendigung der Aufnahme", + "settingEnable": "Aktiviert", + "settingDisable": "Deaktiviert" } diff --git a/locales/en.json b/locales/en.json index 113edc9..85b09df 100644 --- a/locales/en.json +++ b/locales/en.json @@ -62,5 +62,8 @@ "low": "Low", "middle": "Middle", "high": "High", - "pwaInstall": "Install browser application (PWA)" + "pwaInstall": "Install browser application (PWA)", + "autoStopRecord": "Automatically end recording", + "settingEnable": "Enabled", + "settingDisable": "Disabled" } diff --git a/locales/es-ES.json b/locales/es-ES.json index f9e8f1b..824943d 100644 --- a/locales/es-ES.json +++ b/locales/es-ES.json @@ -60,5 +60,8 @@ "low": "Bajo", "middle": "Medio", "high": "Alto", - "pwaInstall": "Instalar aplicación de navegador (PWA)" + "pwaInstall": "Instalar aplicación de navegador (PWA)", + "autoStopRecord": "Finalizar la grabación automáticamente", + "settingEnable": "Activado", + "settingDisable": "Desactivado" } diff --git a/locales/fr-FR.json b/locales/fr-FR.json index fd18abd..5708a8b 100644 --- a/locales/fr-FR.json +++ b/locales/fr-FR.json @@ -60,5 +60,8 @@ "low": "Bas", "middle": "Milieu", "high": "Haut", - "pwaInstall": "Installer l'application de navigateur (PWA)" + "pwaInstall": "Installer l'application de navigateur (PWA)", + "autoStopRecord": "Arrêter l'enregistrement automatiquement", + "settingEnable": "Activé", + "settingDisable": "Désactivé" } diff --git a/locales/ja-JP.json b/locales/ja-JP.json index d41935f..0b4ab42 100644 --- a/locales/ja-JP.json +++ b/locales/ja-JP.json @@ -60,5 +60,8 @@ "low": "低い", "middle": "中い", "high": "高い", - "pwaInstall": "ブラウザアプリ（PWA）をインストールする" + "pwaInstall": "ブラウザアプリ（PWA）をインストールする", + "autoStopRecord": "録音を自動的に終了する", + "settingEnable": "有効", + "settingDisable": "無効" } diff --git a/locales/ko-KR.json b/locales/ko-KR.json index 2679a6b..1c625f2 100644 --- a/locales/ko-KR.json +++ b/locales/ko-KR.json @@ -60,5 +60,8 @@ "low": "낮은", "middle": "중간", "high": "높은", - "pwaInstall": "브라우저 애플리케이션(PWA) 설치하기" + "pwaInstall": "브라우저 애플리케이션(PWA) 설치하기", + "autoStopRecord": "녹음 자동 종료", + "settingEnable": "활성화되었습니다", + "settingDisable": "비활성화됨" } diff --git a/locales/pt-BR.json b/locales/pt-BR.json index 0779170..5397b72 100644 --- a/locales/pt-BR.json +++ b/locales/pt-BR.json @@ -60,5 +60,8 @@ "low": "Baixo", "middle": "Médio", "high": "Alto", - "pwaInstall": "Instalar aplicativo de navegador (PWA)" + "pwaInstall": "Instalar aplicativo de navegador (PWA)", + "autoStopRecord": "Encerrar a gravação automaticamente", + "settingEnable": "Ativado", + "settingDisable": "Desativado" } diff --git a/locales/ru-RU.json b/locales/ru-RU.json index fc66263..445ebec 100644 --- a/locales/ru-RU.json +++ b/locales/ru-RU.json @@ -60,5 +60,8 @@ "low": "Низкий", "middle": "Средний", "high": "Высокий", - "pwaInstall": "Установить приложение браузера (PWA)" + "pwaInstall": "Установить приложение браузера (PWA)", + "autoStopRecord": "Автоматическое завершение записи", + "settingEnable": "Включено", + "settingDisable": "Отключено" } diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 00c8ab3..12c3845 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -60,5 +60,8 @@ "low": "低", "middle": "中", "high": "高", - "pwaInstall": "安裝瀏覽器應用(PWA)" + "pwaInstall": "安裝瀏覽器應用(PWA)", + "autoStopRecord": "自動結束錄音", + "settingEnable": "已啟用", + "settingDisable": "未啟用" } diff --git a/locales/zh.json b/locales/zh.json index e954891..53e59e8 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -62,5 +62,8 @@ "low": "低", "middle": "中", "high": "高", - "pwaInstall": "安装浏览器应用(PWA)" + "pwaInstall": "安装浏览器应用(PWA)", + "autoStopRecord": "自动结束录音", + "settingEnable": "已启用", + "settingDisable": "未启用" } diff --git a/package.json b/package.json index 7d0649c..332658b 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@radix-ui/react-separator": "^1.0.3", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-slot": "^1.0.2", + "@radix-ui/react-switch": "^1.0.3", "@radix-ui/react-tabs": "^1.0.4", "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-tooltip": "^1.0.7", @@ -43,7 +44,7 @@ "katex": "^0.16.10", "localforage": "^1.10.0", "lodash-es": "^4.17.21", - "lucide-react": "^0.303.0", + "lucide-react": "^0.390.0", "markdown-it": "^14.1.0", "markdown-it-highlightjs": "^4.1.0", "nanoid": "^5.0.7", @@ -57,6 +58,7 @@ "tailwind-merge": "^2.3.0", "tailwindcss-animate": "^1.0.7", "vaul": "^0.9.1", + "yet-another-react-lightbox": "^3.19.0", "zustand": "^4.5.2" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 98d05c9..199ef35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: '@radix-ui/react-slot': specifier: ^1.0.2 version: 1.0.2(@types/react@18.3.2)(react@18.3.1) + '@radix-ui/react-switch': + specifier: ^1.0.3 + version: 1.0.3(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@radix-ui/react-tabs': specifier: ^1.0.4 version: 1.0.4(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -99,8 +102,8 @@ importers: specifier: ^4.17.21 version: 4.17.21 lucide-react: - specifier: ^0.303.0 - version: 0.303.0(react@18.3.1) + specifier: ^0.390.0 + version: 0.390.0(react@18.3.1) markdown-it: specifier: ^14.1.0 version: 14.1.0 @@ -140,6 +143,9 @@ importers: vaul: specifier: ^0.9.1 version: 0.9.1(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + yet-another-react-lightbox: + specifier: ^3.19.0 + version: 3.19.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) zustand: specifier: ^4.5.2 version: 4.5.2(@types/react@18.3.2)(react@18.3.1) @@ -206,16 +212,16 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} - '@babel/helper-string-parser@7.24.6': - resolution: {integrity: sha512-WdJjwMEkmBicq5T9fm/cHND3+UlFa2Yj8ALLgmoSQAJZysYbBjw+azChSGPN4DSPLXOcooGRvDwZWMcF/mLO2Q==} + '@babel/helper-string-parser@7.24.7': + resolution: {integrity: sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==} engines: {node: '>=6.9.0'} - '@babel/helper-validator-identifier@7.24.6': - resolution: {integrity: sha512-4yA7s865JHaqUdRbnaxarZREuPTHrjpDT+pXoAZ1yhyo6uFnIEpS8VMu16siFOHDpZNKYv5BObhsB//ycbICyw==} + '@babel/helper-validator-identifier@7.24.7': + resolution: {integrity: sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==} engines: {node: '>=6.9.0'} - '@babel/parser@7.24.6': - resolution: {integrity: sha512-eNZXdfU35nJC2h24RznROuOpO94h6x8sg9ju0tT9biNtLZ2vuP8SduLqqV+/8+cebSLV9SJEAN5Z3zQbJG/M+Q==} + '@babel/parser@7.24.7': + resolution: {integrity: sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==} engines: {node: '>=6.0.0'} hasBin: true @@ -223,8 +229,8 @@ packages: resolution: {integrity: sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==} engines: {node: '>=6.9.0'} - '@babel/types@7.24.6': - resolution: {integrity: sha512-WaMsgi6Q8zMgMth93GvWPXkhAIEobfsIkLTacoVZoK1J0CevIPGYY2Vo5YvJGqyHqXM6P4ppOYGsIRU8MM9pFQ==} + '@babel/types@7.24.7': + resolution: {integrity: sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==} engines: {node: '>=6.9.0'} '@eslint-community/eslint-utils@4.4.0': @@ -657,6 +663,19 @@ packages: '@types/react': optional: true + '@radix-ui/react-switch@1.0.3': + resolution: {integrity: sha512-mxm87F88HyHztsI7N+ZUmEoARGkC22YVW5CaC+Byc+HRpuvCrOBPTAnXgf+tZ/7i0Sg/eOePGdMhUKhPaQEqow==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + '@radix-ui/react-tabs@1.0.4': resolution: {integrity: sha512-egZfYY/+wRNCflXNHx+dePvnz9FbmssDTJBtgRfDY7e8SE5oIo3Py2eCB1ckAbh1Q7cQ/6yJZThJ++sgbxibog==} peerDependencies: @@ -1938,8 +1957,8 @@ packages: resolution: {integrity: sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==} engines: {node: 14 || >=16.14} - lucide-react@0.303.0: - resolution: {integrity: sha512-B0B9T3dLEFBYPCUlnUS1mvAhW1craSbF9HO+JfBjAtpFUJ7gMIqmEwNSclikY3RiN2OnCkj/V1ReAQpaHae8Bg==} + lucide-react@0.390.0: + resolution: {integrity: sha512-APqbfEcVuHnZbiy3E97gYWLeBdkE4e6NbY6AuVETZDZVn/bQCHYUoHyxcUHyvRopfPOHhFUEvDyyQzHwM+S9/w==} peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 @@ -2795,6 +2814,13 @@ packages: engines: {node: '>= 14'} hasBin: true + yet-another-react-lightbox@3.19.0: + resolution: {integrity: sha512-HbvKVKfl17J51MdE2wMiZqwIDlse3CJYffQhl4RUOG1BZLcQ74Jf/CxGEa/EIECwCaabRiHzZbX88+z4Flvl0A==} + engines: {node: '>=14'} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -2826,22 +2852,22 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 - '@babel/helper-string-parser@7.24.6': {} + '@babel/helper-string-parser@7.24.7': {} - '@babel/helper-validator-identifier@7.24.6': {} + '@babel/helper-validator-identifier@7.24.7': {} - '@babel/parser@7.24.6': + '@babel/parser@7.24.7': dependencies: - '@babel/types': 7.24.6 + '@babel/types': 7.24.7 '@babel/runtime@7.24.5': dependencies: regenerator-runtime: 0.14.1 - '@babel/types@7.24.6': + '@babel/types@7.24.7': dependencies: - '@babel/helper-string-parser': 7.24.6 - '@babel/helper-validator-identifier': 7.24.6 + '@babel/helper-string-parser': 7.24.7 + '@babel/helper-validator-identifier': 7.24.7 to-fast-properties: 2.0.0 '@eslint-community/eslint-utils@4.4.0(eslint@8.57.0)': @@ -3286,6 +3312,22 @@ snapshots: optionalDependencies: '@types/react': 18.3.2 + '@radix-ui/react-switch@1.0.3(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@babel/runtime': 7.24.5 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.3.2)(react@18.3.1) + '@radix-ui/react-context': 1.0.1(@types/react@18.3.2)(react@18.3.1) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.3.2)(react@18.3.1) + '@radix-ui/react-use-previous': 1.0.1(@types/react@18.3.2)(react@18.3.1) + '@radix-ui/react-use-size': 1.0.1(@types/react@18.3.2)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.2 + '@types/react-dom': 18.3.0 + '@radix-ui/react-tabs@1.0.4(@types/react-dom@18.3.0)(@types/react@18.3.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: '@babel/runtime': 7.24.5 @@ -3600,7 +3642,7 @@ snapshots: '@vue/compiler-core@3.4.27': dependencies: - '@babel/parser': 7.24.6 + '@babel/parser': 7.24.7 '@vue/shared': 3.4.27 entities: 4.5.0 estree-walker: 2.0.2 @@ -3613,7 +3655,7 @@ snapshots: '@vue/compiler-sfc@3.4.27': dependencies: - '@babel/parser': 7.24.6 + '@babel/parser': 7.24.7 '@vue/compiler-core': 3.4.27 '@vue/compiler-dom': 3.4.27 '@vue/compiler-ssr': 3.4.27 @@ -4762,7 +4804,7 @@ snapshots: lru-cache@10.2.2: {} - lucide-react@0.303.0(react@18.3.1): + lucide-react@0.390.0(react@18.3.1): dependencies: react: 18.3.1 @@ -5633,6 +5675,11 @@ snapshots: yaml@2.4.2: {} + yet-another-react-lightbox@3.19.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + yocto-queue@0.1.0: {} zod@3.22.4: {} diff --git a/store/setting.ts b/store/setting.ts index be3ca4c..f040b6e 100644 --- a/store/setting.ts +++ b/store/setting.ts @@ -23,6 +23,7 @@ interface SettingStore extends Setting { setTemperature: (value: number) => void setMaxOutputTokens: (value: number) => void setSafety: (level: string) => void + setAutoStopRecord: (active: boolean) => void } const ASSISTANT_INDEX_URL = process.env.NEXT_PUBLIC_ASSISTANT_INDEX_URL @@ -56,6 +57,7 @@ export const useSettingStore = create((set) => ({ temperature: 1, maxOutputTokens: 8192, safety: 'low', + autoStopRecord: false, init: async (isProtected) => { await dataMigration() const sttLang = await storage.getItem('sttLang') @@ -86,6 +88,7 @@ export const useSettingStore = create((set) => ({ temperature: (await storage.getItem('temperature')) ?? defaultModelConfig.temperature, maxOutputTokens: (await storage.getItem('maxOutputTokens')) ?? defaultModelConfig.maxOutputTokens, safety: (await storage.getItem('safety')) || 'low', + autoStopRecord: (await storage.getItem('autoStopRecord')) || false, } set(() => state) return state @@ -163,4 +166,8 @@ export const useSettingStore = create((set) => ({ set(() => ({ safety: level })) storage.setItem('safety', level) }, + setAutoStopRecord: (active) => { + set(() => ({ autoStopRecord: active })) + storage.setItem('autoStopRecord', active) + }, })) diff --git a/types.d.ts b/types.d.ts index e4c9609..c9bb81a 100644 --- a/types.d.ts +++ b/types.d.ts @@ -25,6 +25,7 @@ declare global { temperature: number maxOutputTokens: number safety: string + autoStopRecord: boolean } interface Assistant { diff --git a/utils/Recorder.ts b/utils/Recorder.ts index 43ce610..aa4d8be 100644 --- a/utils/Recorder.ts +++ b/utils/Recorder.ts @@ -1,7 +1,7 @@ import { isFunction } from 'lodash-es' export interface AudioRecorderPayload { - autoRecord?: boolean + autoStop?: boolean volumeThreshold?: number silenceThreshold?: number onStart?: () => void @@ -19,11 +19,11 @@ export class AudioRecorder { public blob: Blob | null = null public time: number = 0 public isRecording: boolean = false + public autoStop: boolean = false protected audioContext: AudioContext protected mediaRecorder: MediaRecorder | null = null protected volumeThreshold: number = 30 protected silenceThreshold: number = 2000 - protected autoRecord: boolean = false protected onStart() {} protected onTimeUpdate(time: number) {} protected onFinish(audioData: Blob) {} @@ -57,7 +57,7 @@ export class AudioRecorder { return `${minutesStr}:${secondsStr}` } constructor({ - autoRecord, + autoStop, volumeThreshold, silenceThreshold, onStart, @@ -66,7 +66,7 @@ export class AudioRecorder { onError, }: AudioRecorderPayload) { this.audioContext = new AudioContext() - if (autoRecord) this.autoRecord = autoRecord + if (autoStop) this.autoStop = autoStop // 设置音量阈值 if (volumeThreshold) this.volumeThreshold = volumeThreshold // 设置静音持续时间阈值（单位：毫秒） @@ -78,12 +78,7 @@ export class AudioRecorder { } public start() { if (this.mediaRecorder) { - if (this.mediaRecorder.state === 'paused') { - this.mediaRecorder.resume() - } else { - this.mediaRecorder.start(1000) - } - this.onStart() + this.mediaRecorder.start(1000) } else { // 获取麦克风音频流 navigator.mediaDevices @@ -95,9 +90,6 @@ export class AudioRecorder { this.onError(error) }) } - if (!this.autoRecord) { - this.isRecording = true - } } protected recording(stream: MediaStream) { let chunks: Blob[] = [] @@ -127,6 +119,7 @@ export class AudioRecorder { mediaRecorder.addEventListener('start', () => { this.isRecording = true this.startTimer() + this.onStart() }) mediaRecorder.addEventListener('pause', () => { const blob = new Blob(chunks) @@ -164,14 +157,6 @@ export class AudioRecorder { const volume = getVolumeFromFrequencyData(frequencyData, bufferLength) if (volume > this.volumeThreshold) { - // 声音超过阈值，判断为发言开始 - if (mediaRecorder.state === 'paused') { - mediaRecorder.resume() - } else { - mediaRecorder.start(1000) - } - this.onStart() - // 重置静音计时器 clearTimeout(silenceTimer) silenceTimer = null @@ -180,7 +165,7 @@ export class AudioRecorder { if (!silenceTimer) { silenceTimer = setTimeout(() => { if (mediaRecorder.state === 'recording') { - mediaRecorder.pause() + mediaRecorder.stop() } cancelAnimationFrame(rafID) this.isRecording = false @@ -196,12 +181,10 @@ export class AudioRecorder { } // 开始处理音频流 - if (this.autoRecord) { + if (this.autoStop) { processAudio() - } else { - mediaRecorder.start(1000) - this.onStart() } + mediaRecorder.start(1000) } public stop() { this.mediaRecorder?.stop()