From 258e49253d149fd1ffeb5b72fd7f6cc4774bbbec Mon Sep 17 00:00:00 2001 From: miro Date: Sat, 14 Dec 2024 14:50:18 +0000 Subject: [PATCH] pt citrinet fallback + dutch image --- .github/workflows/build_img_nl.yml | 135 +++++++++++++++++++++++++++++ build_raspOVOS_nl.sh | 68 +++++++++++++++ build_raspOVOS_pt.sh | 7 ++ mycroft_nl.conf | 24 +++++ mycroft_pt.conf | 3 +- packages/download_citrinet_nl.py | 9 ++ packages/download_citrinet_pt.py | 9 ++ 7 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_img_nl.yml create mode 100644 build_raspOVOS_nl.sh create mode 100644 mycroft_nl.conf create mode 100644 packages/download_citrinet_nl.py create mode 100644 packages/download_citrinet_pt.py diff --git a/.github/workflows/build_img_nl.yml b/.github/workflows/build_img_nl.yml new file mode 100644 index 00000000..e17b4785 --- /dev/null +++ b/.github/workflows/build_img_nl.yml @@ -0,0 +1,135 @@ +name: build dutch headless image + +on: + workflow_dispatch: + push: + branches: + - master + paths: + - '.github/workflows/build_img_nl.yml' + - 'build_raspOVOS_nl.sh' + - 'intent_cache/nl/**' + +jobs: + # Stage 1: Get Current Date and Release Name + get-date-release-name: + runs-on: ubuntu-latest + outputs: + release_name: ${{ steps.current-date.outputs.release_name }} + cache_key: ${{ steps.current-date.outputs.cache_key }} + steps: + - name: Get Current Date and Release Name + id: current-date + run: | + current_date=$(date +'%Y-%m-%d') + release_name="raspOVOS-dutch-bookworm-arm64-lite-${current_date}" + cache_key="raspOVOS-${{ github.run_id }}" + echo "release_name=$release_name" >> $GITHUB_OUTPUT + echo "cache_key=$cache_key" >> $GITHUB_OUTPUT + + - name: Print outputs + shell: bash + run: | + echo "cache_key: ${{ steps.current-date.outputs.cache_key }}" + echo "release_name: ${{ steps.current-date.outputs.release_name }}" + + # Stage 2: Build Image + modify-rpi-image: + runs-on: ubuntu-latest + needs: get-date-release-name + outputs: + image-path: ${{ steps.create-image.outputs.image-path }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Configuring OVOS for Dutch + uses: TigreGotico/rpi-image-modifier@main + id: create-image + env: + USER: 'ovos' + PASSWORD: 'ovos' + HOSTNAME: "raspOVOS" + CONSTRAINTS: "https://github.com/OpenVoiceOS/ovos-releases/raw/refs/heads/main/constraints-alpha.txt" + MYCROFT_CONFIG_FILES: "mycroft.conf,mycroft_nl.conf" + with: + base-image-url: https://github.com/TigreGotico/raspOVOS/releases/download/raspOVOS-bookworm-arm64-lite-2024-12-11/raspOVOS-bookworm-arm64-lite.img.xz + image-path: raspOVOS-dutch-bookworm-arm64-lite.img + compress-with-xz: true + shrink: true + cache: false + mount-repository: true + env-vars: USER,PASSWORD,HOSTNAME,CONSTRAINTS,MYCROFT_CONFIG_FILES + script-path: build_raspOVOS_nl.sh + - name: Print outputs + shell: bash + run: | + echo "image-path: ${{ steps.create-image.outputs.image-path }}" + echo "image-size: ${{ steps.create-image.outputs.image-size }}" + echo "image-sha256sum: ${{ steps.create-image.outputs.image-sha256sum }}" + + - name: Save Image to Cache + uses: actions/cache@v4 + with: + path: . + key: ${{ needs.get-date-release-name.outputs.cache_key }} + + # Stage 3: Check if Release Exists and Create if Needed + check-release-or-create: + runs-on: ubuntu-latest + needs: [get-date-release-name, modify-rpi-image] + outputs: + release_id: ${{ steps.get-release-id.outputs.release_id }} + steps: + - name: Check if Release Exists + id: get-release-id + run: | + release_name="${{ needs.get-date-release-name.outputs.release_name }}" + release=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/releases/tags/$release_name") + release_id=$(echo $release | jq -r .id) + if [[ "$release_id" == "null" ]]; then + release=$(curl -s -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d "{\"tag_name\":\"$release_name\",\"name\":\"$release_name\",\"body\":\"RaspOVOS pre-configured for Dutch\"}" \ + "https://api.github.com/repos/${{ github.repository }}/releases") + release_id=$(echo $release | jq -r .id) + fi + echo "release_id=$release_id" >> $GITHUB_OUTPUT + + - name: Print outputs + shell: bash + run: | + echo "release_id: ${{ steps.get-release-id.outputs.release_id }}" + if [[ $release_id == "null" ]]; then + echo "Failed to create release. Response: $release" + exit 1 + fi + + # Stage 4: Upload to Release + upload-image: + runs-on: ubuntu-latest + if: ${{ needs.check-release-or-create.outputs.release_id }} + needs: [modify-rpi-image, check-release-or-create, get-date-release-name] + steps: + - name: Restore Cache for Image + uses: actions/cache@v4 + with: + path: . + key: ${{ needs.get-date-release-name.outputs.cache_key }} + fail-on-cache-miss: true + + - name: Debug Cache Content + run: ls . + + - name: Upload to release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + file: ${{ needs.modify-rpi-image.outputs.image-path }} + release_id: ${{ needs.check-release-or-create.outputs.release_id }} + draft: false + overwrite: true + prerelease: false + verbose: true diff --git a/build_raspOVOS_nl.sh b/build_raspOVOS_nl.sh new file mode 100644 index 00000000..f1efc9fc --- /dev/null +++ b/build_raspOVOS_nl.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Exit on error +# If something goes wrong just stop. +# it allows the user to see issues at once rather than having +# scroll back and figure out what went wrong. +set -e + +# Activate the virtual environment +source /home/$USER/.venvs/ovos/bin/activate + + +echo "Setting up default wifi country..." +/usr/bin/raspi-config nonint do_wifi_country NL + +echo "Caching pre-trained padatious intents..." +mkdir -p /home/$USER/.local/share/mycroft/intent_cache +cp -rv /mounted-github-repo/intent_cache/nl-NL /home/$USER/.local/share/mycroft/intent_cache/ + +echo "Downloading dutch citrinet model..." +python /mounted-github-repo/packages/download_citrinet_nl.py +# since script was run as root, we need to move downloaded files +mkdir -p /home/ovos/.cache/huggingface/hub/ +mv /root/.cache/huggingface/hub/models--neongeckocom--stt_nl_citrinet_512_gamma_0_25/ /home/ovos/.cache/huggingface/hub/models--neongeckocom--stt_nl_citrinet_512_gamma_0_25/ + +echo "Downloading dutch vosk model..." +# Download and extract VOSK model +VOSK_DIR="/home/$USER/.local/share/vosk" +mkdir -p $VOSK_DIR +wget https://alphacephei.com/vosk/models/vosk-model-small-nl-0.22.zip -P $VOSK_DIR +unzip -o $VOSK_DIR/vosk-model-small-nl-0.22.zip -d $VOSK_DIR +rm $VOSK_DIR/vosk-model-small-nl-0.22.zip + + +echo "Installing Piper TTS..." +uv pip install --no-progress ovos-tts-plugin-piper -c $CONSTRAINTS + +# download default piper voice for dutch +PIPER_DIR="/home/$USER/.local/share/piper_tts/mls_5809-low" +VOICE_URL="https://github.com/rhasspy/piper/releases/download/v0.0.2/voice-nl-mls_5809-low.tar.gz" +VOICE_ARCHIVE="$PIPER_DIR/voice-nl-mls_5809-low.tar.gz" +mkdir -p "$PIPER_DIR" +echo "Downloading voice from $VOICE_URL..." +wget "$VOICE_URL" -O "$VOICE_ARCHIVE" +tar -xvzf "$VOICE_ARCHIVE" -C "$PIPER_DIR" +# if we remove the voice archive the plugin will think its missing and redownload voice on boot... +rm "$VOICE_ARCHIVE" +touch $VOICE_ARCHIVE + + +echo "Creating system level mycroft.conf..." +mkdir -p /etc/mycroft + +CONFIG_ARGS="" +# Loop through the MYCROFT_CONFIG_FILES variable and append each file to the jq command +IFS=',' read -r -a config_files <<< "$MYCROFT_CONFIG_FILES" +for file in "${config_files[@]}"; do + CONFIG_ARGS="$CONFIG_ARGS /mounted-github-repo/$file" +done +# Execute the jq command and merge the files into mycroft.conf +jq -s 'reduce .[] as $item ({}; . * $item)' $CONFIG_ARGS > /etc/mycroft/mycroft.conf + + +echo "Ensuring permissions for $USER user..." +# Replace 1000:1000 with the correct UID:GID if needed +chown -R 1000:1000 /home/$USER + +echo "Cleaning up apt packages..." +apt-get --purge autoremove -y && apt-get clean \ No newline at end of file diff --git a/build_raspOVOS_pt.sh b/build_raspOVOS_pt.sh index 20c47e2d..007ef8e7 100644 --- a/build_raspOVOS_pt.sh +++ b/build_raspOVOS_pt.sh @@ -16,6 +16,13 @@ echo "Caching pre-trained padatious intents..." mkdir -p /home/$USER/.local/share/mycroft/intent_cache cp -rv /mounted-github-repo/intent_cache/pt-PT /home/$USER/.local/share/mycroft/intent_cache/ +echo "Downloading portuguese citrinet model..." +python /mounted-github-repo/packages/download_citrinet_pt.py +# since script was run as root, we need to move downloaded files +mkdir -p /home/ovos/.cache/huggingface/hub/ +mv /root/.cache/huggingface/hub/models--neongeckocom--stt_pt_citrinet_512_gamma_0_25/ /home/ovos/.cache/huggingface/hub/models--neongeckocom--stt_pt_citrinet_512_gamma_0_25/ + + echo "Installing Piper TTS..." uv pip install --no-progress ovos-tts-plugin-piper -c $CONSTRAINTS diff --git a/mycroft_nl.conf b/mycroft_nl.conf new file mode 100644 index 00000000..babe5b7f --- /dev/null +++ b/mycroft_nl.conf @@ -0,0 +1,24 @@ +{ + "lang": "nl-NL", + "system_unit": "metric", + "temperature_unit": "celsius", + "windspeed_unit": "km/h", + "precipitation_unit": "mm", + "time_format": "full", + "spoken_time_format": "half", + "date_format": "DMY", + "stt": { + "module": "ovos-stt-plugin-citrinet", + "fallback_module": "", + "ovos-stt-plugin-citrinet": { + "lang": "nl" + } + }, + "tts": { + "module": "ovos-tts-plugin-piper", + "ovos-tts-plugin-piper": { + "voice": "mls_5809-low" + } + } + +} \ No newline at end of file diff --git a/mycroft_pt.conf b/mycroft_pt.conf index cc8443f4..402c2e5f 100644 --- a/mycroft_pt.conf +++ b/mycroft_pt.conf @@ -26,7 +26,8 @@ }, "stt": { "module": "ovos-stt-plugin-server", - "fallback_module": "", + "fallback_module": "ovos-stt-plugin-citrinet", + "ovos-stt-plugin-citrinet": {"lang": "pt"}, "ovos-stt-plugin-server": { "urls": [ "https://stt.smartgic.io/mynorthai/stt", diff --git a/packages/download_citrinet_nl.py b/packages/download_citrinet_nl.py new file mode 100644 index 00000000..a6a2e187 --- /dev/null +++ b/packages/download_citrinet_nl.py @@ -0,0 +1,9 @@ +from huggingface_hub import hf_hub_download + +repo_id = "neongeckocom/stt_nl_citrinet_512_gamma_0_25" +subfolder = "onnx" + +files = ["model.onnx", "tokenizer.spm", "preprocessor.ts"] +for file in files: + file_path = hf_hub_download(repo_id=repo_id, filename=file, subfolder=subfolder) + print(f"Downloaded {file} to {file_path}") diff --git a/packages/download_citrinet_pt.py b/packages/download_citrinet_pt.py new file mode 100644 index 00000000..b02972ea --- /dev/null +++ b/packages/download_citrinet_pt.py @@ -0,0 +1,9 @@ +from huggingface_hub import hf_hub_download + +repo_id = "neongeckocom/stt_pt_citrinet_512_gamma_0_25" +subfolder = "onnx" + +files = ["model.onnx", "tokenizer.spm", "preprocessor.ts"] +for file in files: + file_path = hf_hub_download(repo_id=repo_id, filename=file, subfolder=subfolder) + print(f"Downloaded {file} to {file_path}")