diff --git a/PlatformIO/src/General.hpp b/PlatformIO/src/General.hpp index a28f94a..8b0b99a 100644 --- a/PlatformIO/src/General.hpp +++ b/PlatformIO/src/General.hpp @@ -42,6 +42,7 @@ struct Config { int hotword_brightness = 15; uint16_t volume = 100; int gain = 5; + int animation = SOLID; }; const char *configfile = "/config.json"; Config config; @@ -68,6 +69,7 @@ int retryCount = 0; int I2SMode = -1; bool mqttConnected = false; bool DEBUG = false; +bool configChanged = false; std::string audioFrameTopic = std::string("hermes/audioServer/") + config.siteid + std::string("/audioFrame"); std::string playBytesTopic = std::string("hermes/audioServer/") + config.siteid + std::string("/playBytes/#"); @@ -79,6 +81,7 @@ std::string debugTopic = config.siteid + std::string("/debug"); std::string restartTopic = config.siteid + std::string("/restart"); std::string sayTopic = "hermes/tts/say"; std::string sayFinishedTopic = "hermes/tts/sayFinished"; +std::string errorTopic = "hermes/nlu/intentNotRecognized"; AsyncMqttClient asyncClient; WiFiClient net; PubSubClient audioServer(net); @@ -88,28 +91,39 @@ int queueDelay = 10; int sampleRate = 16000; int numChannels = 2; int bitDepth = 16; +int current_colors = COLORS_IDLE; static EventGroupHandle_t audioGroup; SemaphoreHandle_t wbSemaphore; TaskHandle_t i2sHandle; +struct WifiConnected; struct WifiDisconnected; +struct MQTTConnected; struct MQTTDisconnected; -struct HotwordDetected; +struct Listening; +struct ListeningPlay; struct Idle; -struct Speaking; +struct IdlePlay; +struct Tts; +struct TtsPlay; +struct Error; +struct ErrorPlay; struct Updating; -struct PlayAudio; struct WifiDisconnectEvent : tinyfsm::Event { }; struct WifiConnectEvent : tinyfsm::Event { }; struct MQTTDisconnectedEvent : tinyfsm::Event { }; struct MQTTConnectedEvent : tinyfsm::Event { }; struct IdleEvent : tinyfsm::Event { }; -struct SpeakEvent : tinyfsm::Event { }; -struct OtaEvent : tinyfsm::Event { }; +struct TtsEvent : tinyfsm::Event { }; +struct ErrorEvent : tinyfsm::Event { }; +struct UpdateEvent : tinyfsm::Event { }; +struct BeginPlayAudioEvent : tinyfsm::Event {}; +struct EndPlayAudioEvent : tinyfsm::Event {}; struct StreamAudioEvent : tinyfsm::Event { }; -struct PlayAudioEvent : tinyfsm::Event {}; -struct HotwordDetectedEvent : tinyfsm::Event { }; +struct PlayBytesEvent : tinyfsm::Event {}; +struct ListeningEvent : tinyfsm::Event { }; +struct UpdateConfigurationEvent : tinyfsm::Event { }; void onMqttConnect(bool sessionPresent); void onMqttDisconnect(AsyncMqttClientDisconnectReason reason); @@ -181,6 +195,11 @@ const std::map processor_values = { {"VOLUME", []() { return String(config.volume); } }, {"GAIN", []() { return String(config.gain); } }, {"SITEID", []() -> String { return config.siteid.c_str(); } }, + {"ANIMATIONSUPPORT", []() -> String { return device->animationSupported() ? "block" : "none"; } }, + {"ANIM_SOLID", []() -> String { return (config.animation == SOLID) ? "selected" : ""; } }, + {"ANIM_RUNNING", []() -> String { return device->runningSupported() ? (config.animation == RUN) ? "selected" : "" : "hidden"; } }, + {"ANIM_PULSING", []() -> String { return device->pulsingSupported() ? (config.animation == PULSE) ? "selected" : "" : "hidden"; } }, + {"ANIM_BLINKING", []() -> String { return device->blinkingSupported() ? (config.animation == BLINK) ? "selected" : "" : "hidden"; } }, }; // this function supplies template variables to the template engine @@ -218,6 +237,7 @@ template bool processParam(AsyncWebParameter *p, const char* p_name void handleFSf ( AsyncWebServerRequest* request, const String& route ) { AsyncWebServerResponse *response ; bool saveNeeded = false; + bool rebootNeeded = false; if ( route.indexOf ( "index.html" ) >= 0 ) // Index page is in PROGMEM { @@ -229,17 +249,18 @@ void handleFSf ( AsyncWebServerRequest* request, const String& route ) { AsyncWebParameter* p = request->getParam(i); Serial.printf("Parameter %s, value %s\r\n", p->name().c_str(), p->value().c_str()); - saveNeeded |= processParam(p, "siteid", config.siteid); - saveNeeded |= processParam(p, "mqtt_host", config.mqtt_host); - saveNeeded |= processParam(p, "mqtt_pass", config.mqtt_pass); - saveNeeded |= processParam(p, "mqtt_user", config.mqtt_user); - saveNeeded |= processParam(p, "mqtt_port", config.mqtt_port); + rebootNeeded |= processParam(p, "siteid", config.siteid); + rebootNeeded |= processParam(p, "mqtt_host", config.mqtt_host); + rebootNeeded |= processParam(p, "mqtt_pass", config.mqtt_pass); + rebootNeeded |= processParam(p, "mqtt_user", config.mqtt_user); + rebootNeeded |= processParam(p, "mqtt_port", config.mqtt_port); saveNeeded |= processParam(p, "mute_input", config.mute_input); saveNeeded |= processParam(p, "mute_output", config.mute_output); saveNeeded |= processParam(p, "amp_output", config.amp_output); saveNeeded |= processParam(p, "brightness", config.brightness); saveNeeded |= processParam(p, "hw_brightness", config.hotword_brightness); saveNeeded |= processParam(p, "hotword_detection", config.hotword_detection); + saveNeeded |= processParam(p, "animation", config.animation); saveNeeded |= processParam(p, "gain", config.gain); saveNeeded |= processParam(p, "volume", config.volume); @@ -258,14 +279,15 @@ void handleFSf ( AsyncWebServerRequest* request, const String& route ) { config.mute_output = false; saveNeeded = true; } - if (saveNeeded) { + if (saveNeeded || rebootNeeded) { Serial.println("Settings changed, saving configuration"); saveConfiguration(configfile, config); + configChanged = true; } else { Serial.println("No settings changed"); } } - if (saveNeeded) { + if (rebootNeeded) { response = request->beginResponse_P ( 200, "text/html", "Rebooting...

Configuration saved, rebooting!

"); } else { response = request->beginResponse_P ( 200, "text/html", index_html, processor ); @@ -277,7 +299,7 @@ void handleFSf ( AsyncWebServerRequest* request, const String& route ) { request->send ( response ) ; - if (saveNeeded) { + if (rebootNeeded) { Serial.println("Rebooting!"); ESP.restart(); } @@ -289,7 +311,25 @@ void handleRequest ( AsyncWebServerRequest* request ) handleFSf ( request, String( "/index.html") ) ; } +void initHeader(int readSize, int width, int rate) { + strncpy(header.riff_tag, "RIFF", 4); + strncpy(header.wave_tag, "WAVE", 4); + strncpy(header.fmt_tag, "fmt ", 4); + strncpy(header.data_tag, "data", 4); + + header.riff_length = (uint32_t)sizeof(header) + (readSize * width); + header.fmt_length = 16; + header.audio_format = 1; + header.num_channels = 1; + header.sample_rate = rate; + header.byte_rate = rate * width; + header.block_align = width; + header.bits_per_sample = width * 8; + header.data_length = readSize * width; +} + void publishDebug(const char* message) { + Serial.println(message); if (DEBUG) { asyncClient.publish(debugTopic.c_str(), 0, false, message); } @@ -323,6 +363,7 @@ void loadConfiguration(const char *filename, Config &config) { config.volume = doc.getMember("volume").as(); device->setVolume(config.volume); config.gain = doc.getMember("gain").as(); + config.animation = doc.getMember("animation").as(); device->setGain(config.gain); audioFrameTopic = std::string("hermes/audioServer/") + config.siteid + std::string("/audioFrame"); playBytesTopic = std::string("hermes/audioServer/") + config.siteid + std::string("/playBytes/#"); @@ -345,7 +386,7 @@ void saveConfiguration(const char *filename, Config &config) { Serial.println(F("Failed to create file")); return; } - StaticJsonDocument<256> doc; + StaticJsonDocument<512> doc; doc["siteid"] = config.siteid; doc["mqtt_host"] = config.mqtt_host; doc["mqtt_port"] = config.mqtt_port; @@ -359,6 +400,7 @@ void saveConfiguration(const char *filename, Config &config) { doc["hotword_detection"] = config.hotword_detection; doc["volume"] = config.volume; doc["gain"] = config.gain; + doc["animation"] = config.animation; if (serializeJson(doc, file) == 0) { Serial.println(F("Failed to write to file")); } diff --git a/PlatformIO/src/Satellite.cpp b/PlatformIO/src/Satellite.cpp index a38cec4..ba08406 100644 --- a/PlatformIO/src/Satellite.cpp +++ b/PlatformIO/src/Satellite.cpp @@ -1,13 +1,12 @@ /* ************************************************************************* * - Matrix Voice Audio Streamer + ESP32 Rhasspy Satellite - This program is written to be a streaming audio server running on the Matrix - Voice. This is typically used for Rhasspy. + This program is written to be a streaming audio microphone for Rhasspy. See https://rhasspy.readthedocs.io/en/latest/ for more information Author: Paul Romkes - Date: Januari 2021 - Version: 7.0 + Date: October 2021 + Version: 7.8 Changelog: ========== @@ -101,6 +100,9 @@ - Added ESP32_POE_ISO and TAUDIO - Added animation function, work in progress - Added Speaking state for animation preparation (works for matrixvoice) + v7.8 + - Added animations during audio playback, every device has those animation defaulted to not supported + - Implemented for M5 Atom echo and Matrix Voice * ************************************************************************ */ @@ -183,6 +185,8 @@ void setup() { device->setGain(config.gain); device->setVolume(config.volume); + initHeader(device->readSize, device->width, device->rate); + // --------------------------------------------------------------------------- // ArduinoOTA // --------------------------------------------------------------------------- @@ -191,7 +195,7 @@ void setup() { ArduinoOTA .onStart([]() { Serial.println("Uploading..."); - send_event(OtaEvent()); + send_event(UpdateEvent()); }) .onEnd([]() { Serial.println("\nEnd"); diff --git a/PlatformIO/src/StateMachine.hpp b/PlatformIO/src/StateMachine.hpp index 72d1854..767ef27 100644 --- a/PlatformIO/src/StateMachine.hpp +++ b/PlatformIO/src/StateMachine.hpp @@ -7,95 +7,136 @@ class StateMachine : public tinyfsm::Fsm { public: - virtual void react(WifiDisconnectEvent const &) {}; - virtual void react(WifiConnectEvent const &) {}; - virtual void react(MQTTConnectedEvent const &) {}; - virtual void react(MQTTDisconnectedEvent const &) {}; - virtual void react(StreamAudioEvent const &) {}; + virtual void react(WifiDisconnectEvent const &) { + transit(); + }; + virtual void react(WifiConnectEvent const &) { + transit(); + }; + virtual void react(MQTTConnectedEvent const &) { + transit(); + }; + virtual void react(MQTTDisconnectedEvent const &) { + transit(); + }; + virtual void react(BeginPlayAudioEvent const &) {}; + virtual void react(EndPlayAudioEvent const &) {}; + virtual void react(StreamAudioEvent const &) { + xEventGroupClearBits(audioGroup, PLAY); + Serial.println("Send EndPlayAudioEvent in StreamAudioEvent"); + dispatch(EndPlayAudioEvent()); + xEventGroupSetBits(audioGroup, STREAM); + }; virtual void react(IdleEvent const &) {}; - virtual void react(SpeakEvent const &) {}; - virtual void react(OtaEvent const &) {}; - virtual void react(PlayAudioEvent const &) {}; - virtual void react(HotwordDetectedEvent const &) {}; + virtual void react(ErrorEvent const &) {}; + virtual void react(TtsEvent const &) {}; + virtual void react(UpdateEvent const &) {}; + virtual void react(PlayBytesEvent const &) { + xEventGroupClearBits(audioGroup, STREAM); + Serial.println("Send BeginPlayAudioEvent in PlayBytesEvent"); + dispatch(BeginPlayAudioEvent()); + xEventGroupSetBits(audioGroup, PLAY); + }; + virtual void react(ListeningEvent const &) {}; + virtual void react(UpdateConfigurationEvent const &) { + current_colors = COLORS_IDLE; + device->updateBrightness(config.brightness); + xSemaphoreTake(wbSemaphore, portMAX_DELAY); + device->updateColors(current_colors); + xSemaphoreGive(wbSemaphore); + }; - virtual void entry(void) {}; + virtual void entry(void) { + xEventGroupClearBits(audioGroup, PLAY); + xEventGroupClearBits(audioGroup, STREAM); + device->updateBrightness(config.brightness); + xSemaphoreTake(wbSemaphore, portMAX_DELAY); + device->updateColors(current_colors); + xSemaphoreGive(wbSemaphore); + }; virtual void run(void) {}; void exit(void) {}; }; -class Speaking : public StateMachine +class Tts : public StateMachine { void entry(void) override { - Serial.println("Enter Speaking"); + publishDebug("Enter Tts"); + current_colors = COLORS_TTS; + StateMachine::entry(); } void react(IdleEvent const &) override { + publishDebug("IdleEvent in Tts"); transit(); } - void react(HotwordDetectedEvent const &) override { - transit(); + void react(ListeningEvent const &) override { + publishDebug("ListeningEvent in Tts"); + transit(); } - void react(StreamAudioEvent const &) override { - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupSetBits(audioGroup, STREAM); - }; + void react(BeginPlayAudioEvent const &) override { + publishDebug("BeginPlayAudioEvent in Tts"); + transit(); + } +}; - void react(PlayAudioEvent const &) override { - xEventGroupClearBits(audioGroup, STREAM); - xEventGroupSetBits(audioGroup, PLAY); - }; +class TtsPlay : public StateMachine +{ + void entry(void) override { + publishDebug("Enter TtsPlay"); + } + + void react(EndPlayAudioEvent const &) override { + publishDebug("EndPlayAudioEvent in TtsPlay"); + transit(); + } }; class Updating : public StateMachine { void entry(void) override { Serial.println("Enter Updating"); - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupClearBits(audioGroup, STREAM); - device->updateBrightness(config.brightness); - xSemaphoreTake(wbSemaphore, portMAX_DELAY); - device->updateColors(COLORS_OTA); - xSemaphoreGive(wbSemaphore); + current_colors = COLORS_OTA; + StateMachine::entry(); } }; -class HotwordDetected : public StateMachine +class Listening : public StateMachine { void entry(void) override { - Serial.println("Enter HotwordDetected"); - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupClearBits(audioGroup, STREAM); - device->updateBrightness(config.hotword_brightness); - xSemaphoreTake(wbSemaphore, portMAX_DELAY); - device->updateColors(COLORS_HOTWORD); - xSemaphoreGive(wbSemaphore); - initHeader(device->readSize, device->width, device->rate); + publishDebug("Enter Listening"); + current_colors = COLORS_HOTWORD; + StateMachine::entry(); xEventGroupSetBits(audioGroup, STREAM); } - void react(StreamAudioEvent const &) override { - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupSetBits(audioGroup, STREAM); - }; - - void react(PlayAudioEvent const &) override { - xEventGroupClearBits(audioGroup, STREAM); - xEventGroupSetBits(audioGroup, PLAY); - }; - void react(IdleEvent const &) override { + publishDebug("IdleEvent in Listening"); transit(); } - void react(SpeakEvent const &) override { - transit(); + void react(TtsEvent const &) override { + transit(); } - void react(WifiDisconnectEvent const &) override { - transit(); - }; + void react(BeginPlayAudioEvent const &) override { + publishDebug("BeginPlayAudioEvent in Listening"); + transit(); + } +}; + +class ListeningPlay : public StateMachine +{ + void entry(void) override { + publishDebug("Enter ListeningPlay"); + } + + void react(EndPlayAudioEvent const &) override { + publishDebug("EndPlayAudioEvent in ListeningPlay"); + transit(); + } }; class Idle : public StateMachine @@ -103,15 +144,10 @@ class Idle : public StateMachine bool hotwordDetected = false; void entry(void) override { - Serial.println("Enter Idle"); + publishDebug("Enter Idle"); hotwordDetected = false; - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupClearBits(audioGroup, STREAM); - device->updateBrightness(config.brightness); - xSemaphoreTake(wbSemaphore, portMAX_DELAY); - device->updateColors(COLORS_IDLE); - xSemaphoreGive(wbSemaphore); - initHeader(device->readSize, device->width, device->rate); + current_colors = COLORS_IDLE; + StateMachine::entry(); xEventGroupSetBits(audioGroup, STREAM); } @@ -122,35 +158,77 @@ class Idle : public StateMachine std::string message = "{\"init\":{\"type\":\"action\",\"canBeEnqueued\": false},\"siteId\":\"" + std::string(config.siteid) + "\"}"; asyncClient.publish("hermes/dialogueManager/startSession", 0, false, message.c_str()); } + if (configChanged) { + configChanged = false; + transit(); + } } - void react(WifiDisconnectEvent const &) override { - transit(); + void react(ListeningEvent const &) override { + transit(); } - void react(MQTTDisconnectedEvent const &) override { - transit(); + void react(BeginPlayAudioEvent const &) override { + publishDebug("BeginPlayAudioEvent in Idle"); + transit(); } - void react(HotwordDetectedEvent const &) override { - transit(); + void react(TtsEvent const &) override { + transit(); } - void react(StreamAudioEvent const &) override { - xEventGroupClearBits(audioGroup, PLAY); - xEventGroupSetBits(audioGroup, STREAM); - }; + void react(ErrorEvent const &) override { + transit(); + } + + void react(UpdateEvent const &) override { + transit(); + } +}; - void react(PlayAudioEvent const &) override { - xEventGroupClearBits(audioGroup, STREAM); - xEventGroupSetBits(audioGroup, PLAY); - }; +class IdlePlay : public StateMachine +{ + void entry(void) override { + publishDebug("Enter IdlePlay"); + } + + void react(EndPlayAudioEvent const &) override { + publishDebug("EndPlayAudioEvent in IdlePlay"); + transit(); + } +}; + +class Error : public StateMachine +{ + void entry(void) override { + publishDebug("Enter Error"); + current_colors = COLORS_ERROR; + StateMachine::entry(); + } + + void react(IdleEvent const &) override { + publishDebug("IdleEvent in Error"); + transit(); + } + + void react(BeginPlayAudioEvent const &) override { + publishDebug("BeginPlayAudioEvent in Error"); + transit(); + } +}; + +class ErrorPlay : public StateMachine +{ + void entry(void) override { + publishDebug("Enter ErrorPlay"); + } - void react(SpeakEvent const &) override { - transit(); + void react(EndPlayAudioEvent const &) override { + publishDebug("EndPlayAudioEvent in ErrorPlay"); + transit(); } - void react(OtaEvent const &) override { + void react(UpdateEvent const &) override { transit(); } }; @@ -163,21 +241,14 @@ class MQTTConnected : public StateMachine { asyncClient.subscribe(playBytesTopic.c_str(), 0); asyncClient.subscribe(hotwordTopic.c_str(), 0); asyncClient.subscribe(audioTopic.c_str(), 0); - asyncClient.subscribe(debugTopic.c_str(), 0); + //asyncClient.subscribe(debugTopic.c_str(), 0); asyncClient.subscribe(ledTopic.c_str(), 0); asyncClient.subscribe(restartTopic.c_str(), 0); asyncClient.subscribe(sayTopic.c_str(), 0); asyncClient.subscribe(sayFinishedTopic.c_str(), 0); + asyncClient.subscribe(errorTopic.c_str(), 0); transit(); } - - void react(MQTTDisconnectedEvent const &) override { - transit(); - } - - void react(WifiDisconnectEvent const &) override { - transit(); - } }; class MQTTDisconnected : public StateMachine { @@ -222,14 +293,6 @@ class MQTTDisconnected : public StateMachine { } } } - - void react(MQTTConnectedEvent const &) override { - transit(); - } - - void react(WifiDisconnectEvent const &) override { - transit(); - } }; class WifiConnected : public StateMachine @@ -248,11 +311,6 @@ class WifiConnected : public StateMachine ArduinoOTA.begin(); transit(); } - - void react(WifiDisconnectEvent const &) override { - Serial.println("DisconnectEvent"); - transit(); - }; }; class WifiDisconnected : public StateMachine @@ -356,10 +414,6 @@ class WifiDisconnected : public StateMachine #endif } - - void react(WifiConnectEvent const &) override { - transit(); - }; }; FSM_INITIAL_STATE(StateMachine, WifiDisconnected) @@ -415,11 +469,11 @@ void push_i2s_data(const uint8_t *const payload, size_t len) { if (xEventGroupGetBits(audioGroup) != PLAY) { - send_event(PlayAudioEvent()); + Serial.println("Send PlayBytesEvent"); + send_event(PlayBytesEvent()); } vTaskDelay(pdMS_TO_TICKS(50)); } while (audioData.isFull()); - /// Serial.print("."); } } @@ -453,7 +507,8 @@ void handle_playBytes(const std::string& topicstr, uint8_t *payload, size_t len, //At the end, make sure to start play in case the buffer is not full yet if (!audioData.isEmpty() && xEventGroupGetBits(audioGroup) != PLAY) { - send_event(PlayAudioEvent()); + Serial.println("Send PlayBytesEvent"); + send_event(PlayBytesEvent()); } std::vector topicparts = explode("/", topicstr); @@ -465,10 +520,10 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties { const std::string topicstr(topic); - // complete or enf of message has been received + // complete or end of message has been received if (len + index == total) { - if (topicstr.find(sayFinishedTopic.c_str()) != std::string::npos) + if (topicstr.find(errorTopic.c_str()) != std::string::npos) { std::string payloadstr(payload); StaticJsonDocument<300> doc; @@ -477,6 +532,20 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties if (!err) { JsonObject root = doc.as(); if (root["siteId"] == config.siteid.c_str()) { + Serial.println("Send ErrorEvent from errorTopic"); + send_event(ErrorEvent()); + } + } + } else if (topicstr.find(sayFinishedTopic.c_str()) != std::string::npos) + { + std::string payloadstr(payload); + StaticJsonDocument<300> doc; + DeserializationError err = deserializeJson(doc, payloadstr.c_str()); + // Check if this is for us + if (!err) { + JsonObject root = doc.as(); + if (root["siteId"] == config.siteid.c_str()) { + Serial.println("Send IdleEvent from sayFinishedTopic"); send_event(IdleEvent()); } } @@ -489,7 +558,8 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties if (!err) { JsonObject root = doc.as(); if (root["siteId"] == config.siteid.c_str()) { - send_event(SpeakEvent()); + Serial.println("Send TtsEvent from sayTopic"); + send_event(TtsEvent()); } } } else if (topicstr.find("toggleOff") != std::string::npos) @@ -502,10 +572,16 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties JsonObject root = doc.as(); if (root["siteId"] == config.siteid.c_str() && root.containsKey("reason")) { if (root["reason"] == "dialogueSession") { - send_event(HotwordDetectedEvent()); + Serial.println("Send ListeningEvent from toggleOff (dialogueSession)"); + send_event(ListeningEvent()); } if (root["reason"] == "ttsSay") { - send_event(SpeakEvent()); + Serial.println("Send TtsEvent from toggleOff (ttsSay)"); + send_event(TtsEvent()); + } + if (root["reason"] == "playAudio") { + Serial.println("Send ListeningEvent from toggleOff (playAudio)"); + send_event(ListeningEvent()); } } } @@ -517,9 +593,18 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties if (!err) { JsonObject root = doc.as(); if (root["siteId"] == config.siteid.c_str() && root.containsKey("reason")) { - if (root["reason"] == "dialogueSession" || root["reason"] == "ttsSay") { + if (root["reason"] == "dialogueSession") { + Serial.println("Send IdleEvent from toggleOn (dialogueSession)"); send_event(IdleEvent()); } + if (root["reason"] == "ttsSay") { + Serial.println("Send IdleEvent from toggleOn (ttsSay)"); + send_event(IdleEvent()); + } + if (root["reason"] == "playAudio") { + Serial.println("Send IdleEvent from toggleOn (playAudio)"); + send_event(IdleEvent()); + } } } } @@ -533,6 +618,10 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties DeserializationError err = deserializeJson(doc, payloadstr.c_str()); if (!err) { JsonObject root = doc.as(); + if (root.containsKey("animation")) { + config.animation = (uint16_t)(root["animation"]); + saveNeeded = true; + } if (root.containsKey("brightness")) { if (config.brightness != (int)root["brightness"]) { config.brightness = (int)(root["brightness"]); @@ -543,39 +632,51 @@ void onMqttMessage(char *topic, char *payload, AsyncMqttClientMessageProperties config.hotword_brightness = (int)(root["hotword_brightness"]); } if (root.containsKey("hotword")) { - hotword_colors[0] = root["hotword"][0]; - hotword_colors[1] = root["hotword"][1]; - hotword_colors[2] = root["hotword"][2]; - hotword_colors[3] = root["hotword"][3]; + ColorMap[COLORS_HOTWORD][0] = root["hotword"][0]; + ColorMap[COLORS_HOTWORD][1] = root["hotword"][1]; + ColorMap[COLORS_HOTWORD][2] = root["hotword"][2]; + ColorMap[COLORS_HOTWORD][3] = root["hotword"][3]; + } + if (root.containsKey("tts")) { + ColorMap[COLORS_TTS][0] = root["tts"][0]; + ColorMap[COLORS_TTS][1] = root["tts"][1]; + ColorMap[COLORS_TTS][2] = root["tts"][2]; + ColorMap[COLORS_TTS][3] = root["tts"][3]; } if (root.containsKey("idle")) { - idle_colors[0] = root["idle"][0]; - idle_colors[1] = root["idle"][1]; - idle_colors[2] = root["idle"][2]; - idle_colors[3] = root["idle"][3]; + ColorMap[COLORS_IDLE][0] = root["idle"][0]; + ColorMap[COLORS_IDLE][1] = root["idle"][1]; + ColorMap[COLORS_IDLE][2] = root["idle"][2]; + ColorMap[COLORS_IDLE][3] = root["idle"][3]; } if (root.containsKey("wifi_disconnect")) { - wifi_disc_colors[0] = root["wifi_disconnect"][0]; - wifi_disc_colors[1] = root["wifi_disconnect"][1]; - wifi_disc_colors[2] = root["wifi_disconnect"][2]; - wifi_disc_colors[3] = root["wifi_disconnect"][3]; + ColorMap[COLORS_WIFI_DISCONNECTED][0] = root["wifi_disconnect"][0]; + ColorMap[COLORS_WIFI_DISCONNECTED][1] = root["wifi_disconnect"][1]; + ColorMap[COLORS_WIFI_DISCONNECTED][2] = root["wifi_disconnect"][2]; + ColorMap[COLORS_WIFI_DISCONNECTED][3] = root["wifi_disconnect"][3]; } if (root.containsKey("wifi_connect")) { - wifi_conn_colors[0] = root["wifi_connect"][0]; - wifi_conn_colors[1] = root["wifi_connect"][1]; - wifi_conn_colors[2] = root["wifi_connect"][2]; - wifi_conn_colors[3] = root["wifi_connect"][3]; + ColorMap[COLORS_WIFI_CONNECTED][0] = root["wifi_connect"][0]; + ColorMap[COLORS_WIFI_CONNECTED][1] = root["wifi_connect"][1]; + ColorMap[COLORS_WIFI_CONNECTED][2] = root["wifi_connect"][2]; + ColorMap[COLORS_WIFI_CONNECTED][3] = root["wifi_connect"][3]; } if (root.containsKey("update")) { - ota_colors[0] = root["update"][0]; - ota_colors[1] = root["update"][1]; - ota_colors[2] = root["update"][2]; - ota_colors[3] = root["update"][3]; + ColorMap[COLORS_OTA][0] = root["update"][0]; + ColorMap[COLORS_OTA][1] = root["update"][1]; + ColorMap[COLORS_OTA][2] = root["update"][2]; + ColorMap[COLORS_OTA][3] = root["update"][3]; + } + if (root.containsKey("error")) { + ColorMap[COLORS_ERROR][0] = root["error"][0]; + ColorMap[COLORS_ERROR][1] = root["error"][1]; + ColorMap[COLORS_ERROR][2] = root["error"][2]; + ColorMap[COLORS_ERROR][3] = root["error"][3]; } if (saveNeeded) { saveConfiguration(configfile, config); } - send_event(IdleEvent()); + send_event(UpdateConfigurationEvent()); } else { publishDebug(err.c_str()); } @@ -658,11 +759,9 @@ void I2Stask(void *p) { while (played < message_size && timeout == false) { - if (fsm::is_in_state()) { - xSemaphoreTake(wbSemaphore, portMAX_DELAY); - device->animate(COLORS_OTA); - xSemaphoreGive(wbSemaphore); - } + xSemaphoreTake(wbSemaphore, portMAX_DELAY); + device->animate(current_colors, config.animation); + xSemaphoreGive(wbSemaphore); int bytes_to_write = device->writeSize; if (message_size - played < device->writeSize) { @@ -703,6 +802,7 @@ void I2Stask(void *p) { xSemaphoreGive(wbSemaphore); audioData.clear(); Serial.println("Done"); + Serial.println("Send StreamAudioEvent"); send_event(StreamAudioEvent()); } if (xEventGroupGetBits(audioGroup) == STREAM && !config.mute_input) { @@ -747,23 +847,6 @@ void I2Stask(void *p) { vTaskDelete(NULL); } -void initHeader(int readSize, int width, int rate) { - strncpy(header.riff_tag, "RIFF", 4); - strncpy(header.wave_tag, "WAVE", 4); - strncpy(header.fmt_tag, "fmt ", 4); - strncpy(header.data_tag, "data", 4); - - header.riff_length = (uint32_t)sizeof(header) + (readSize * width); - header.fmt_length = 16; - header.audio_format = 1; - header.num_channels = 1; - header.sample_rate = rate; - header.byte_rate = rate * width; - header.block_align = width; - header.bits_per_sample = width * 8; - header.data_length = readSize * width; -} - void WiFiEvent(WiFiEvent_t event) { switch (event) { diff --git a/PlatformIO/src/device.h b/PlatformIO/src/device.h index 24d8197..0e96534 100644 --- a/PlatformIO/src/device.h +++ b/PlatformIO/src/device.h @@ -1,17 +1,34 @@ #pragma once +#include + int hotword_colors[4] = {0, 255, 0, 0}; int idle_colors[4] = {0, 0, 255, 0}; int wifi_conn_colors[4] = {0, 0, 255, 0}; int wifi_disc_colors[4] = {255, 0, 0, 0}; int ota_colors[4] = {0, 0, 0, 255}; +int tts_colors[4] = {173, 17, 240, 0}; +int error_colors[4] = {150, 255, 0, 0}; enum { COLORS_HOTWORD = 0, COLORS_WIFI_CONNECTED = 1, COLORS_WIFI_DISCONNECTED = 2, COLORS_IDLE = 3, - COLORS_OTA = 4 + COLORS_OTA = 4, + COLORS_TTS = 5, + COLORS_ERROR = 6 +}; + +std::map ColorMap = { + {COLORS_IDLE, idle_colors }, + {COLORS_HOTWORD, hotword_colors }, + {COLORS_WIFI_CONNECTED, wifi_conn_colors }, + {COLORS_WIFI_DISCONNECTED, wifi_disc_colors }, + {COLORS_OTA, ota_colors }, + {COLORS_TTS, tts_colors }, + {COLORS_ERROR, error_colors } }; + enum DeviceMode { MODE_UNUSED = -1, // indicates the no explicit mode change has been stored yet MODE_MIC = 0, @@ -26,6 +43,13 @@ enum AmpOut { AMP_OUT_BOTH = 2, }; +enum AnimationMode { + SOLID = 0, + RUN = 1, + PULSE = 2, + BLINK = 3 +}; + class Device { protected: // for derived classes which switch between read and write mode, we store there which mode is active. Otherwise it should remain as MODE_UNUSED @@ -36,7 +60,11 @@ class Device { //If your device has leds, override these methods to set the colors and brightness virtual void updateColors(int colors) {}; //You can create some animation here - virtual void animate(int colors) {}; + virtual void animate(int colors, int mode) {}; + virtual void animateRunning(int colors) {}; + virtual void animatePulsing(int colors) {}; + virtual void animateBlinking(int colors) {}; + // virtual void updateBrightness(int brightness) {}; //It may be needed to switch between read and write, i.e. if the need the same PIN //Override both methods @@ -58,6 +86,11 @@ class Device { // how many different output configurations does this devices support (1 = single output channel, 2 = 2 output channels, i.e. speaker or headphone, 3 = speaker, headphone, speaker + headphone) virtual int numAmpOutConfigurations() { return 2; }; + + virtual bool animationSupported() { return false; }; + virtual bool runningSupported() { return false; }; + virtual bool pulsingSupported() { return false; }; + virtual bool blinkingSupported() { return false; }; // //You can override these in your device int readSize = 256; diff --git a/PlatformIO/src/devices/M5AtomEcho.hpp b/PlatformIO/src/devices/M5AtomEcho.hpp index 3078f56..1d44494 100644 --- a/PlatformIO/src/devices/M5AtomEcho.hpp +++ b/PlatformIO/src/devices/M5AtomEcho.hpp @@ -17,6 +17,9 @@ class M5AtomEcho : public Device public: M5AtomEcho(); void init(); + void animate(int colors, int mode); + void animateBlinking(int colors); + void animatePulsing(int colors); void updateColors(int colors); void updateBrightness(int brightness); void setReadMode(); @@ -25,9 +28,17 @@ class M5AtomEcho : public Device bool readAudio(uint8_t *data, size_t size); bool isHotwordDetected(); int numAmpOutConfigurations() { return 1; }; + bool animationSupported() { return true; }; + bool runningSupported() { return false; }; + bool pulsingSupported() { return true; }; + bool blinkingSupported() { return true; }; private: void InitI2SSpeakerOrMic(int mode); + long currentMillis, startMillis; + bool ledsOn = true; + bool directionDown = false; + int brightness, pulse; }; M5AtomEcho::M5AtomEcho() @@ -37,6 +48,8 @@ M5AtomEcho::M5AtomEcho() void M5AtomEcho::init() { M5.begin(true,true,true); + currentMillis = millis(); + startMillis = millis(); }; bool M5AtomEcho::isHotwordDetected() { @@ -46,27 +59,54 @@ bool M5AtomEcho::isHotwordDetected() { void M5AtomEcho::updateColors(int colors) { - switch (colors) { - case COLORS_HOTWORD: - M5.dis.drawpix(0, CRGB(hotword_colors[1],hotword_colors[0],hotword_colors[2])); - break; - case COLORS_WIFI_CONNECTED: - M5.dis.drawpix(0, CRGB(wifi_conn_colors[0],wifi_conn_colors[1],wifi_conn_colors[2])); - break; - case COLORS_IDLE: - M5.dis.drawpix(0, CRGB(idle_colors[0],idle_colors[1],idle_colors[2])); + //Red and Green seem to be switched, we also need to map the white + float alpha = 0.6 * (1.0 - (1.0 - ColorMap[colors][3]) * (1.0 - ColorMap[colors][3])); + int r = (1.0 - alpha) * ColorMap[colors][1] + alpha; + int g = (1.0 - alpha) * ColorMap[colors][0] + alpha; + int b = (1.0 - alpha) * ColorMap[colors][2] + alpha; + M5.dis.drawpix(0, CRGB(r, g, b)); +}; + +void M5AtomEcho::updateBrightness(int brightness) { + M5.dis.setBrightness(brightness); + M5AtomEcho::pulse = brightness; + M5AtomEcho::brightness = 0; +} + +void M5AtomEcho::animate(int colors, int mode) { + switch (mode) + { + case AnimationMode::BLINK: + animateBlinking(colors); break; - case COLORS_WIFI_DISCONNECTED: - M5.dis.drawpix(0, CRGB(wifi_disc_colors[1],wifi_disc_colors[0],wifi_disc_colors[2])); + case AnimationMode::PULSE: + animatePulsing(colors); break; - case COLORS_OTA: - M5.dis.drawpix(0, CRGB(ota_colors[3],ota_colors[3],ota_colors[3])); + default: break; } -}; +} -void M5AtomEcho::updateBrightness(int brightness) { - M5.dis.setBrightness(brightness); +void M5AtomEcho::animatePulsing(int colors) { + currentMillis = millis(); + if (currentMillis - startMillis > 5) { + if (M5AtomEcho::pulse > M5AtomEcho::brightness) { directionDown = true; } + M5AtomEcho::pulse = directionDown ? M5AtomEcho::pulse - 5 : M5AtomEcho::pulse + 5; + if (M5AtomEcho::pulse < 5) { directionDown = false; } + startMillis = millis(); + M5.dis.setBrightness(M5AtomEcho::pulse); + updateColors(colors); + } +} + +void M5AtomEcho::animateBlinking(int colors) { + currentMillis = millis(); + if (currentMillis - startMillis > 300) { + M5.dis.setBrightness(ledsOn ? M5AtomEcho::brightness : 0); + ledsOn = !ledsOn; + startMillis = millis(); + updateColors(colors); + } } void M5AtomEcho::InitI2SSpeakerOrMic(int mode) diff --git a/PlatformIO/src/devices/MatrixVoice.hpp b/PlatformIO/src/devices/MatrixVoice.hpp index a763a8b..f92bff3 100644 --- a/PlatformIO/src/devices/MatrixVoice.hpp +++ b/PlatformIO/src/devices/MatrixVoice.hpp @@ -44,39 +44,49 @@ class MatrixVoice : public Device public: MatrixVoice(); void init(); - void animate(int colors); - void updateColors(int colors); - void updateBrightness(int brightness); + void animate(int colors, int mode); + void animateRunning(int colors); + void animateBlinking(int colors); + void animatePulsing(int colors); + void updateColors(int colors); + void updateBrightness(int brightness); void muteOutput(bool mute); void setVolume(uint16_t volume); void setWriteMode(int sampleRate, int bitDepth, int numChannels); - bool readAudio(uint8_t *data, size_t size); + bool readAudio(uint8_t *data, size_t size); void writeAudio(uint8_t *data, size_t size, size_t *bytes_written); void ampOutput(int output); - int readSize = 512; - int writeSize = 1024; - int width = 2; - int rate = 16000; + bool animationSupported() { return true; }; + bool runningSupported() { return true; }; + bool pulsingSupported() { return true; }; + bool blinkingSupported() { return true; }; + int readSize = 512; + int writeSize = 1024; + int width = 2; + int rate = 16000; private: matrix_hal::WishboneBus wb; matrix_hal::Everloop everloop; - matrix_hal::MicrophoneArray *mics; + matrix_hal::MicrophoneArray *mics; matrix_hal::EverloopImage image1d; void playBytes(int16_t* input, uint32_t length); - void interleave(const int16_t * in_L, const int16_t * in_R, int16_t * out, const size_t num_samples); - bool FIFOFlush(); - uint16_t GetFIFOStatus(); - uint32_t PCM_sampling_frequency = 16000; - int fifoSize = 4096; + void interleave(const int16_t * in_L, const int16_t * in_R, int16_t * out, const size_t num_samples); + bool FIFOFlush(); + void updateColors(int colors, bool usePulse); + uint16_t GetFIFOStatus(); + uint32_t PCM_sampling_frequency = 16000; + int fifoSize = 4096; int sampleRate, bitDepth, numChannels; - int brightness = 15; - float sample_time = 1.0 / 16000; - uint32_t spiLength = 1024; + int brightness, pulse = 15; + float sample_time = 1.0 / 16000; + uint32_t spiLength = 1024; int sleep = int(spiLength * sample_time * 1000); - int count = 0; - int position = 0; - long currentMillis, startMillis; + int count = 0; + int position = 0; + long currentMillis, startMillis; + bool ledsOn = true; + bool directionDown = false; }; MatrixVoice::MatrixVoice() @@ -85,102 +95,114 @@ MatrixVoice::MatrixVoice() void MatrixVoice::init() { - Serial.println("Matrix Voice Initialized"); + Serial.println("Matrix Voice Initialized"); wb.Init(); everloop.Setup(&wb); - mics = new matrix_hal::MicrophoneArray(); + mics = new matrix_hal::MicrophoneArray(); mics->Setup(&wb); mics->SetSamplingRate(rate); matrix_hal::MicrophoneCore mic_core(*mics); mic_core.Setup(&wb); uint16_t PCM_constant = 492; wb.SpiWrite(matrix_hal::kConfBaseAddress + 9, (const uint8_t *)(&PCM_constant), sizeof(uint16_t)); - currentMillis = millis(); - startMillis = millis(); + currentMillis = millis(); + startMillis = millis(); }; void MatrixVoice::updateBrightness(int brightness) { - // all values below 10 is read as 0 in gamma8, we map 0 to 10 - if (brightness > 100) { brightness = 100; } - MatrixVoice::brightness = brightness * 90 / 100 + 10; + // all values below 10 is read as 0 in gamma8, we map 0 to 10 + if (brightness > 100) { brightness = 100; } + MatrixVoice::brightness = brightness * 90 / 100 + 10; + MatrixVoice::pulse = brightness * 90 / 100 + 10; } -void MatrixVoice::animate(int colors) { - currentMillis = millis(); - if (currentMillis - startMillis > 10) { - startMillis = millis(); - int r = 0; - int g = 0; - int b = 0; - int w = 0; - for (int i = 0; i < image1d.leds.size(); i++) { - r = ((i + 1) * brightness / image1d.leds.size()) * ota_colors[0] / 100; - g = ((i + 1) * brightness / image1d.leds.size()) * ota_colors[1] / 100; - b = ((i + 1) * brightness / image1d.leds.size()) * ota_colors[2] / 100; - w = ((i + 1) * brightness / image1d.leds.size()) * ota_colors[3] / 100; - image1d.leds[(i + position) % image1d.leds.size()].red = pgm_read_byte(&gamma8[r]); - image1d.leds[(i + position) % image1d.leds.size()].green = pgm_read_byte(&gamma8[g]); - image1d.leds[(i + position) % image1d.leds.size()].blue = pgm_read_byte(&gamma8[b]); - image1d.leds[(i + position) % image1d.leds.size()].white = pgm_read_byte(&gamma8[w]); - } - position++; - position %= image1d.leds.size(); - everloop.Write(&image1d); - } -} - -void MatrixVoice::updateColors(int colors) { - int r = 0; - int g = 0; - int b = 0; - int w = 0; - switch (colors) { - case COLORS_HOTWORD: - r = hotword_colors[0]; - g = hotword_colors[1]; - b = hotword_colors[2]; - w = hotword_colors[3]; +void MatrixVoice::animate(int colors, int mode) { + switch (mode) + { + case AnimationMode::RUN: + animateRunning(colors); break; - case COLORS_WIFI_CONNECTED: - r = wifi_conn_colors[0]; - g = wifi_conn_colors[1]; - b = wifi_conn_colors[2]; - w = wifi_conn_colors[3]; + case AnimationMode::BLINK: + animateBlinking(colors); break; - case COLORS_IDLE: - r = idle_colors[0]; - g = idle_colors[1]; - b = idle_colors[2]; - w = idle_colors[3]; + case AnimationMode::PULSE: + animatePulsing(colors); break; - case COLORS_WIFI_DISCONNECTED: - r = wifi_disc_colors[0]; - g = wifi_disc_colors[1]; - b = wifi_disc_colors[2]; - w = wifi_disc_colors[3]; - break; - case COLORS_OTA: - r = ota_colors[0]; - g = ota_colors[1]; - b = ota_colors[2]; - w = ota_colors[3]; + default: break; } - r = floor(MatrixVoice::brightness * r / 100); - r = pgm_read_byte(&gamma8[r]); - g = floor(MatrixVoice::brightness * g / 100); - g = pgm_read_byte(&gamma8[g]); - b = floor(MatrixVoice::brightness * b / 100); - b = pgm_read_byte(&gamma8[b]); - w = floor(MatrixVoice::brightness * w / 100); - w = pgm_read_byte(&gamma8[w]); - for (matrix_hal::LedValue &led : image1d.leds) { - led.red = r; - led.green = g; - led.blue = b; - led.white = w; - } - everloop.Write(&image1d); +} + +void MatrixVoice::animateRunning(int colors) { + currentMillis = millis(); + if (currentMillis - startMillis > 10) { + int r = ColorMap[colors][0]; + int g = ColorMap[colors][1]; + int b = ColorMap[colors][2]; + int w = ColorMap[colors][3]; + startMillis = millis(); + for (int i = 0; i < image1d.leds.size(); i++) { + int red = ((i + 1) * brightness / image1d.leds.size()) * r / 100; + int green = ((i + 1) * brightness / image1d.leds.size()) * g / 100; + int blue = ((i + 1) * brightness / image1d.leds.size()) * b / 100; + int white = ((i + 1) * brightness / image1d.leds.size()) * w / 100; + image1d.leds[(i + position) % image1d.leds.size()].red = pgm_read_byte(&gamma8[red]); + image1d.leds[(i + position) % image1d.leds.size()].green = pgm_read_byte(&gamma8[green]); + image1d.leds[(i + position) % image1d.leds.size()].blue = pgm_read_byte(&gamma8[blue]); + image1d.leds[(i + position) % image1d.leds.size()].white = pgm_read_byte(&gamma8[white]); + } + position++; + position %= image1d.leds.size(); + everloop.Write(&image1d); + } +} + +void MatrixVoice::animateBlinking(int colors) { + currentMillis = millis(); + if (currentMillis - startMillis > 300) { + MatrixVoice::pulse = ledsOn ? MatrixVoice::brightness : 0; + ledsOn = !ledsOn; + startMillis = millis(); + updateColors(colors, true); + } +} + +void MatrixVoice::animatePulsing(int colors) { + currentMillis = millis(); + if (currentMillis - startMillis > 5) { + if (MatrixVoice::pulse > MatrixVoice::brightness) { directionDown = true; } + MatrixVoice::pulse = directionDown ? MatrixVoice::pulse - 5 : MatrixVoice::pulse + 5; + if (MatrixVoice::pulse < 5) { directionDown = false; } + startMillis = millis(); + updateColors(colors, true); + } +} + +void MatrixVoice::updateColors(int colors, bool usePulse) { + int r = ColorMap[colors][0]; + int g = ColorMap[colors][1]; + int b = ColorMap[colors][2]; + int w = ColorMap[colors][3]; + int brightness = usePulse ? MatrixVoice::pulse : MatrixVoice::brightness; + r = floor(brightness * r / 100); + r = pgm_read_byte(&gamma8[r]); + g = floor(brightness * g / 100); + g = pgm_read_byte(&gamma8[g]); + b = floor(brightness * b / 100); + b = pgm_read_byte(&gamma8[b]); + w = floor(brightness * w / 100); + w = pgm_read_byte(&gamma8[w]); + for (matrix_hal::LedValue &led : image1d.leds) { + led.red = r; + led.green = g; + led.blue = b; + led.white = w; + } + everloop.Write(&image1d); +} + +void MatrixVoice::updateColors(int colors) { + updateColors(colors, false); } void MatrixVoice::muteOutput(bool mute) { @@ -189,8 +211,8 @@ void MatrixVoice::muteOutput(bool mute) { } void MatrixVoice::setVolume(uint16_t volume) { - uint16_t outputVolume = (100 - volume) * 25 / 100; //25 is minimum volume - wb.SpiWrite(matrix_hal::kConfBaseAddress+8,(const uint8_t *)(&outputVolume), sizeof(uint16_t)); + uint16_t outputVolume = (100 - volume) * 25 / 100; //25 is minimum volume + wb.SpiWrite(matrix_hal::kConfBaseAddress+8,(const uint8_t *)(&outputVolume), sizeof(uint16_t)); }; void MatrixVoice::ampOutput(int output) { @@ -198,126 +220,126 @@ void MatrixVoice::ampOutput(int output) { }; void MatrixVoice::setWriteMode(int sampleRate, int bitDepth, int numChannels) { - MatrixVoice::sampleRate = sampleRate; - MatrixVoice::bitDepth = bitDepth; - MatrixVoice::numChannels = numChannels; - FIFOFlush(); - speex_resampler_set_rate(resampler,sampleRate,16000); - speex_resampler_skip_zeros(resampler); - switch (sampleRate) - { - case 22050: - writeSize = 1411; - break; - case 44100: - writeSize = 2823; - break; - default: - writeSize = 1024; - break; - } - if (numChannels == 1) { - writeSize = writeSize / sizeof(uint16_t); - } - count = 0; - spiLength = 1024; + MatrixVoice::sampleRate = sampleRate; + MatrixVoice::bitDepth = bitDepth; + MatrixVoice::numChannels = numChannels; + FIFOFlush(); + speex_resampler_set_rate(resampler,sampleRate,16000); + speex_resampler_skip_zeros(resampler); + switch (sampleRate) + { + case 22050: + writeSize = 1411; + break; + case 44100: + writeSize = 2823; + break; + default: + writeSize = 1024; + break; + } + if (numChannels == 1) { + writeSize = writeSize / sizeof(uint16_t); + } + count = 0; + spiLength = 1024; }; bool MatrixVoice::readAudio(uint8_t *data, size_t size) { - mics->Read(); - uint16_t voicebuffer[readSize]; - for (uint32_t s = 0; s < readSize; s++) { - voicebuffer[s] = mics->Beam(s); - } + mics->Read(); + uint16_t voicebuffer[readSize]; + for (uint32_t s = 0; s < readSize; s++) { + voicebuffer[s] = mics->Beam(s); + } memcpy(data, voicebuffer, readSize * width); - return true; + return true; } void MatrixVoice::writeAudio(uint8_t *data, size_t inputLength, size_t *bytes_written) { - *bytes_written = inputLength; - uint32_t outputLength = (numChannels == 1) ? inputLength * sizeof(int16_t) : inputLength; - int16_t output[outputLength]; + *bytes_written = inputLength; + uint32_t outputLength = (numChannels == 1) ? inputLength * sizeof(int16_t) : inputLength; + int16_t output[outputLength]; if (numChannels == 1) { - uint32_t monoLength = inputLength / sizeof(int16_t); - int16_t mono[monoLength]; - int16_t stereo[inputLength]; - for (int i = 0; i < inputLength; i += 2) { - mono[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); - } - interleave(mono, mono, stereo, monoLength); - for (int i = 0; i < inputLength; i++) { - output[i] = stereo[i]; - } - } else { - for (int i = 0; i < inputLength; i += 2) { - output[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); - } - } - - //At this point, we always have a stereo audio message, at whatever rate. - //The length of the bytes is dependant on the input sampleRate. - //1024 with 16000Hz, 2822 for 441000. (44100/16000) * 1024 = 2822 - //We might need to resample of the inputRate is higher than 16000 - uint16_t spiSamples[spiLength]; - if (sampleRate > 16000) { - int16_t resampled[spiLength]; - //Serial.printf("before resample %d, %d\r\n", spiLength, outputLength); - speex_resampler_process_interleaved_int(resampler, output, &outputLength, resampled, &spiLength); - //Serial.printf("after resample %d, %d\r\n", spiLength, outputLength); - for (int i = 0; i < spiLength; i++) { - spiSamples[i] = resampled[i]; - } - } else { - for (int i = 0; i < spiLength; i++) { - spiSamples[i] = output[i]; - } - } - - - if (GetFIFOStatus() > fifoSize * 3 / 4) { - std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); - } - Serial.printf("Write %d, %d, %d, %d\r\n", spiLength, sizeof(spiSamples), sleep, count); - wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)spiSamples, spiLength); - count++; + uint32_t monoLength = inputLength / sizeof(int16_t); + int16_t mono[monoLength]; + int16_t stereo[inputLength]; + for (int i = 0; i < inputLength; i += 2) { + mono[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); + } + interleave(mono, mono, stereo, monoLength); + for (int i = 0; i < inputLength; i++) { + output[i] = stereo[i]; + } + } else { + for (int i = 0; i < inputLength; i += 2) { + output[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); + } + } + + //At this point, we always have a stereo audio message, at whatever rate. + //The length of the bytes is dependant on the input sampleRate. + //1024 with 16000Hz, 2822 for 441000. (44100/16000) * 1024 = 2822 + //We might need to resample of the inputRate is higher than 16000 + uint16_t spiSamples[spiLength]; + if (sampleRate > 16000) { + int16_t resampled[spiLength]; + //Serial.printf("before resample %d, %d\r\n", spiLength, outputLength); + speex_resampler_process_interleaved_int(resampler, output, &outputLength, resampled, &spiLength); + //Serial.printf("after resample %d, %d\r\n", spiLength, outputLength); + for (int i = 0; i < spiLength; i++) { + spiSamples[i] = resampled[i]; + } + } else { + for (int i = 0; i < spiLength; i++) { + spiSamples[i] = output[i]; + } + } + + + if (GetFIFOStatus() > fifoSize * 3 / 4) { + std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); + } + Serial.printf("Write %d, %d, %d, %d\r\n", spiLength, sizeof(spiSamples), sleep, count); + wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)spiSamples, spiLength); + count++; // if (numChannels == 1) { - // // int16_t mono[size / sizeof(int16_t)]; - // // int16_t stereo[size]; - // // for (int i = 0; i < size; i += 2) { - // // mono[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); - // // } - // // interleave(mono, mono, stereo, size / sizeof(int16_t)); - // // if (fifo_status > fifoSize * 3 / 4) { - // // int sleep = int(size * sizeof(int16_t) * sample_time * 1000); - // // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); - // // } - // // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)stereo, size * sizeof(int16_t)); - // } else { - // if (sampleRate == 44100) { - // uint32_t in_len = size / sizeof(int16_t); - // uint32_t out_len = 1024 / sizeof(int16_t); - // int16_t input[in_len]; - // int16_t output[out_len]; - // for (int i = 0; i < size; i += 2) { - // input[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); - // } - // speex_resampler_process_interleaved_int(resampler, input, &in_len, output, &out_len); - // if (fifo_status > fifoSize * 3 / 4) { - // int sleep = int( ( out_len / sizeof(int16_t)) * sample_time * 1000); - // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); - // } - // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)output, out_len * sizeof(int16_t)); - // } else { - // if (fifo_status > fifoSize * 3 / 4) { - // int sleep = int(size * sample_time * 1000); - // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); - // } - // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)data, size); - // } - //} + // // int16_t mono[size / sizeof(int16_t)]; + // // int16_t stereo[size]; + // // for (int i = 0; i < size; i += 2) { + // // mono[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); + // // } + // // interleave(mono, mono, stereo, size / sizeof(int16_t)); + // // if (fifo_status > fifoSize * 3 / 4) { + // // int sleep = int(size * sizeof(int16_t) * sample_time * 1000); + // // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); + // // } + // // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)stereo, size * sizeof(int16_t)); + // } else { + // if (sampleRate == 44100) { + // uint32_t in_len = size / sizeof(int16_t); + // uint32_t out_len = 1024 / sizeof(int16_t); + // int16_t input[in_len]; + // int16_t output[out_len]; + // for (int i = 0; i < size; i += 2) { + // input[i/2] = ((data[i] & 0xff) | (data[i + 1] << 8)); + // } + // speex_resampler_process_interleaved_int(resampler, input, &in_len, output, &out_len); + // if (fifo_status > fifoSize * 3 / 4) { + // int sleep = int( ( out_len / sizeof(int16_t)) * sample_time * 1000); + // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); + // } + // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)output, out_len * sizeof(int16_t)); + // } else { + // if (fifo_status > fifoSize * 3 / 4) { + // int sleep = int(size * sample_time * 1000); + // std::this_thread::sleep_for(std::chrono::milliseconds(sleep)); + // } + // wb.SpiWrite(matrix_hal::kDACBaseAddress, (const uint8_t *)data, size); + // } + //} } // void MatrixVoice::writeAudio(uint8_t *data, size_t size, size_t *bytes_written) { @@ -393,7 +415,7 @@ void MatrixVoice::writeAudio(uint8_t *data, size_t inputLength, size_t *bytes_wr // if (MatrixVoice::numChannels == 2) { // speex_resampler_process_interleaved_int(resampler, input, &in_len, output, &out_len); - + // //play it! // playBytes(output, out_len); // } else { @@ -423,7 +445,7 @@ void MatrixVoice::interleave(const int16_t * in_L, const int16_t * in_R, int16_t bool MatrixVoice::FIFOFlush() { int16_t value = 1; wb.SpiWrite(matrix_hal::kConfBaseAddress + 12,(const uint8_t *)(&value), sizeof(uint16_t)); - value = 0; + value = 0; wb.SpiWrite(matrix_hal::kConfBaseAddress + 12,(const uint8_t *)(&value), sizeof(uint16_t)); return true; } @@ -431,8 +453,8 @@ bool MatrixVoice::FIFOFlush() { uint16_t MatrixVoice::GetFIFOStatus() { uint16_t write_pointer; uint16_t read_pointer; - wb.SpiRead(matrix_hal::kDACBaseAddress + 2050, (uint8_t *)(&read_pointer), sizeof(uint16_t)); - wb.SpiRead(matrix_hal::kDACBaseAddress + 2051, (uint8_t *)(&write_pointer), sizeof(uint16_t)); + wb.SpiRead(matrix_hal::kDACBaseAddress + 2050, (uint8_t *)(&read_pointer), sizeof(uint16_t)); + wb.SpiRead(matrix_hal::kDACBaseAddress + 2051, (uint8_t *)(&write_pointer), sizeof(uint16_t)); if (write_pointer > read_pointer) return write_pointer - read_pointer; diff --git a/PlatformIO/src/devices/TAudio.hpp b/PlatformIO/src/devices/TAudio.hpp index 88928b4..8b3d7a4 100644 --- a/PlatformIO/src/devices/TAudio.hpp +++ b/PlatformIO/src/devices/TAudio.hpp @@ -134,23 +134,7 @@ void TAudio::InitI2S() { } void TAudio::updateColors(int colors) { - switch (colors) { - case COLORS_HOTWORD: - strip.ClearTo(RgbColor(RgbwColor(hotword_colors[0],hotword_colors[1],hotword_colors[2],hotword_colors[3])).Dim(brightness)); - break; - case COLORS_WIFI_CONNECTED: - strip.ClearTo(RgbColor(RgbwColor(wifi_conn_colors[0],wifi_conn_colors[1],wifi_conn_colors[2],wifi_conn_colors[3])).Dim(brightness)); - break; - case COLORS_IDLE: - strip.ClearTo(RgbColor(RgbwColor(idle_colors[0],idle_colors[1],idle_colors[2],idle_colors[3])).Dim(brightness)); - break; - case COLORS_WIFI_DISCONNECTED: - strip.ClearTo(RgbColor(RgbwColor(wifi_disc_colors[0],wifi_disc_colors[1],wifi_disc_colors[2],wifi_disc_colors[3])).Dim(brightness)); - break; - case COLORS_OTA: - strip.ClearTo(RgbColor(RgbwColor(ota_colors[0],ota_colors[1],ota_colors[2],ota_colors[3])).Dim(brightness)); - break; - } + strip.ClearTo(RgbColor(RgbwColor(ColorMap[colors][0],ColorMap[colors][1],ColorMap[colors][2],ColorMap[colors][3])).Dim(brightness)); strip.Show(); } diff --git a/PlatformIO/src/index_html.h b/PlatformIO/src/index_html.h index 090da18..b2d6b37 100644 --- a/PlatformIO/src/index_html.h +++ b/PlatformIO/src/index_html.h @@ -102,6 +102,15 @@ input::-moz-focus-inner,input::-moz-focus-outer {border: 0;} 0 +
+ + +
diff --git a/README.md b/README.md index fc470ac..777efd3 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Support for Snips is dropped. - Supports multiple devices, and you are welcomed to add more devices. Read futher below as to how. - For the Matrix Voice, during first flash a Raspberri Pi is needed, afer that OTA can be used - LED Support +- Various colors per state - OTA Updating - Dynamic brightness and colors for idle, hotword and disconnected - Mute / unmute microphones via MQTT @@ -18,8 +19,9 @@ Support for Snips is dropped. - Adjust gain via MQTT (if supported by device) - Reboot device by sending hashed password - Configuration possible in browser -- Audio playback, recommended not higher than 441000 samplerate (see Known Issues) +- Audio playback, recommended not higher than 16000 samplerate (see Known Issues) - Hardware button to start session (if supported by device) +- Animations when audio is played ## Getting started [Matrix Voice](matrixvoice.md) @@ -40,7 +42,7 @@ The ESP32 Satellite is subscribed to various topics. The topic SITEID/led, where SITEID is the name you have given the device in settings.ini, is used for commands concerning the leds on the device. When publishing to this topic, the led colors and brightness can be altered without coding and will be saved to a config file -The message can contain 7 keys: +The message can contain 10 keys: - brightness: integer value between 0 and 100 (%) - hotword_brightness: integer value between 0 and 100 (%) @@ -49,6 +51,9 @@ The message can contain 7 keys: - update: array of 4 codes: [red,green,blue,white], ranging 0-255 - wifi_disconnect: array of 4 codes: [red,green,blue,white], ranging 0-255 - wifi_connect: array of 4 codes: [red,green,blue,white], ranging 0-255 +- tts: array of 4 codes: [red,green,blue,white], ranging 0-255 +- error: array of 4 codes: [red,green,blue,white], ranging 0-255 +- animation: integer value of 0..3 (SOLID, RUNNING, PULSE, BLINK). Only if device supports it. Example: {"brightness":20,"idle":[240,210,17,0],"hotword":[173,17,240,0]} @@ -64,10 +69,9 @@ Restart the device by publishing {"passwordhash":"yourpasswordhash"} to SITEID/r ## Known issues -- Uploading sometimes fails or an error is thrown when the uploading is done. Lower the uploadspeed to fix it - Audio playback with sample rate higher than 44100 can lead to jitter due to network. Recommended is to use a samplerate of 16000 or 22050 -- Audio playback with matrix voice is not good, code needs to resample to 44100. WIP -- Update colors do not work yet +- Audio playback with matrix voice is not good, code needs to resample to 44100. 16000 Mono or Stereo should be fine +- Some settings (like the colors if update via MQTT) do not survive a reboot yet # Adding devices