Skip to content

Commit

Permalink
Merge pull request #17 from untitledaiproject/ethan/spresense
Browse files Browse the repository at this point in the history
Spresense board support
  • Loading branch information
etown authored Feb 9, 2024
2 parents f174b11 + dc7bec1 commit d1e1acb
Show file tree
Hide file tree
Showing 6 changed files with 194 additions and 1 deletion.
21 changes: 21 additions & 0 deletions clients/sony_spresense/firmware/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
## About Sony Spresense

The Sony Spresense is a compact, low-power board designed. It has a Sony CXD5602 microcontroller, a 6-core ARM® Cortex®-M4F processor that operates up to 156MHz.

### Key Features

- **High-Performance CPU:** Multi-core ARM® Cortex®-M4F processors.
- **Low Power Consumption:** Designed for battery-operated IoT devices, so can support extended operational times.
- **High-Quality Image Sensor:** 5M pixel Sony CMOS image sensor for high-quality image capture.
- **Audio Processing:** Includes support for MP3 audio codecs.
- **LTE-M:** Capable of capturing untethered with the LTE extension board.
- **GNSS Receiver:** Integrated GPS for applications requiring location tracking.

### Limitations

- LTE-M throughput is limited.
- Requires soldering a microphone for audio input.

## Getting Started

See the [Sony documentation](https://developer.sony.com/spresense/) for instructions on how to use the board with Arduino the IDE.
105 changes: 105 additions & 0 deletions clients/sony_spresense/firmware/spresense/spresense.ino
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#include <Audio.h>
#include <LTE.h>
#include <queue>
#include <deque>
#include <LTEUdp.h>
#include <SDHCI.h>

AudioClass *theAudio;
const int mic_channel_num = 1;
#define APP_LTE_APN "iot.truphone.com"
#define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4V6)
#define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_NONE)
#define APP_LTE_RAT (LTE_NET_RAT_CATM)

// host configuration
char serverAddress[] = "";
int port = 8001;

LTE lteAccess;
LTEUDP udp;

static void audio_attention_cb(const ErrorAttentionParam *atprm) {
puts("Attention!");

if (atprm->error_code >= AS_ATTENTION_CODE_WARNING) {

theAudio->startRecorder();
}
}

void setup() {
char apn[LTE_NET_APN_MAXLEN] = APP_LTE_APN;
LTENetworkAuthType authtype = APP_LTE_AUTH_TYPE;

Serial.begin(115200);
while (!Serial)
;

Serial.println("Starting LTE client setup.");

Serial.println("=========== APN information ===========");
Serial.print("Access Point Name : ");
Serial.println(apn);
Serial.print("Authentication Type: ");
Serial.println((authtype == LTE_NET_AUTHTYPE_CHAP) ? "CHAP" : (authtype == LTE_NET_AUTHTYPE_NONE) ? "NONE" : "PAP");

while (true) {
if (lteAccess.begin() != LTE_SEARCHING) {
Serial.println("Could not transition to LTE_SEARCHING.");
Serial.println("Please check the status of the LTE board.");
for (;;) {
sleep(1);
}
}

if (lteAccess.attach(APP_LTE_RAT,
apn,
"",
"",
authtype,
APP_LTE_IP_TYPE)
== LTE_READY) {
Serial.println("attach succeeded.");

break;
}
}

if (!udp.begin(port)) {
Serial.println("Failed to start UDP");
while (true)
;
}

Serial.println("UDP started.");

Serial.println("Init Audio Library");
theAudio = AudioClass::getInstance();
theAudio->begin(audio_attention_cb);

Serial.println("Init Audio Recorder");
theAudio->setRecorderMode(AS_SETRECDR_STS_INPUTDEVICE_MIC, 10, 200 * 1024);

uint8_t channel = AS_CHANNEL_MONO;
theAudio->initRecorder(AS_CODECTYPE_MP3, "/mnt/sd0/BIN", AS_SAMPLINGRATE_16000, channel);

theAudio->startRecorder();

Serial.println("Rec start!");
}

void loop() {
static const size_t bufferSize = 4096;
char buffer[bufferSize];
uint32_t readSize;
int err = theAudio->readFrames(buffer, bufferSize, &readSize);

if (readSize > 0) {
if (readSize > 0) {
udp.beginPacket(serverAddress, port);
udp.write(buffer, readSize);
udp.endPacket();
}
}
}
3 changes: 3 additions & 0 deletions untitledai/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,6 @@ conversation_endpointing:
notification:
apn_team_id: ""

udp:
enabled: false

9 changes: 8 additions & 1 deletion untitledai/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ class ConversationEndpointingConfiguration(BaseModel):
class NotificationConfiguration(BaseModel):
apn_team_id: str | None

class UDPConfiguration(BaseModel):
enabled: bool
host: str | None
port: int | None


class Configuration(BaseModel):

@classmethod
Expand Down Expand Up @@ -81,4 +87,5 @@ def load_config_yaml(cls, config_file_path: str) -> 'Configuration':
user: UserConfiguration
database: DatabaseConfiguration
conversation_endpointing: ConversationEndpointingConfiguration
notification: NotificationConfiguration
notification: NotificationConfiguration
udp: UDPConfiguration
8 changes: 8 additions & 0 deletions untitledai/server/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .routes.capture import router as capture_router
from .routes.conversations import router as conversations_router
from .capture_socket import CaptureSocketApp
from .udp_capture_socket import UDPCaptureSocketApp
from ..services import LLMService, ConversationService, NotificationService
from ..database.database import Database
from ..services.stt.asynchronous.async_transcription_service_factory import AsyncTranscriptionServiceFactory
Expand All @@ -27,6 +28,8 @@
from colorama import init, Fore, Style, Back
from fastapi import Depends

from .streaming_capture_handler import StreamingCaptureHandler

logger = logging.getLogger(__name__)

# TODO: How to handle logging configuration?
Expand Down Expand Up @@ -103,6 +106,11 @@ async def startup_event():
# Initialize the database
app.state._app_state.database.init_db()
asyncio.create_task(process_queue(app.state._app_state))
if config.udp.enabled:
loop = asyncio.get_running_loop()
await loop.create_datagram_endpoint(
lambda: UDPCaptureSocketApp(app.state._app_state), local_addr=(config.udp.host, config.udp.port)
)

@app.on_event("shutdown")
async def shutdown_event():
Expand Down
49 changes: 49 additions & 0 deletions untitledai/server/udp_capture_socket.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#
# udp_capture_socket.py
#
# UDP socket handlers for streaming audio capture.
#
# UDP is necessary for some LTE-M boards, such as the Spresense, where bandwidth is limited.
#

import asyncio
import uuid
from .app_state import AppState
from .streaming_capture_handler import StreamingCaptureHandler
import logging

logger = logging.getLogger(__name__)

class UDPCaptureSocketApp(asyncio.DatagramProtocol):
def __init__(self, app_state: AppState, timeout_seconds=5):
self._app_state = app_state
self._transport = None
self._capture_uuid = None
self._timeout_seconds = timeout_seconds
self._timeout_handle = None

def connection_made(self, transport: asyncio.DatagramTransport) -> None:
self._transport = transport
# Create a new capture session with id generated by the server since we don't have a UPD protocol for this
self._capture_uuid = uuid.uuid1().hex
self._app_state.capture_handlers[self._capture_uuid] = StreamingCaptureHandler(
self._app_state, "spresense", self._capture_uuid, "mp3"
)

def datagram_received(self, data: bytes, addr):
if self._timeout_handle:
self._timeout_handle.cancel()
self._timeout_handle = asyncio.get_running_loop().call_later(
self._timeout_seconds, self.connection_timed_out)

asyncio.create_task(self.send_info_to_client(data))

async def send_info_to_client(self, data: bytes):
await self._app_state.capture_handlers[self._capture_uuid].handle_audio_data(data)

def connection_timed_out(self):
if self._capture_uuid not in self._app_state.capture_handlers:
logger.error(f"Capture session not found: {self._capture_uuid}")
return
capture_handler = self._app_state.capture_handlers[self._capture_uuid]
capture_handler.finish_capture_session()

0 comments on commit d1e1acb

Please sign in to comment.