From dd13dd04181646483ffbbf60990574f366c8f968 Mon Sep 17 00:00:00 2001
From: free-x <oroitburd@gmail.com>
Date: Sun, 2 Jun 2024 14:31:16 +0200
Subject: [PATCH 01/37] Add BMP280 Sensor and deprecated PGN 130311 for
 Raymarine

---
 lib/hardware/GwM5Grove.in  |  19 ++++
 lib/iictask/GwBME280.cpp   |  14 ++-
 lib/iictask/GwBMP280.cpp   | 184 +++++++++++++++++++++++++++++++++
 lib/iictask/GwBMP280.h     |   6 ++
 lib/iictask/GwIicSensors.h |  11 +-
 lib/iictask/GwIicTask.cpp  |   2 +
 lib/iictask/config.json    | 201 ++++++++++++++++++++++++++++++++++++-
 lib/iictask/platformio.ini |  13 +++
 platformio.ini             |   3 +-
 9 files changed, 447 insertions(+), 6 deletions(-)
 create mode 100644 lib/iictask/GwBMP280.cpp
 create mode 100644 lib/iictask/GwBMP280.h

diff --git a/lib/hardware/GwM5Grove.in b/lib/hardware/GwM5Grove.in
index 6d9ce21a..75dad36a 100644
--- a/lib/hardware/GwM5Grove.in
+++ b/lib/hardware/GwM5Grove.in
@@ -143,3 +143,22 @@
     #error "both serial devices already in use"
   #endif
 #endif
+
+#GROVE
+#ifdef GWBMP280G1$GS$
+  #ifndef M5_GROOVEIIC$GS$
+    #define M5_GROOVEIIC$GS$
+  #endif
+  GROOVE_IIC(BMP280,$Z$,1)
+  #define _GWBMP280
+#endif
+
+#GROVE
+#ifdef GWBMP280G2$GS$
+  #ifndef M5_GROOVEIIC$GS$
+    #define M5_GROOVEIIC$GS$
+  #endif
+  GROOVE_IIC(BMP280,$Z$,2)
+  #define _GWBMP280
+#endif
+
diff --git a/lib/iictask/GwBME280.cpp b/lib/iictask/GwBME280.cpp
index 12dcc7e8..cc241134 100644
--- a/lib/iictask/GwBME280.cpp
+++ b/lib/iictask/GwBME280.cpp
@@ -68,16 +68,20 @@ class BME280Config : public IICSensorBase{
         if (!device)
             return;
         GwLog *logger = api->getLogger();
+        float pressure = N2kDoubleNA;
+        float temperature = N2kDoubleNA;
+        float humidity = N2kDoubleNA;
+        float computed = N2kDoubleNA;
         if (prAct)
         {
-            float pressure = device->readPressure();
-            float computed = pressure + prOff;
+            pressure = device->readPressure();
+            computed = pressure + prOff;
             LOG_DEBUG(GwLog::DEBUG, "%s measure %2.0fPa, computed %2.0fPa", prefix.c_str(), pressure, computed);
             sendN2kPressure(api, *this, computed, counterId);
         }
         if (tmAct)
         {
-            float temperature = device->readTemperature(); // offset is handled internally
+            temperature = device->readTemperature(); // offset is handled internally
             temperature = CToKelvin(temperature);
             LOG_DEBUG(GwLog::DEBUG, "%s measure temp=%2.1f", prefix.c_str(), temperature);
             sendN2kTemperature(api, *this, temperature, counterId);
@@ -88,6 +92,10 @@ class BME280Config : public IICSensorBase{
             LOG_DEBUG(GwLog::DEBUG, "%s read humid=%02.0f", prefix.c_str(), humidity);
             sendN2kHumidity(api, *this, humidity, counterId);
         }
+        if (tmAct || prAct || (huAct && sensorId == 0x60))
+        {
+            sendN2kEnvironmentalParameters(api, *this, temperature, humidity, computed,counterId);
+        }
     }
     #define CFG280(prefix) \
         CFG_GET(prAct,prefix); \
diff --git a/lib/iictask/GwBMP280.cpp b/lib/iictask/GwBMP280.cpp
new file mode 100644
index 00000000..11878bfd
--- /dev/null
+++ b/lib/iictask/GwBMP280.cpp
@@ -0,0 +1,184 @@
+#include "GwBMP280.h"
+#ifdef _GWIIC
+    #if defined(GWBMP280) || defined(GWBMP28011) || defined(GWBMP28012)|| defined(GWBMP28021)|| defined(GWBMP28022)
+        #define _GWBMP280
+    #else
+        #undef _GWBMP280
+    #endif
+#else
+    #undef _GWBMP280
+    #undef GWBMP280
+    #undef GWBMP28011
+    #undef GWBMP28012
+    #undef GWBMP28021
+    #undef GWBMP28022
+#endif
+#ifdef _GWBMP280
+    #include <Adafruit_BMP280.h>
+#endif
+#ifdef _GWBMP280
+#define TYPE "BMP280"
+#define PRFX1 TYPE "11"
+#define PRFX2 TYPE "12"
+#define PRFX3 TYPE "21"
+#define PRFX4 TYPE "22"
+class BMP280Config : public IICSensorBase{
+    public:
+    bool prAct=true;
+    bool tmAct=true;
+    tN2kTempSource tmSrc=tN2kTempSource::N2kts_InsideTemperature;
+    tN2kPressureSource prSrc=tN2kPressureSource::N2kps_Atmospheric;
+    tN2kHumiditySource huSrc=tN2kHumiditySource::N2khs_Undef;
+    String tmNam="Temperature";
+    String prNam="Pressure";
+    float tmOff=0;
+    float prOff=0;
+    Adafruit_BMP280 *device=nullptr;
+    uint32_t sensorId=-1;
+    BMP280Config(GwApi * api, const String &prfx):SensorBase(TYPE,api,prfx){
+        }
+    virtual bool isActive(){return prAct||tmAct;}
+    virtual bool initDevice(GwApi *api,TwoWire *wire){
+        GwLog *logger=api->getLogger();
+	//Wire.begin(GWIIC_SDA,GWIIC_SCL);        
+        device= new Adafruit_BMP280(wire);
+        if (! device->begin(addr)){
+            LOG_DEBUG(GwLog::ERROR,"unable to initialize %s at %d",prefix.c_str(),addr);
+            delete device;
+            device=nullptr;
+            return false;
+        }
+        sensorId=device->sensorID();
+        LOG_DEBUG(GwLog::LOG, "initialized %s at %d, sensorId 0x%x", prefix.c_str(), addr, sensorId);
+        return (sensorId == 0x56 || sensorId == 0x57 || sensorId == 0x58)?true:false;
+    }
+    virtual bool preinit(GwApi * api){
+        GwLog *logger=api->getLogger();
+        LOG_DEBUG(GwLog::LOG,"%s configured",prefix.c_str());
+        api->addCapability(prefix,"true");
+        addPressureXdr(api,*this);
+        addTempXdr(api,*this);
+        return isActive();
+    }
+    virtual void measure(GwApi *api, TwoWire *wire, int counterId)
+    {
+        if (!device)
+            return;
+        GwLog *logger = api->getLogger();
+        float pressure = N2kDoubleNA;
+        float temperature = N2kDoubleNA;
+        float humidity = N2kDoubleNA; 
+        float computed = N2kDoubleNA;
+        if (prAct)
+        {
+            pressure = device->readPressure();
+            computed = pressure + prOff;
+            LOG_DEBUG(GwLog::DEBUG, "%s measure %2.0fPa, computed %2.0fPa", prefix.c_str(), pressure, computed);
+            sendN2kPressure(api, *this, computed, counterId);
+        }
+        if (tmAct)
+        {
+            temperature = device->readTemperature(); // offset is handled internally
+            temperature = CToKelvin(temperature);
+            LOG_DEBUG(GwLog::DEBUG, "%s measure temp=%2.1f", prefix.c_str(), temperature);
+            sendN2kTemperature(api, *this, temperature, counterId);
+        }
+        if (tmAct || prAct )
+        {
+            sendN2kEnvironmentalParameters(api, *this, temperature, humidity, computed,counterId);
+        }
+    }
+    #define CFGBMP280(prefix) \
+        CFG_GET(prAct,prefix); \
+        CFG_GET(tmAct,prefix);\
+        CFG_GET(tmSrc,prefix);\
+        CFG_GET(iid,prefix);\
+        CFG_GET(intv,prefix);\
+        CFG_GET(tmNam,prefix);\
+        CFG_GET(prNam,prefix);\
+        CFG_GET(tmOff,prefix);\
+        CFG_GET(prOff,prefix);
+
+    virtual void readConfig(GwConfigHandler *cfg) override
+    {
+        if (prefix == PRFX1)
+        {
+            busId = 1;
+            addr = 0x76;
+            CFGBMP280(BMP28011);
+            ok=true;
+        }
+        if (prefix == PRFX2)
+        {
+            busId = 1;
+            addr = 0x77;
+            CFGBMP280(BMP28012);
+            ok=true;
+        }
+        if (prefix == PRFX3)
+        {
+            busId = 2;
+            addr = 0x76;
+            CFGBMP280(BMP28021);
+            ok=true;
+        }
+        if (prefix == PRFX4)
+        {
+            busId = 2;
+            addr = 0x77;
+            CFGBMP280(BMP28022);
+        }
+        intv *= 1000;
+    }
+};
+
+static IICSensorBase::Creator creator([](GwApi *api, const String &prfx){
+    return new BMP280Config(api,prfx);
+});
+IICSensorBase::Creator registerBMP280(GwApi *api,IICSensorList &sensors){
+    #if defined(GWBMP280) || defined(GWBMP28011)
+    {   
+        auto *cfg=creator(api,PRFX1);
+        //BMP280Config *cfg=new BMP280Config(api,PRFX1);
+        sensors.add(api,cfg);
+        CHECK_IIC1();
+        #pragma message "GWBMP28011 defined"
+    }
+    #endif
+    #if defined(GWBMP28012)
+    {
+        auto *cfg=creator(api,PRFX2);
+        //BMP280Config *cfg=new BMP280Config(api,PRFX2);
+        sensors.add(api,cfg);
+        CHECK_IIC1();
+        #pragma message "GWBMP28012 defined"
+    }
+    #endif
+    #if defined(GWBMP28021)
+    {
+        auto *cfg=creator(api,PRFX3);
+        //BMP280Config *cfg=new BMP280Config(api,PRFX3);
+        sensors.add(api,cfg);
+        CHECK_IIC2();
+        #pragma message "GWBMP28021 defined"
+    }
+    #endif
+    #if defined(GWBMP28022)
+    {
+        auto *cfg=creator(api,PRFX4);
+        //BMP280Config *cfg=new BMP280Config(api,PRFX4);
+        sensors.add(api,cfg);
+        CHECK_IIC1();
+        #pragma message "GWBMP28022 defined"
+    }
+    #endif
+    return creator;
+}
+#else
+IICSensorBase::Creator registerBMP280(GwApi *api,IICSensorList &sensors){
+   return IICSensorBase::Creator();
+}
+
+#endif
+
+
diff --git a/lib/iictask/GwBMP280.h b/lib/iictask/GwBMP280.h
new file mode 100644
index 00000000..97fd4fe1
--- /dev/null
+++ b/lib/iictask/GwBMP280.h
@@ -0,0 +1,6 @@
+#ifndef _GWBMP280_H
+#define _GWBMP280_H
+#include "GwIicSensors.h"
+IICSensorBase::Creator registerBMP280(GwApi *api,IICSensorList &sensors);
+#endif
+
diff --git a/lib/iictask/GwIicSensors.h b/lib/iictask/GwIicSensors.h
index dcb7f3ff..bade5d0b 100644
--- a/lib/iictask/GwIicSensors.h
+++ b/lib/iictask/GwIicSensors.h
@@ -102,6 +102,15 @@ void sendN2kTemperature(GwApi *api,CFG &cfg,double value, int counterId){
     api->increment(counterId,cfg.prefix+String("temp"));
 }
 
+template <class CFG>
+void sendN2kEnvironmentalParameters(GwApi *api,CFG &cfg,double tmValue, double huValue, double prValue, int counterId){
+    tN2kMsg msg;
+    SetN2kEnvironmentalParameters(msg,1,cfg.tmSrc,tmValue,cfg.huSrc,huValue,prValue);
+    api->sendN2kMessage(msg);
+    api->increment(counterId,cfg.prefix+String("hum"));
+    api->increment(counterId,cfg.prefix+String("press"));
+    api->increment(counterId,cfg.prefix+String("temp"));
+}
 
 #ifndef _GWI_IIC1
     #define CHECK_IIC1() checkDef(GWIIC_SCL,GWIIC_SDA)
@@ -114,4 +123,4 @@ void sendN2kTemperature(GwApi *api,CFG &cfg,double value, int counterId){
     #define CHECK_IIC2()
 #endif
 
-#endif
\ No newline at end of file
+#endif
diff --git a/lib/iictask/GwIicTask.cpp b/lib/iictask/GwIicTask.cpp
index fee671f2..22a7167f 100644
--- a/lib/iictask/GwIicTask.cpp
+++ b/lib/iictask/GwIicTask.cpp
@@ -21,6 +21,7 @@ static std::vector<IICGrove> iicGroveList;
 #include "GwIicSensors.h"
 #include "GwHardware.h"
 #include "GwBME280.h"
+#include "GwBMP280.h"
 #include "GwQMP6988.h"
 #include "GwSHT3X.h"
 #include <map>
@@ -92,6 +93,7 @@ void initIicTask(GwApi *api){
     creators.push_back(registerSHT3X(api,sensors));
     creators.push_back(registerQMP6988(api,sensors));
     creators.push_back(registerBME280(api,sensors));
+    creators.push_back(registerBMP280(api,sensors));
     #ifdef _GWI_IIC1
         addGroveItems(creators,api,sensors,"1",_GWI_IIC1);    
     #endif
diff --git a/lib/iictask/config.json b/lib/iictask/config.json
index ec7d6130..54da57e7 100644
--- a/lib/iictask/config.json
+++ b/lib/iictask/config.json
@@ -530,5 +530,204 @@
                 }
             }
         ]
+    },
+    {
+        "type": "array",
+        "name": "BMP280",
+        "replace": [
+            {
+                "b": "1",
+                "i": "11",
+                "n": "93"
+            },
+            {
+                "b": "1",
+                "i": "12",
+                "n": "92"
+            },
+            {
+                "b": "2",
+                "i": "21",
+                "n": "103"
+            },
+            {
+                "b": "2",
+                "i": "22",
+                "n": "102"
+            }
+        ],
+        "children": [
+            {
+                "name": "BMP280$itmAct",
+                "label": "BMP280-$i Temp",
+                "type": "boolean",
+                "default": "true",
+                "description": "Enable the $i. I2C BMP280 temp sensor (bus $b)",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$itmSrc",
+                "label": "BMP280-$i Temp Type",
+                "type": "list",
+                "default": "2",
+                "description": "the NMEA2000 source type for the temperature",
+                "list": [
+                    {
+                        "l": "SeaTemperature",
+                        "v": "0"
+                    },
+                    {
+                        "l": "OutsideTemperature",
+                        "v": "1"
+                    },
+                    {
+                        "l": "InsideTemperature",
+                        "v": "2"
+                    },
+                    {
+                        "l": "EngineRoomTemperature",
+                        "v": "3"
+                    },
+                    {
+                        "l": "MainCabinTemperature",
+                        "v": "4"
+                    },
+                    {
+                        "l": "LiveWellTemperature",
+                        "v": "5"
+                    },
+                    {
+                        "l": "BaitWellTemperature",
+                        "v": "6"
+                    },
+                    {
+                        "l": "RefridgerationTemperature",
+                        "v": "7"
+                    },
+                    {
+                        "l": "HeatingSystemTemperature",
+                        "v": "8"
+                    },
+                    {
+                        "l": "DewPointTemperature",
+                        "v": "9"
+                    },
+                    {
+                        "l": "ApparentWindChillTemperature",
+                        "v": "10"
+                    },
+                    {
+                        "l": "TheoreticalWindChillTemperature",
+                        "v": "11"
+                    },
+                    {
+                        "l": "HeatIndexTemperature",
+                        "v": "12"
+                    },
+                    {
+                        "l": "FreezerTemperature",
+                        "v": "13"
+                    },
+                    {
+                        "l": "ExhaustGasTemperature",
+                        "v": "14"
+                    },
+                    {
+                        "l": "ShaftSealTemperature",
+                        "v": "15"
+                    }
+                ],
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$itmOff",
+                "label": "BMP280-$i Temperature Offset",
+                "type": "number",
+                "description": "offset (in °) to be added to the BMP280 temperature measurements",
+                "default": "0",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$iprAct",
+                "label": "BMP280-$i Pressure",
+                "type": "boolean",
+                "default": "true",
+                "description": "Enable the $i. I2C BMP280 pressure sensor (bus $b)",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$iprOff",
+                "label": "BMP280 Pressure Offset",
+                "type": "number",
+                "description": "offset (in pa) to be added to the BMP280 pressure measurements",
+                "default": "0",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$iiid",
+                "label": "BMP280-$i N2K iid",
+                "type": "number",
+                "default": "$n",
+                "description": "the N2K instance id for the BMP280 Temperature",
+                "category": "iicsensors$b",
+                "min": 0,
+                "max": 253,
+                "check": "checkMinMax",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$iintv",
+                "label": "BMP280-$i Interval",
+                "type": "number",
+                "default": 2,
+                "description": "Interval(s) to query BME280 Temperature (1...300)",
+                "category": "iicsensors$b",
+                "min": 1,
+                "max": 300,
+                "check": "checkMinMax",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$itmNam",
+                "label": "BMP280-$i Temp XDR",
+                "type": "String",
+                "default": "BTemp$i",
+                "description": "set the XDR transducer name for the BMP280 Temperature, leave empty to disable NMEA0183 XDR ",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            },
+            {
+                "name": "BMP280$iprNam",
+                "label": "BMP280-$i Pressure XDR",
+                "type": "String",
+                "default": "BPressure$i",
+                "description": "set the XDR transducer name for the BMP280 Pressure, leave empty to disable NMEA0183 XDR",
+                "category": "iicsensors$b",
+                "capabilities": {
+                    "BMP280$i": "true"
+                }
+            }
+        ]
     }
-]
\ No newline at end of file
+]
diff --git a/lib/iictask/platformio.ini b/lib/iictask/platformio.ini
index db82892d..a7857972 100644
--- a/lib/iictask/platformio.ini
+++ b/lib/iictask/platformio.ini
@@ -35,3 +35,16 @@ build_flags=
     -D M5_GROOVEIIC
     -D M5_CAN_KIT
     ${env.build_flags}
+
+[env:m5stack-atom-envbps]
+extends = sensors
+board = m5stack-atom
+lib_deps =
+    ${env.lib_deps}
+    ${sensors.lib_deps}
+build_flags=
+    #-D M5_ENVBPS
+    -D GWBMP280
+    -D M5_GROOVEIIC
+    -D M5_CAN_KIT
+    ${env.build_flags}
diff --git a/platformio.ini b/platformio.ini
index b249ef8c..f774c6bc 100644
--- a/platformio.ini
+++ b/platformio.ini
@@ -60,8 +60,9 @@ lib_deps =
 	Wire
 	SPI
 	adafruit/Adafruit BME280 Library @ 2.2.2
+	adafruit/Adafruit BMP280 Library@^2.6.8
 	adafruit/Adafruit BusIO @ 1.14.5
-    adafruit/Adafruit Unified Sensor @ 1.1.13
+	adafruit/Adafruit Unified Sensor @ 1.1.13
 
 [env:m5stack-atom]
 board = m5stack-atom

From 0ddc0d055db754d2113b3c39070edc6fddec601f Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 1 Nov 2024 11:57:53 +0100
Subject: [PATCH 02/37] add some doc for serial and USB ports

---
 Readme.md         |  1 +
 doc/serial-usb.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 doc/serial-usb.md

diff --git a/Readme.md b/Readme.md
index 40f8d9bc..6de6c6ab 100644
--- a/Readme.md
+++ b/Readme.md
@@ -47,6 +47,7 @@ Hardware
 --------
 The software is prepared to run on different kinds of ESP32 based modules and accessoirs. For some of them prebuild binaries are available that only need to be flashed, others would require to add some definitions of the used PINs and features and to build the binary.
 For the list of hardware set ups refer to [Hardware](doc/Hardware.md).
+For details of the usage of serial devices and the USB connection refer to [Serial and USB](doc/serial-usb.md).
 
 There is a couple of prebuild binaries that you can directly flash to your device. For other combinations of hardware there is an [online build service](doc/BuildService.md) that will allow you to select your hardware and trigger a build.
 
diff --git a/doc/serial-usb.md b/doc/serial-usb.md
new file mode 100644
index 00000000..428e6079
--- /dev/null
+++ b/doc/serial-usb.md
@@ -0,0 +1,51 @@
+Serial and USB
+==============
+The gateway software uses the [arduino layer](https://github.com/espressif/arduino-esp32) on top of the [espressif IDF framework](https://github.com/espressif/esp-idf).
+The handling of serial devices is mainly done by the implementations in the arduino-espressif layer.
+The gateway code adds some buffering on top of this implementation and ensures that normally only full nmea records are sent.
+If the external connection is to slow the gateway will drop complete records.
+All handling of the serial channels is done in the main task of the gateway. 
+
+Serial Devices
+--------------
+THe arduino espressif layer provides the serial devices as [Streams](https://github.com/espressif/arduino-esp32/blob/master/cores/esp32/Stream.h#L48).
+Main implementations are [HardwareSerial](https://github.com/espressif/arduino-esp32/blob/3670e2bf2aca822f2e1225fdb0e0796e490005a8/cores/esp32/HardwareSerial.h#L71) - for the UARTS and [HWCDC](https://github.com/espressif/arduino-esp32/blob/3670e2bf2aca822f2e1225fdb0e0796e490005a8/cores/esp32/HWCDC.h#L43)(C3/S3 only) - for the USB CDC hardware device.
+
+For the github versions: arduino-espressif 3.20009 maps to github tag 2.0.9.
+
+The arduino espressif layer creates a couple of global instances for the serial devices (potentially depending on some defines).
+The important defines for C3/S3 are:
+
+ * ARDUINO_USB_MODE - the gateway always expects this to be 1
+ * ARDUINO_USB_CDC_ON_BOOT - 0 or 1 (CB in the table below)
+
+The created devices for framework 3.20009:
+
+| Device(Variable) | Type(ESP32) | C3/S3 CB=0 | C3/S3 CB=1 |
+| ------------ | ------- | ------ | -----|
+| Serial0 | --- | --- | HardwareSerial(0) |
+| Serial1 | HardwareSerial(1) | HardwareSerial(1) | HardwareSerial(1) |
+| Serial2 | HardwareSerial(2) | HardwareSerial(2) | HardwareSerial(2) |
+| USBSerial | --- | HWCDC | ---- | 
+| Serial | HardwareSerial(0) | HardwareSerial(0) | HWCDC | 
+
+Unfortunately it seems that in newer versions of the framework the devices could change.
+
+The gateway will use the following serial devices:
+
+* USBserial:<br> 
+  For debug output and NMEA as USB connection. If you do not use an S3/C3 with ARDUINO_USB_CDC_ON_BOOT=0 you need to add a <br>_define USBSerial Serial_ somewhere in your build flags or in your task header.<br>
+  Currently the gateway does not support setting the pins for the USB channel (that would be possible in principle only if an external PHY device is used and the USB is connected to a normal UART).
+* Serial1:<br>
+  If you define GWSERIAL_TYPE (1,2,3,4) - see [defines](../lib/hardware/GwHardware.h#23) or GWSERIAL_MODE ("UNI","BI","TX","RX") it will be used for the first serial channel.
+* Serial2:<br>
+  If you define GWSERIAL2_TYPE (1,2,3,4) - see [defines](../lib/hardware/GwHardware.h#23) or GWSERIAL2_MODE ("UNI","BI","TX","RX") it will be used for the second serial channel.
+
+Hints
+-----
+For normal ESP32 chips you need to set <br>_define USBSerial Serial_ <br>and you can use up to 2 serial channels beside the USB channel.
+For C3/S3 chips you can go for 2 options:
+1. set ARDUINO_USB_CDC_ON_BOOT=1: in This case you still need to set<br> _define USBSerial Serial_<br> You can use up to 2 serial channels in the gateway core - but you still have Serial0 available for a third channel (not yet supported by the core - but can be used in your user code)
+2. set ARDUINO_USB_CDC_ON_BOOT=0: in this case USBSerial is already defined as the USB channel. You can use 2 channels in the gateway core and optional you can use Serial in your user code.
+
+If you do not set any of the GWSERIAL* defines (and they are not set by the core hardware definitions) you can freely use Serial1 and / or Serial2 in your user code.

From 56ec7a04068303289b0544d2915e3489a849ac22 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 1 Nov 2024 17:01:44 +0100
Subject: [PATCH 03/37] intermediate: restructure serial handling

---
 lib/channel/GwChannelList.cpp | 13 +++++++------
 lib/channel/GwChannelList.h   | 11 +++--------
 lib/serial/GwSerial.cpp       |  4 ++--
 lib/serial/GwSerial.h         | 22 ++++++++++++++++++++--
 4 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index c736ae28..58b96b5a 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -83,7 +83,7 @@ class GwSerialLog : public GwLogWriter
 };
 
 template<typename T>
-    class SerialWrapper : public GwChannelList::SerialWrapperBase{
+    class SerialWrapper : public GwSerial::SerialWrapperBase{
         private:
         template<class C>
         void beginImpl(C *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){}
@@ -184,7 +184,7 @@ void GwChannelList::addSerial(int id, int rx, int tx, int type){
     }
     LOG_DEBUG(GwLog::ERROR,"invalid serial config with id %d",id);
 }
-void GwChannelList::addSerial(GwChannelList::SerialWrapperBase *stream,int type,int rx,int tx){
+void GwChannelList::addSerial(GwSerial::SerialWrapperBase *stream,int type,int rx,int tx){
     const char *mode=nullptr;
     switch (type)
     {
@@ -207,7 +207,7 @@ void GwChannelList::addSerial(GwChannelList::SerialWrapperBase *stream,int type,
     }
     addSerial(stream,mode,rx,tx);
 }
-void GwChannelList::addSerial(GwChannelList::SerialWrapperBase *serialStream,const String &mode,int rx,int tx){
+void GwChannelList::addSerial(GwSerial::SerialWrapperBase *serialStream,const String &mode,int rx,int tx){
     int id=serialStream->getId();
     for (auto &&it:theChannels){
         if (it->isOwnSource(id)){
@@ -251,7 +251,7 @@ void GwChannelList::addSerial(GwChannelList::SerialWrapperBase *serialStream,con
     LOG_DEBUG(GwLog::DEBUG,"serial set up: mode=%s,rx=%d,canRead=%d,tx=%d,canWrite=%d",
         mode.c_str(),rx,(int)canRead,tx,(int)canWrite);
     serialStream->begin(logger,config->getInt(param->baud,115200),SERIAL_8N1,rx,tx);
-    GwSerial *serial = new GwSerial(logger, serialStream->getStream(), id, canRead);
+    GwSerial *serial = new GwSerial(logger, serialStream, canRead);
     LOG_DEBUG(GwLog::LOG, "starting serial %d ", id);
     GwChannel *channel = new GwChannel(logger, param->name, id);
     channel->setImpl(serial);
@@ -303,8 +303,9 @@ void GwChannelList::begin(bool fallbackSerial){
     GwChannel *channel=NULL;
     //usb
     if (! fallbackSerial){
-        GwSerial *usb=new GwSerial(NULL,&USBSerial,USB_CHANNEL_ID);
-        USBSerial.begin(config->getInt(config->usbBaud));
+        GwSerial::SerialWrapperBase *usbWrapper=new SerialWrapper<decltype(USBSerial)>(&USBSerial,USB_CHANNEL_ID);
+        usbWrapper->begin(NULL,config->getInt(config->usbBaud));
+        GwSerial *usb=new GwSerial(NULL,usbWrapper);
         logger->setWriter(new GwSerialLog(usb,config->getBool(config->usbActisense),getFlushTimeout(USBSerial)));
         logger->prefix="GWSERIAL:";
         channel=new GwChannel(logger,"USB",USB_CHANNEL_ID);
diff --git a/lib/channel/GwChannelList.h b/lib/channel/GwChannelList.h
index 373dc6ae..6ca20f8b 100644
--- a/lib/channel/GwChannelList.h
+++ b/lib/channel/GwChannelList.h
@@ -8,6 +8,7 @@
 #include "GWConfig.h"
 #include "GwJsonDocument.h"
 #include "GwApi.h"
+#include "GwSerial.h"
 #include <HardwareSerial.h>
 
 //NMEA message channels
@@ -23,12 +24,6 @@ class GwSocketServer;
 class GwTcpClient;
 class GwChannelList{
     private:
-    class SerialWrapperBase{
-        public:
-        virtual void begin(GwLog* logger,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1)=0;
-        virtual Stream *getStream()=0;
-        virtual int getId()=0;
-    };
         GwLog *logger;
         GwConfigHandler *config;
         typedef std::vector<GwChannel *> ChannelList;
@@ -36,8 +31,8 @@ class GwChannelList{
         std::map<int,String> modes;
         GwSocketServer *sockets;
         GwTcpClient *client;
-        void addSerial(SerialWrapperBase *stream,const String &mode,int rx,int tx);
-        void addSerial(SerialWrapperBase *stream,int type,int rx,int tx);
+        void addSerial(GwSerial::SerialWrapperBase *stream,const String &mode,int rx,int tx);
+        void addSerial(GwSerial::SerialWrapperBase *stream,int type,int rx,int tx);
     public:
         void addSerial(int id, int rx, int tx, int type);
         GwChannelList(GwLog *logger, GwConfigHandler *config);
diff --git a/lib/serial/GwSerial.cpp b/lib/serial/GwSerial.cpp
index c810e582..28c8d346 100644
--- a/lib/serial/GwSerial.cpp
+++ b/lib/serial/GwSerial.cpp
@@ -40,10 +40,10 @@ class GwSerialStream: public Stream{
 
 
 
-GwSerial::GwSerial(GwLog *logger, Stream *s, int id,bool allowRead):serial(s)
+GwSerial::GwSerial(GwLog *logger, GwSerial::SerialWrapperBase *s, bool allowRead):serial(s)
 {
     LOG_DEBUG(GwLog::DEBUG,"creating GwSerial %p id %d",this,id);
-    this->id=id;
+    this->id=s->getId();
     this->logger = logger;
     String bufName="Ser(";
     bufName+=String(id);
diff --git a/lib/serial/GwSerial.h b/lib/serial/GwSerial.h
index 5298bc61..f50149ef 100644
--- a/lib/serial/GwSerial.h
+++ b/lib/serial/GwSerial.h
@@ -16,11 +16,27 @@ class GwSerial : public GwChannelInterface{
         int id=-1;
         int overflows=0;
         size_t enqueue(const uint8_t *data, size_t len,bool partial=false);
-        Stream *serial;
         bool availableWrite=false; //if this is false we will wait for availabkleWrite until we flush again
     public:
+        class SerialWrapperBase{
+        public:
+        virtual void begin(GwLog* logger,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1)=0;
+        virtual int getId()=0;
+        virtual int available(){return getStream()->available();}
+        size_t readBytes(uint8_t *buffer, size_t length){
+            return getStream()->readBytes(buffer,length);
+        }
+        virtual int availableForWrite(void){
+            return getStream()->availableForWrite();
+        }
+        size_t write(const uint8_t *buffer, size_t size){
+            return getStream()->write(buffer,size);
+        }
+        private:
+        virtual Stream *getStream()=0;
+        };
         static const int bufferSize=200;
-        GwSerial(GwLog *logger,Stream *stream,int id,bool allowRead=true);
+        GwSerial(GwLog *logger,SerialWrapperBase *stream,bool allowRead=true);
         ~GwSerial();
         bool isInitialized();
         virtual size_t sendToClients(const char *buf,int sourceId,bool partial=false);
@@ -30,5 +46,7 @@ class GwSerial : public GwChannelInterface{
         virtual Stream *getStream(bool partialWrites);
         bool getAvailableWrite(){return availableWrite;}
     friend GwSerialStream;
+    private:
+        SerialWrapperBase *serial;
 };
 #endif
\ No newline at end of file

From c6f601377ca3babb437242779fc604a91041744c Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 3 Nov 2024 16:15:52 +0100
Subject: [PATCH 04/37] intermediate,untested: reorganize channel handling

---
 lib/channel/GwChannel.cpp        |  17 +-
 lib/channel/GwChannel.h          |   7 +-
 lib/channel/GwChannelInterface.h |   1 +
 lib/channel/GwChannelList.cpp    | 325 ++++++++++++++-----------------
 lib/channel/GwChannelList.h      |   5 +-
 lib/hardware/GwHardware.h        |   1 +
 lib/serial/GwSerial.cpp          |  38 +++-
 lib/serial/GwSerial.h            |  79 +++++---
 8 files changed, 249 insertions(+), 224 deletions(-)

diff --git a/lib/channel/GwChannel.cpp b/lib/channel/GwChannel.cpp
index 93a2da3f..069b5ae7 100644
--- a/lib/channel/GwChannel.cpp
+++ b/lib/channel/GwChannel.cpp
@@ -218,14 +218,23 @@ void GwChannel::sendActisense(const tN2kMsg &msg, int sourceId){
     if (!enabled || ! impl || ! writeActisense || ! channelStream) return;
     //currently actisense only for channels with a single source id
     //so we can check it here
-    if (isOwnSource(sourceId)) return;
+    if (maxSourceId < 0 && this->sourceId == sourceId) return;
+    if (sourceId >= this->sourceId && sourceId <= maxSourceId) return;
     countOut->add(String(msg.PGN)); 
     msg.SendInActisenseFormat(channelStream);
 }
 
-bool GwChannel::isOwnSource(int id){
-    if (maxSourceId < 0) return id == sourceId;
-    else return (id >= sourceId && id <= maxSourceId);
+bool GwChannel::overlaps(const GwChannel *other) const{
+    if (maxSourceId < 0){
+        if (other->maxSourceId < 0) return sourceId == other->sourceId;
+        return (other->sourceId <= sourceId && other->maxSourceId >= sourceId);
+    }
+    if (other->maxSourceId < 0){
+        return other->sourceId >= sourceId && other->sourceId <= maxSourceId;
+    }
+    if (other->maxSourceId < sourceId) return false;
+    if (other->sourceId > maxSourceId) return false;
+    return true;
 }
 
 unsigned long GwChannel::countRx(){
diff --git a/lib/channel/GwChannel.h b/lib/channel/GwChannel.h
index 77af597c..951290b0 100644
--- a/lib/channel/GwChannel.h
+++ b/lib/channel/GwChannel.h
@@ -50,7 +50,7 @@ class GwChannel{
     );
 
     void setImpl(GwChannelInterface *impl);
-    bool isOwnSource(int id);
+    bool overlaps(const GwChannel *) const;
     void enable(bool enabled){
         this->enabled=enabled;
     }
@@ -73,5 +73,10 @@ class GwChannel{
     void sendActisense(const tN2kMsg &msg, int sourceId);
     unsigned long countRx();
     unsigned long countTx();
+    bool isOwnSource(int source){
+        if (maxSourceId < 0) return source == sourceId;
+        return (source >= sourceId && source <= maxSourceId);
+    }
+    String getMode(){return impl->getMode();}
 };
 
diff --git a/lib/channel/GwChannelInterface.h b/lib/channel/GwChannelInterface.h
index f9b076c2..68f519bc 100644
--- a/lib/channel/GwChannelInterface.h
+++ b/lib/channel/GwChannelInterface.h
@@ -6,4 +6,5 @@ class GwChannelInterface{
         virtual void readMessages(GwMessageFetcher *writer)=0;
         virtual size_t sendToClients(const char *buffer, int sourceId, bool partial=false)=0;
         virtual Stream * getStream(bool partialWrites){ return NULL;}
+        virtual String getMode(){return "UNKNOWN";}
 };
\ No newline at end of file
diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index 58b96b5a..c09966c8 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -18,16 +18,53 @@ class SerInit{
 };
 std::vector<SerInit> serialInits;
 
+static int typeFromMode(const char *mode){
+    if (strcmp(mode,"UNI") == 0) return GWSERIAL_TYPE_UNI;
+    if (strcmp(mode,"BI") == 0) return GWSERIAL_TYPE_BI;
+    if (strcmp(mode,"RX") == 0) return GWSERIAL_TYPE_RX;
+    if (strcmp(mode,"TX") == 0) return GWSERIAL_TYPE_TX;
+    return GWSERIAL_TYPE_UNK;
+}
+
 #define CFG_SERIAL(ser,...) \
     __MSG("serial config " #ser); \
     static GwInitializer<SerInit> __serial ## ser ## _init \
         (serialInits,SerInit(ser,__VA_ARGS__));
 #ifdef _GWI_SERIAL1
-    CFG_SERIAL(1,_GWI_SERIAL1)
+    CFG_SERIAL(0,_GWI_SERIAL1)
 #endif
 #ifdef _GWI_SERIAL2
-    CFG_SERIAL(2,_GWI_SERIAL2)
+    CFG_SERIAL(1,_GWI_SERIAL2)
 #endif
+//handle separate defines
+    //serial 1
+    #ifndef GWSERIAL_TX
+      #define GWSERIAL_TX -1
+    #endif
+    #ifndef GWSERIAL_RX
+      #define GWSERIAL_RX -1
+    #endif
+    #ifdef GWSERIAL_TYPE
+        CFG_SERIAL(0,GWSERIAL_RX,GWSERIAL_TX,GWSERIAL_TYPE)
+    #else
+        #ifdef GWSERIAL_MODE
+            CFG_SERIAL(0,GWSERIAL_RX,GWSERIAL_TX,typeFromMode(GWSERIAL_MODE))
+        #endif
+    #endif
+    //serial 2
+    #ifndef GWSERIAL2_TX
+      #define GWSERIAL2_TX -1
+    #endif
+    #ifndef GWSERIAL2_RX
+      #define GWSERIAL2_RX -1
+    #endif
+    #ifdef GWSERIAL2_TYPE
+        CFG_SERIAL(1,GWSERIAL2_RX,GWSERIAL2_TX,GWSERIAL2_TYPE)
+    #else
+        #ifdef GWSERIAL2_MODE
+            CFG_SERIAL(1,GWSERIAL2_RX,GWSERIAL2_TX,typeFromMode(GWSERIAL2_MODE))
+        #endif
+    #endif
 class GwSerialLog : public GwLogWriter
 {
     static const size_t bufferSize = 4096;
@@ -35,13 +72,11 @@ class GwSerialLog : public GwLogWriter
     int wp = 0;
     GwSerial *writer;
     bool disabled = false;
-    long flushTimeout=200;
 public:
-    GwSerialLog(GwSerial *writer, bool disabled,long flushTimeout=200)
+    GwSerialLog(GwSerial *writer, bool disabled)
     {
         this->writer = writer;
         this->disabled = disabled;
-        this->flushTimeout=flushTimeout;
         logBuffer = new char[bufferSize];
         wp = 0;
     }
@@ -64,7 +99,7 @@ class GwSerialLog : public GwLogWriter
         {
             while (handled < wp)
             {
-                if ( !writer->flush(flushTimeout)) break;
+                if ( !writer->flush()) break;
                 size_t rt = writer->sendToClients(logBuffer + handled, -1, true);
                 handled += rt;
             }
@@ -82,44 +117,6 @@ class GwSerialLog : public GwLogWriter
     }
 };
 
-template<typename T>
-    class SerialWrapper : public GwSerial::SerialWrapperBase{
-        private:
-        template<class C>
-        void beginImpl(C *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){}
-        void beginImpl(HardwareSerial *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){
-            s->begin(baud,config,rxPin,txPin);
-        }
-        template<class C>
-        void setError(C* s, GwLog *logger){}
-        void setError(HardwareSerial *s,GwLog *logger){
-            LOG_DEBUG(GwLog::LOG,"enable serial errors for channel %d",id);
-            s->onReceiveError([logger,this](hardwareSerial_error_t err){
-                LOG_DEBUG(GwLog::ERROR,"serial error on id %d: %d",this->id,(int)err);
-            });
-        }
-        #if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
-            void beginImpl(HWCDC *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){
-            s->begin(baud);
-        }
-        #endif
-        T *serial;
-        int id;
-        public:
-        SerialWrapper(T* s,int i):serial(s),id(i){}
-        virtual void begin(GwLog* logger,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1) override{
-            beginImpl(serial,baud,config,rxPin,txPin);
-            setError(serial,logger);
-        };
-        virtual Stream *getStream() override{
-            return serial;
-        }
-        virtual int getId() override{
-            return id;
-        }
-
-    };
-
 
 GwChannelList::GwChannelList(GwLog *logger, GwConfigHandler *config){
     this->logger=logger;
@@ -139,10 +136,27 @@ typedef struct {
     const char *toN2K;
     const char *readF;
     const char *writeF;
+    const char *preventLog;
+    const char *readAct;
+    const char *writeAct;
     const char *name;
 } SerialParam;
 
 static  SerialParam serialParameters[]={
+    {
+        .id=USB_CHANNEL_ID,
+        .baud=GwConfigDefinitions::usbBaud,
+        .receive=GwConfigDefinitions::receiveUsb,
+        .send=GwConfigDefinitions::sendUsb,
+        .direction="",
+        .toN2K=GwConfigDefinitions::usbToN2k,
+        .readF=GwConfigDefinitions::usbReadFilter,
+        .writeF=GwConfigDefinitions::usbWriteFilter,
+        .preventLog=GwConfigDefinitions::usbActisense,
+        .readAct=GwConfigDefinitions::usbActisense,
+        .writeAct=GwConfigDefinitions::usbActSend,
+        .name="USB"
+    },
     {
         .id=SERIAL1_CHANNEL_ID,
         .baud=GwConfigDefinitions::serialBaud,
@@ -152,6 +166,9 @@ static  SerialParam serialParameters[]={
         .toN2K=GwConfigDefinitions::serialToN2k,
         .readF=GwConfigDefinitions::serialReadF,
         .writeF=GwConfigDefinitions::serialWriteF,
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
         .name="Serial"
     },
     {
@@ -163,81 +180,38 @@ static  SerialParam serialParameters[]={
         .toN2K=GwConfigDefinitions::serial2ToN2k,
         .readF=GwConfigDefinitions::serial2ReadF,
         .writeF=GwConfigDefinitions::serial2WriteF,
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
         .name="Serial2"
     }
 };
 
-static SerialParam *getSerialParam(int id){
-    for (size_t idx=0;idx<sizeof(serialParameters)/sizeof(SerialParam*);idx++){
-        if (serialParameters[idx].id == id) return &serialParameters[idx];
-    }
-    return nullptr;
-}
-void GwChannelList::addSerial(int id, int rx, int tx, int type){
-    if (id == 1){
-        addSerial(new SerialWrapper<decltype(Serial1)>(&Serial1,SERIAL1_CHANNEL_ID),type,rx,tx);
-        return;   
-    }
-    if (id == 2){
-        addSerial(new SerialWrapper<decltype(Serial2)>(&Serial2,SERIAL2_CHANNEL_ID),type,rx,tx);
-        return;   
-    }
-    LOG_DEBUG(GwLog::ERROR,"invalid serial config with id %d",id);
-}
-void GwChannelList::addSerial(GwSerial::SerialWrapperBase *stream,int type,int rx,int tx){
-    const char *mode=nullptr;
-    switch (type)
-    {
-    case GWSERIAL_TYPE_UNI:
-        mode="UNI";
-        break;
-    case GWSERIAL_TYPE_BI:
-        mode="BI";
-        break;
-    case GWSERIAL_TYPE_RX:
-        mode="RX";
-        break;
-    case GWSERIAL_TYPE_TX:
-        mode="TX";
-        break;
-    }
-    if (mode == nullptr) {
-        LOG_DEBUG(GwLog::ERROR,"unknown serial type %d",type);
-        return;
-    }
-    addSerial(stream,mode,rx,tx);
-}
-void GwChannelList::addSerial(GwSerial::SerialWrapperBase *serialStream,const String &mode,int rx,int tx){
-    int id=serialStream->getId();
-    for (auto &&it:theChannels){
-        if (it->isOwnSource(id)){
-            LOG_DEBUG(GwLog::ERROR,"trying to re-add serial id=%d, ignoring",id);
-            return;
-        }
-    }
-    SerialParam *param=getSerialParam(id);
-    if (param == nullptr){
-        logger->logDebug(GwLog::ERROR,"trying to set up an unknown serial channel: %d",id);
-        return;
-    }
-    if (rx < 0 && tx < 0){
-        logger->logDebug(GwLog::ERROR,"useless config for serial %d: both rx/tx undefined");
-        return;
-    }
-    modes[id]=String(mode);
+template<typename T>
+GwSerial* createSerial(GwLog *logger, T* s,int id, bool canRead=true){
+    return new GwSerialImpl<T>(logger,s,id,canRead);
+} 
+
+static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, int idx,int type,int rx,int tx, bool setLog=false){
+    if (idx < 0 || idx >= sizeof(serialParameters)/sizeof(SerialParam*)) return nullptr;
+    SerialParam *param=&serialParameters[idx];
     bool canRead=false;
     bool canWrite=false;
-    if (mode == "BI"){
+    bool validType=false;
+    if (type == GWSERIAL_TYPE_BI){
         canRead=config->getBool(param->receive);
         canWrite=config->getBool(param->send);
+        validType=true;
     }
-    if (mode == "TX"){
+    if (type == GWSERIAL_TYPE_TX){
         canWrite=true;
+        validType=true;
     }
-    if (mode == "RX"){
+    if (type == GWSERIAL_TYPE_RX){
         canRead=true;
+        validType=true;
     }
-    if (mode == "UNI"){
+    if (type == GWSERIAL_TYPE_UNI ){
         String cfgMode=config->getString(param->direction);
         if (cfgMode == "receive"){
             canRead=true;
@@ -245,16 +219,41 @@ void GwChannelList::addSerial(GwSerial::SerialWrapperBase *serialStream,const St
         if (cfgMode == "send"){
             canWrite=true;
         }
+        validType=true;
+    }
+    if (! validType){
+        LOG_DEBUG(GwLog::ERROR,"invalid type for serial channel %d: %d",param->id,type);
+        return nullptr;
     }
     if (rx < 0) canRead=false;
     if (tx < 0) canWrite=false;
-    LOG_DEBUG(GwLog::DEBUG,"serial set up: mode=%s,rx=%d,canRead=%d,tx=%d,canWrite=%d",
-        mode.c_str(),rx,(int)canRead,tx,(int)canWrite);
-    serialStream->begin(logger,config->getInt(param->baud,115200),SERIAL_8N1,rx,tx);
-    GwSerial *serial = new GwSerial(logger, serialStream, canRead);
-    LOG_DEBUG(GwLog::LOG, "starting serial %d ", id);
-    GwChannel *channel = new GwChannel(logger, param->name, id);
-    channel->setImpl(serial);
+    LOG_DEBUG(GwLog::DEBUG,"serial set up: type=%d,rx=%d,canRead=%d,tx=%d,canWrite=%d",
+        type,rx,(int)canRead,tx,(int)canWrite);
+    GwSerial *serialStream=nullptr;
+    GwLog *streamLog=setLog?nullptr:logger;
+    switch(param->id){
+        case USB_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&USBSerial,param->id);
+            break;
+        case SERIAL1_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&Serial1,param->id);
+            break;
+        case SERIAL2_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&Serial2,param->id);
+            break;
+    }
+    if (serialStream == nullptr){
+        LOG_DEBUG(GwLog::ERROR,"invalid serial config with id %d",param->id);
+        return nullptr;
+    }
+    serialStream->begin(config->getInt(param->baud,115200),SERIAL_8N1,rx,tx);
+    if (setLog){
+        logger->setWriter(new GwSerialLog(serialStream,config->getBool(param->preventLog,false)));
+        logger->prefix="GWSERIAL:";
+    }
+    LOG_DEBUG(GwLog::LOG, "starting serial %d ", param->id);
+    GwChannel *channel = new GwChannel(logger, param->name,param->id);
+    channel->setImpl(serialStream);
     channel->begin(
         canRead || canWrite,
         canWrite,
@@ -263,9 +262,20 @@ void GwChannelList::addSerial(GwSerial::SerialWrapperBase *serialStream,const St
         config->getString(param->writeF),
         false,
         config->getBool(param->toN2K),
-        false,
-        false);
-    LOG_DEBUG(GwLog::LOG, "%s", channel->toString().c_str());
+        config->getBool(param->readAct),
+        config->getBool(param->writeAct));
+    return channel;
+}
+void GwChannelList::addChannel(GwChannel * channel){
+    for (auto &&it:theChannels){
+        if (it->overlaps(channel)){
+            LOG_DEBUG(GwLog::ERROR,"trying to add channel with overlapping ids %s (%s), ignoring",
+                channel->toString().c_str(),
+                it->toString().c_str());
+            return;
+        }
+    }
+    LOG_DEBUG(GwLog::LOG, "adding channel %s", channel->toString().c_str());
     theChannels.push_back(channel);
 }
 void GwChannelList::preinit(){
@@ -290,38 +300,20 @@ void GwChannelList::preinit(){
         }
     }
 }
-template<typename S>
-long getFlushTimeout(S &s){
-    return 200;
-}
-template<>
-long getFlushTimeout(HardwareSerial &s){
-    return 2000;
-}
+#ifndef GWUSB_TX
+  #define GWUSB_TX -1
+#endif
+#ifndef GWUSB_RX
+  #define GWUSB_RX -1
+#endif
+
 void GwChannelList::begin(bool fallbackSerial){
     LOG_DEBUG(GwLog::DEBUG,"GwChannelList::begin");
     GwChannel *channel=NULL;
     //usb
     if (! fallbackSerial){
-        GwSerial::SerialWrapperBase *usbWrapper=new SerialWrapper<decltype(USBSerial)>(&USBSerial,USB_CHANNEL_ID);
-        usbWrapper->begin(NULL,config->getInt(config->usbBaud));
-        GwSerial *usb=new GwSerial(NULL,usbWrapper);
-        logger->setWriter(new GwSerialLog(usb,config->getBool(config->usbActisense),getFlushTimeout(USBSerial)));
-        logger->prefix="GWSERIAL:";
-        channel=new GwChannel(logger,"USB",USB_CHANNEL_ID);
-        channel->setImpl(usb);
-        channel->begin(true,
-            config->getBool(config->sendUsb),
-            config->getBool(config->receiveUsb),
-            config->getString(config->usbReadFilter),
-            config->getString(config->usbWriteFilter),
-            false,
-            config->getBool(config->usbToN2k),
-            config->getBool(config->usbActisense),
-            config->getBool(config->usbActSend)
-        );
-        theChannels.push_back(channel);
-        LOG_DEBUG(GwLog::LOG,"%s",channel->toString().c_str());
+        GwChannel *usb=createSerialChannel(config, logger,0,GWSERIAL_TYPE_BI,GWUSB_RX,GWUSB_TX,true);
+        addChannel(usb);
     }
     //TCP server
     sockets=new GwSocketServer(config,logger,MIN_TCP_CHANNEL_ID);
@@ -339,42 +331,13 @@ void GwChannelList::begin(bool fallbackSerial){
         false,
         false
     );
-    LOG_DEBUG(GwLog::LOG,"%s",channel->toString().c_str());
-    theChannels.push_back(channel);
+    addChannel(channel);
 
     //new serial config handling
     for (auto &&init:serialInits){
-        addSerial(init.serial,init.rx,init.tx,init.mode);
+        (logger,init.serial,init.rx,init.tx,init.mode);
     }
-    //handle separate defines
-    //serial 1
-    #ifndef GWSERIAL_TX
-      #define GWSERIAL_TX -1
-    #endif
-    #ifndef GWSERIAL_RX
-      #define GWSERIAL_RX -1
-    #endif
-    #ifdef GWSERIAL_TYPE
-        addSerial(new SerialWrapper<decltype(Serial1)>(&Serial1,SERIAL1_CHANNEL_ID),GWSERIAL_TYPE,GWSERIAL_RX,GWSERIAL_TX);
-    #else
-        #ifdef GWSERIAL_MODE
-            addSerial(new SerialWrapper<decltype(Serial1)>(&Serial1,SERIAL1_CHANNEL_ID),GWSERIAL_MODE,GWSERIAL_RX,GWSERIAL_TX);
-        #endif
-    #endif
-    //serial 2
-    #ifndef GWSERIAL2_TX
-      #define GWSERIAL2_TX -1
-    #endif
-    #ifndef GWSERIAL2_RX
-      #define GWSERIAL2_RX -1
-    #endif
-    #ifdef GWSERIAL2_TYPE
-        addSerial(new SerialWrapper<decltype(Serial2)>(&Serial2,SERIAL2_CHANNEL_ID),GWSERIAL2_TYPE,GWSERIAL2_RX,GWSERIAL2_TX);
-    #else
-        #ifdef GWSERIAL2_MODE
-            addSerial(new SerialWrapper<decltype(Serial2)>(&Serial2,SERIAL2_CHANNEL_ID),GWSERIAL2_MODE,GWSERIAL2_RX,GWSERIAL2_TX);
-        #endif
-    #endif
+    
     //tcp client
     bool tclEnabled=config->getBool(config->tclEnabled);
     channel=new GwChannel(logger,"TCPClient",TCP_CLIENT_CHANNEL_ID);
@@ -398,13 +361,13 @@ void GwChannelList::begin(bool fallbackSerial){
         false,
         false
         );
-    theChannels.push_back(channel);
-    LOG_DEBUG(GwLog::LOG,"%s",channel->toString().c_str());  
+    addChannel(channel);
     logger->flush();
 }
 String GwChannelList::getMode(int id){
-    auto it=modes.find(id);
-    if (it != modes.end()) return it->second;
+    for (auto && c: theChannels){
+        if (c->isOwnSource(id)) return c->getMode();
+    }
     return "UNKNOWN";
 }
 int GwChannelList::getJsonSize(){
@@ -429,8 +392,8 @@ void GwChannelList::toJson(GwJsonDocument &doc){
     });
 }
 GwChannel *GwChannelList::getChannelById(int sourceId){
-    for (auto it=theChannels.begin();it != theChannels.end();it++){
-        if ((*it)->isOwnSource(sourceId)) return *it;
+    for (auto && it: theChannels){
+        if (it->isOwnSource(sourceId)) return it;
     }
     return NULL;
 }
diff --git a/lib/channel/GwChannelList.h b/lib/channel/GwChannelList.h
index 6ca20f8b..ba45c667 100644
--- a/lib/channel/GwChannelList.h
+++ b/lib/channel/GwChannelList.h
@@ -28,13 +28,10 @@ class GwChannelList{
         GwConfigHandler *config;
         typedef std::vector<GwChannel *> ChannelList;
         ChannelList theChannels;
-        std::map<int,String> modes;
         GwSocketServer *sockets;
         GwTcpClient *client;
-        void addSerial(GwSerial::SerialWrapperBase *stream,const String &mode,int rx,int tx);
-        void addSerial(GwSerial::SerialWrapperBase *stream,int type,int rx,int tx);
     public:
-        void addSerial(int id, int rx, int tx, int type);
+        void addChannel(GwChannel *);
         GwChannelList(GwLog *logger, GwConfigHandler *config);
         typedef std::function<void(GwChannel *)> ChannelAction;
         void allChannels(ChannelAction action);
diff --git a/lib/hardware/GwHardware.h b/lib/hardware/GwHardware.h
index 8306409a..0b14ab32 100644
--- a/lib/hardware/GwHardware.h
+++ b/lib/hardware/GwHardware.h
@@ -24,6 +24,7 @@
 #define GWSERIAL_TYPE_BI 2
 #define GWSERIAL_TYPE_RX 3
 #define GWSERIAL_TYPE_TX 4
+#define GWSERIAL_TYPE_UNK 0
 #include <GwConfigItem.h>
 #include <HardwareSerial.h>
 #include "GwAppInfo.h"
diff --git a/lib/serial/GwSerial.cpp b/lib/serial/GwSerial.cpp
index 28c8d346..c0fb06ea 100644
--- a/lib/serial/GwSerial.cpp
+++ b/lib/serial/GwSerial.cpp
@@ -1,4 +1,5 @@
 #include "GwSerial.h"
+#include "GwHardware.h"
 
 class GwSerialStream: public Stream{
     private:
@@ -40,11 +41,13 @@ class GwSerialStream: public Stream{
 
 
 
-GwSerial::GwSerial(GwLog *logger, GwSerial::SerialWrapperBase *s, bool allowRead):serial(s)
+GwSerial::GwSerial(GwLog *logger, Stream * stream,int id,int type,bool allowRead)
 {
     LOG_DEBUG(GwLog::DEBUG,"creating GwSerial %p id %d",this,id);
-    this->id=s->getId();
     this->logger = logger;
+    this->id=id;
+    this->stream=stream;
+    this->type=type;
     String bufName="Ser(";
     bufName+=String(id);
     bufName+=")";
@@ -62,6 +65,20 @@ GwSerial::~GwSerial()
     if (readBuffer) delete readBuffer;
 }
 
+String GwSerial::getMode(){
+    switch (type){
+        case GWSERIAL_TYPE_UNI:
+            return "UNI";
+        case GWSERIAL_TYPE_BI:
+            return "BI";
+        case GWSERIAL_TYPE_RX:
+            return "RX";
+        case GWSERIAL_TYPE_TX:
+            return "TX";
+    }
+    return "UNKNOWN";
+}
+
 bool GwSerial::isInitialized() { return initialized; }
 size_t GwSerial::enqueue(const uint8_t *data, size_t len, bool partial)
 {
@@ -70,9 +87,9 @@ size_t GwSerial::enqueue(const uint8_t *data, size_t len, bool partial)
 }
 GwBuffer::WriteStatus GwSerial::write(){
     if (! isInitialized()) return GwBuffer::ERROR;
-    size_t numWrite=serial->availableForWrite();          
+    size_t numWrite=availableForWrite();          
     size_t rt=buffer->fetchData(numWrite,[](uint8_t *buffer,size_t len, void *p){
-        return ((GwSerial *)p)->serial->write(buffer,len);
+        return ((GwSerial *)p)->stream->write(buffer,len);
     },this);
     if (rt != 0){
         LOG_DEBUG(GwLog::DEBUG+1,"Serial %d write %d",id,rt);
@@ -93,11 +110,11 @@ void GwSerial::loop(bool handleRead,bool handleWrite){
     write();
     if (! isInitialized()) return;
     if (! handleRead) return;
-    size_t available=serial->available();
+    size_t available=stream->available();
     if (! available) return;
     if (allowRead){
         size_t rd=readBuffer->fillData(available,[](uint8_t *buffer, size_t len, void *p)->size_t{
-            return ((GwSerial *)p)->serial->readBytes(buffer,len);
+            return ((GwSerial *)p)->stream->readBytes(buffer,len);
         },this);
         if (rd != 0){
             LOG_DEBUG(GwLog::DEBUG+2,"GwSerial %d read %d bytes",id,rd);
@@ -106,7 +123,7 @@ void GwSerial::loop(bool handleRead,bool handleWrite){
     else{
         uint8_t buffer[10];
         if (available > 10) available=10;
-        serial->readBytes(buffer,available);
+        stream->readBytes(buffer,available);
     }
 }
 void GwSerial::readMessages(GwMessageFetcher *writer){
@@ -115,10 +132,11 @@ void GwSerial::readMessages(GwMessageFetcher *writer){
     writer->handleBuffer(readBuffer);
 }
 
-bool GwSerial::flush(long max){
+bool GwSerial::flush(){
    if (! isInitialized()) return false;
+   long max=getFlushTimeout();
    if (! availableWrite) {
-    if ( serial->availableForWrite() < 1){
+    if ( availableForWrite() < 1){
         return false;
     }
     availableWrite=true;
@@ -128,7 +146,7 @@ bool GwSerial::flush(long max){
         if (write() != GwBuffer::AGAIN) return true;
         vTaskDelay(1);
    }
-   availableWrite=(serial->availableForWrite() > 0);
+   availableWrite=(availableForWrite() > 0);
    return false;
 }
 Stream * GwSerial::getStream(bool partialWrite){
diff --git a/lib/serial/GwSerial.h b/lib/serial/GwSerial.h
index f50149ef..8e37974d 100644
--- a/lib/serial/GwSerial.h
+++ b/lib/serial/GwSerial.h
@@ -6,10 +6,11 @@
 #include "GwChannelInterface.h"
 class GwSerialStream;
 class GwSerial : public GwChannelInterface{
-    private:
+    protected:
         GwBuffer *buffer;
         GwBuffer *readBuffer=NULL;
         GwLog *logger; 
+        Stream *stream;
         bool initialized=false;
         bool allowRead=true;
         GwBuffer::WriteStatus write();
@@ -17,36 +18,66 @@ class GwSerial : public GwChannelInterface{
         int overflows=0;
         size_t enqueue(const uint8_t *data, size_t len,bool partial=false);
         bool availableWrite=false; //if this is false we will wait for availabkleWrite until we flush again
+        virtual long getFlushTimeout(){return 2000;}
+        virtual int availableForWrite()=0;
+        int type=0;
     public:
-        class SerialWrapperBase{
-        public:
-        virtual void begin(GwLog* logger,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1)=0;
-        virtual int getId()=0;
-        virtual int available(){return getStream()->available();}
-        size_t readBytes(uint8_t *buffer, size_t length){
-            return getStream()->readBytes(buffer,length);
-        }
-        virtual int availableForWrite(void){
-            return getStream()->availableForWrite();
-        }
-        size_t write(const uint8_t *buffer, size_t size){
-            return getStream()->write(buffer,size);
-        }
-        private:
-        virtual Stream *getStream()=0;
-        };
-        static const int bufferSize=200;
-        GwSerial(GwLog *logger,SerialWrapperBase *stream,bool allowRead=true);
-        ~GwSerial();
+        GwSerial(GwLog *logger,Stream *stream,int id,int type,bool allowRead=true);
+        virtual ~GwSerial();
         bool isInitialized();
         virtual size_t sendToClients(const char *buf,int sourceId,bool partial=false);
         virtual void loop(bool handleRead=true,bool handleWrite=true);
         virtual void readMessages(GwMessageFetcher *writer);
-        bool flush(long millis=200);
+        bool flush();
         virtual Stream *getStream(bool partialWrites);
         bool getAvailableWrite(){return availableWrite;}
+        virtual void begin(unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1)=0;
+        virtual String getMode() override;
     friend GwSerialStream;
-    private:
-        SerialWrapperBase *serial;
 };
+
+template<typename T>
+    class GwSerialImpl : public GwSerial{
+        private:
+        template<class C>
+        void beginImpl(C *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){}
+        void beginImpl(HardwareSerial *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){
+            s->begin(baud,config,rxPin,txPin);
+        }
+        template<class C>
+        void setError(C* s, GwLog *logger){}
+        void setError(HardwareSerial *s,GwLog *logger){
+            LOG_DEBUG(GwLog::LOG,"enable serial errors for channel %d",id);
+            s->onReceiveError([logger,this](hardwareSerial_error_t err){
+                LOG_DEBUG(GwLog::ERROR,"serial error on id %d: %d",this->id,(int)err);
+            });
+        }
+        #if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+            void beginImpl(HWCDC *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){
+            s->begin(baud);
+        }
+        #endif
+        template<class C>
+        long getFlushTimeoutImpl(const C*){return 2000;}
+        long getFlushTimeoutImpl(HWCDC *){return 200;}
+
+        T *serial;
+        protected:
+        virtual long getFlushTimeout() override{
+            return getFlushTimeoutImpl(serial);
+        }
+        virtual int availableForWrite(){
+            return serial->availableForWrite();
+        }
+        public:
+        GwSerialImpl(GwLog* logger,T* s,int i,int type,bool allowRead=true): GwSerial(logger,s,i,type,allowRead),serial(s){}
+        virtual ~GwSerialImpl(){}
+        virtual void begin(unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1) override{
+            beginImpl(serial,baud,config,rxPin,txPin);
+            setError(serial,logger);
+        };
+
+    };
+
+
 #endif
\ No newline at end of file

From 01d334d598f7a602352af2782745256edd8d21e2 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 3 Nov 2024 17:33:30 +0100
Subject: [PATCH 05/37] only generate index.js, index.css on demand

---
 extra_script.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/extra_script.py b/extra_script.py
index b8b5c97b..ed62c64e 100644
--- a/extra_script.py
+++ b/extra_script.py
@@ -375,14 +375,30 @@ def getLibs():
             rt.append(e)
     return rt
 
+
+
 def joinFiles(target,pattern,dirlist):
-    with gzip.open(target,"wb") as oh:
-        for dir in dirlist:
+    flist=[]
+    for dir in dirlist:
             fn=os.path.join(dir,pattern)
             if os.path.exists(fn):
-                print("adding %s to %s"%(fn,target))
-                with open(fn,"rb") as rh:
-                    shutil.copyfileobj(rh,oh)
+                flist.append(fn)
+    current=False
+    if os.path.exists(target):
+        current=True
+        for f in flist:
+            if not isCurrent(f,target):
+                current=False
+                break
+    if current:
+        print("%s is up to date"%target)
+        return
+    print("creating %s"%target)
+    with gzip.open(target,"wb") as oh:
+        for fn in flist:
+            print("adding %s to %s"%(fn,target))
+            with open(fn,"rb") as rh:
+                shutil.copyfileobj(rh,oh)
     
 
 OWNLIBS=getLibs()+["FS","WiFi"]

From 40c4089f86cd8a575a83c8f8b5f116d75c3bf224 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 3 Nov 2024 17:33:55 +0100
Subject: [PATCH 06/37] restructure channels, USB + serial1 working

---
 lib/channel/GwChannelList.cpp | 210 ++++++++++++++++++----------------
 lib/log/GwLog.h               |   2 +
 2 files changed, 114 insertions(+), 98 deletions(-)

diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index c09966c8..072d95f0 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -18,6 +18,7 @@ class SerInit{
 };
 std::vector<SerInit> serialInits;
 
+
 static int typeFromMode(const char *mode){
     if (strcmp(mode,"UNI") == 0) return GWSERIAL_TYPE_UNI;
     if (strcmp(mode,"BI") == 0) return GWSERIAL_TYPE_BI;
@@ -31,90 +32,94 @@ static int typeFromMode(const char *mode){
     static GwInitializer<SerInit> __serial ## ser ## _init \
         (serialInits,SerInit(ser,__VA_ARGS__));
 #ifdef _GWI_SERIAL1
-    CFG_SERIAL(0,_GWI_SERIAL1)
+    CFG_SERIAL(SERIAL1_CHANNEL_ID,_GWI_SERIAL1)
 #endif
 #ifdef _GWI_SERIAL2
-    CFG_SERIAL(1,_GWI_SERIAL2)
+    CFG_SERIAL(SERIAL2_CHANNEL_ID,_GWI_SERIAL2)
 #endif
-//handle separate defines
-    //serial 1
-    #ifndef GWSERIAL_TX
-      #define GWSERIAL_TX -1
-    #endif
-    #ifndef GWSERIAL_RX
-      #define GWSERIAL_RX -1
-    #endif
-    #ifdef GWSERIAL_TYPE
-        CFG_SERIAL(0,GWSERIAL_RX,GWSERIAL_TX,GWSERIAL_TYPE)
-    #else
-        #ifdef GWSERIAL_MODE
-            CFG_SERIAL(0,GWSERIAL_RX,GWSERIAL_TX,typeFromMode(GWSERIAL_MODE))
-        #endif
-    #endif
-    //serial 2
-    #ifndef GWSERIAL2_TX
-      #define GWSERIAL2_TX -1
-    #endif
-    #ifndef GWSERIAL2_RX
-      #define GWSERIAL2_RX -1
-    #endif
-    #ifdef GWSERIAL2_TYPE
-        CFG_SERIAL(1,GWSERIAL2_RX,GWSERIAL2_TX,GWSERIAL2_TYPE)
-    #else
-        #ifdef GWSERIAL2_MODE
-            CFG_SERIAL(1,GWSERIAL2_RX,GWSERIAL2_TX,typeFromMode(GWSERIAL2_MODE))
-        #endif
-    #endif
-class GwSerialLog : public GwLogWriter
-{
-    static const size_t bufferSize = 4096;
-    char *logBuffer = NULL;
-    int wp = 0;
-    GwSerial *writer;
-    bool disabled = false;
-public:
-    GwSerialLog(GwSerial *writer, bool disabled)
-    {
-        this->writer = writer;
-        this->disabled = disabled;
-        logBuffer = new char[bufferSize];
-        wp = 0;
-    }
-    virtual ~GwSerialLog() {}
-    virtual void write(const char *data)
-    {
-        if (disabled)
-            return;
-        int len = strlen(data);
-        if ((wp + len) >= (bufferSize - 1))
-            return;
-        strncpy(logBuffer + wp, data, len);
-        wp += len;
-        logBuffer[wp] = 0;
-    }
-    virtual void flush()
+    // handle separate defines
+    // serial 1
+#ifndef GWSERIAL_TX
+#define GWSERIAL_TX -1
+#endif
+#ifndef GWSERIAL_RX
+#define GWSERIAL_RX -1
+#endif
+#ifdef GWSERIAL_TYPE
+    CFG_SERIAL(SERIAL1_CHANNEL_ID, GWSERIAL_RX, GWSERIAL_TX, GWSERIAL_TYPE)
+#else
+#ifdef GWSERIAL_MODE
+CFG_SERIAL(SERIAL1_CHANNEL_ID, GWSERIAL_RX, GWSERIAL_TX, typeFromMode(GWSERIAL_MODE))
+#endif
+#endif
+    // serial 2
+#ifndef GWSERIAL2_TX
+#define GWSERIAL2_TX -1
+#endif
+#ifndef GWSERIAL2_RX
+#define GWSERIAL2_RX -1
+#endif
+#ifdef GWSERIAL2_TYPE
+    CFG_SERIAL(SERIAL2_CHANNEL_ID, GWSERIAL2_RX, GWSERIAL2_TX, GWSERIAL2_TYPE)
+#else
+#ifdef GWSERIAL2_MODE
+CFG_SERIAL(SERIAL2_CHANNEL_ID, GWSERIAL2_RX, GWSERIAL2_TX, typeFromMode(GWSERIAL2_MODE))
+#endif
+#endif
+    class GwSerialLog : public GwLogWriter
     {
-        size_t handled = 0;
-        if (!disabled)
+        static const size_t bufferSize = 4096;
+        char *logBuffer = NULL;
+        int wp = 0;
+        GwSerial *writer;
+        bool disabled = false;
+
+    public:
+        GwSerialLog(GwSerial *writer, bool disabled)
+        {
+            this->writer = writer;
+            this->disabled = disabled;
+            logBuffer = new char[bufferSize];
+            wp = 0;
+        }
+        virtual ~GwSerialLog() {}
+        virtual void write(const char *data)
+        {
+            if (disabled)
+                return;
+            int len = strlen(data);
+            if ((wp + len) >= (bufferSize - 1))
+                return;
+            strncpy(logBuffer + wp, data, len);
+            wp += len;
+            logBuffer[wp] = 0;
+        }
+        virtual void flush()
         {
-            while (handled < wp)
+            size_t handled = 0;
+            if (!disabled)
             {
-                if ( !writer->flush()) break;
-                size_t rt = writer->sendToClients(logBuffer + handled, -1, true);
-                handled += rt;
-            }
-            if (handled < wp){
-                if (handled > 0){
-                    memmove(logBuffer,logBuffer+handled,wp-handled);
-                    wp-=handled;
-                    logBuffer[wp]=0;
+                while (handled < wp)
+                {
+                    if (!writer->flush())
+                        break;
+                    size_t rt = writer->sendToClients(logBuffer + handled, -1, true);
+                    handled += rt;
+                }
+                if (handled < wp)
+                {
+                    if (handled > 0)
+                    {
+                        memmove(logBuffer, logBuffer + handled, wp - handled);
+                        wp -= handled;
+                        logBuffer[wp] = 0;
+                    }
+                    return;
                 }
-                return;
             }
+            wp = 0;
+            logBuffer[0] = 0;
         }
-        wp = 0;
-        logBuffer[0] = 0;
-    }
 };
 
 
@@ -192,9 +197,23 @@ GwSerial* createSerial(GwLog *logger, T* s,int id, bool canRead=true){
     return new GwSerialImpl<T>(logger,s,id,canRead);
 } 
 
+static SerialParam * findSerialParam(int id){
+    SerialParam *param=nullptr;
+    for (auto && p: serialParameters){
+        if (id == p.id){
+            param=&p;
+            break;
+        }
+    }
+    return param;
+}
+
 static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, int idx,int type,int rx,int tx, bool setLog=false){
-    if (idx < 0 || idx >= sizeof(serialParameters)/sizeof(SerialParam*)) return nullptr;
-    SerialParam *param=&serialParameters[idx];
+    SerialParam *param=findSerialParam(idx);
+    if (param == nullptr){
+        LOG_DEBUG(GwLog::ERROR,"invalid serial channel id %d",idx);
+        return nullptr;
+    }
     bool canRead=false;
     bool canWrite=false;
     bool validType=false;
@@ -225,9 +244,8 @@ static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, in
         LOG_DEBUG(GwLog::ERROR,"invalid type for serial channel %d: %d",param->id,type);
         return nullptr;
     }
-    if (rx < 0) canRead=false;
-    if (tx < 0) canWrite=false;
-    LOG_DEBUG(GwLog::DEBUG,"serial set up: type=%d,rx=%d,canRead=%d,tx=%d,canWrite=%d",
+    LOG_DEBUG(GwLog::DEBUG,"serial set up: channel=%d, type=%d,rx=%d,canRead=%d,tx=%d,canWrite=%d",
+        idx,
         type,rx,(int)canRead,tx,(int)canWrite);
     GwSerial *serialStream=nullptr;
     GwLog *streamLog=setLog?nullptr:logger;
@@ -275,28 +293,20 @@ void GwChannelList::addChannel(GwChannel * channel){
             return;
         }
     }
-    LOG_DEBUG(GwLog::LOG, "adding channel %s", channel->toString().c_str());
+    LOG_INFO("adding channel %s", channel->toString().c_str());
     theChannels.push_back(channel);
 }
 void GwChannelList::preinit(){
     for (auto &&init:serialInits){
+        LOG_INFO("serial config found for %d",init.serial);
         if (init.fixedBaud >= 0){
-            switch(init.serial){
-              case 1:
-              {
-                LOG_DEBUG(GwLog::DEBUG,"setting fixed baud %d for serial",init.fixedBaud);
-                config->setValue(GwConfigDefinitions::serialBaud,String(init.fixedBaud),GwConfigInterface::READONLY);
-              }
-              break;
-              case 2:
-              {
-                LOG_DEBUG(GwLog::DEBUG,"setting fixed baud %d for serial2",init.fixedBaud);
-                config->setValue(GwConfigDefinitions::serial2Baud,String(init.fixedBaud),GwConfigInterface::READONLY);
-              }
-              break;
-              default:
-                LOG_DEBUG(GwLog::ERROR,"invalid serial definition %d found",init.serial)
+            SerialParam *param=findSerialParam(init.serial);
+            if (! param){
+                LOG_ERROR("invalid serial definition %d found",init.serial)
+                return;
             }
+            LOG_DEBUG(GwLog::DEBUG,"setting fixed baud %d for serial %d",init.fixedBaud,init.serial);
+            config->setValue(param->baud,String(init.fixedBaud),GwConfigInterface::READONLY);
         }
     }
 }
@@ -312,7 +322,7 @@ void GwChannelList::begin(bool fallbackSerial){
     GwChannel *channel=NULL;
     //usb
     if (! fallbackSerial){
-        GwChannel *usb=createSerialChannel(config, logger,0,GWSERIAL_TYPE_BI,GWUSB_RX,GWUSB_TX,true);
+        GwChannel *usb=createSerialChannel(config, logger,USB_CHANNEL_ID,GWSERIAL_TYPE_BI,GWUSB_RX,GWUSB_TX,true);
         addChannel(usb);
     }
     //TCP server
@@ -335,7 +345,11 @@ void GwChannelList::begin(bool fallbackSerial){
 
     //new serial config handling
     for (auto &&init:serialInits){
-        (logger,init.serial,init.rx,init.tx,init.mode);
+        LOG_INFO("creating serial channel %d, rx=%d,tx=%d,type=%d",init.serial,init.rx,init.tx,init.mode);
+        GwChannel *ser=createSerialChannel(config,logger,init.serial,init.mode,init.rx,init.tx);
+        if (ser != nullptr){
+            addChannel(ser);
+        }
     }
     
     //tcp client
diff --git a/lib/log/GwLog.h b/lib/log/GwLog.h
index 4958760e..bdaa9636 100644
--- a/lib/log/GwLog.h
+++ b/lib/log/GwLog.h
@@ -38,5 +38,7 @@ class GwLog{
         long long getRecordCounter(){return recordCounter;}
 };
 #define LOG_DEBUG(level,...){ if (logger != NULL && logger->isActive(level)) logger->logDebug(level,__VA_ARGS__);}
+#define LOG_INFO(...){ if (logger != NULL && logger->isActive(GwLog::LOG)) logger->logDebug(GwLog::LOG,__VA_ARGS__);}
+#define LOG_ERROR(...){ if (logger != NULL && logger->isActive(GwLog::ERROR)) logger->logDebug(GwLog::ERROR,__VA_ARGS__);}
 
 #endif
\ No newline at end of file

From 18b9946b62f8006d5cb482e7bf37cf1a42030920 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Sun, 3 Nov 2024 18:05:14 +0100
Subject: [PATCH 07/37] intermediate: try special handling for USBCDC

---
 lib/serial/GwSerial.h | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/lib/serial/GwSerial.h b/lib/serial/GwSerial.h
index 8e37974d..75f73a81 100644
--- a/lib/serial/GwSerial.h
+++ b/lib/serial/GwSerial.h
@@ -4,6 +4,7 @@
 #include "GwLog.h"
 #include "GwBuffer.h"
 #include "GwChannelInterface.h"
+#include "hal/usb_serial_jtag_ll.h"
 class GwSerialStream;
 class GwSerial : public GwChannelInterface{
     protected:
@@ -39,6 +40,7 @@ class GwSerial : public GwChannelInterface{
 template<typename T>
     class GwSerialImpl : public GwSerial{
         private:
+        unsigned long lastWritable=0;
         template<class C>
         void beginImpl(C *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){}
         void beginImpl(HardwareSerial *s,unsigned long baud, uint32_t config=SERIAL_8N1, int8_t rxPin=-1, int8_t txPin=-1){
@@ -59,7 +61,29 @@ template<typename T>
         #endif
         template<class C>
         long getFlushTimeoutImpl(const C*){return 2000;}
-        long getFlushTimeoutImpl(HWCDC *){return 200;}
+        #if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+            long getFlushTimeoutImpl(HWCDC *){return 200;}
+        #endif
+
+        template<class C>
+        int availableForWrite(C* c){
+            return c->availableForWrite();
+        }
+
+        #if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+        int availableForWrite(HWCDC* c){
+            int rt=c->availableForWrite();
+            if (rt > 0) {
+                lastWritable=millis();
+                return rt;
+            }
+            if (usb_serial_jtag_ll_txfifo_writable() == 1){
+                LOG_INFO("USBserial restart");
+                usb_serial_jtag_ll_ena_intr_mask(USB_SERIAL_JTAG_INTR_SERIAL_IN_EMPTY);
+            }
+            return rt;
+        }
+        #endif
 
         T *serial;
         protected:
@@ -67,7 +91,7 @@ template<typename T>
             return getFlushTimeoutImpl(serial);
         }
         virtual int availableForWrite(){
-            return serial->availableForWrite();
+            return availableForWrite(serial);
         }
         public:
         GwSerialImpl(GwLog* logger,T* s,int i,int type,bool allowRead=true): GwSerial(logger,s,i,type,allowRead),serial(s){}

From e3bab58f7e425d38bbffc0bf97c28b7107e6475d Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 10:00:05 +0100
Subject: [PATCH 08/37] update to platform 6.8.1

---
 platformio.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/platformio.ini b/platformio.ini
index 2ed6e045..00c9321c 100644
--- a/platformio.ini
+++ b/platformio.ini
@@ -30,10 +30,10 @@ lib_deps =
 	Update
 
 [env]
-platform = espressif32 @ 6.3.2
+platform = espressif32 @ 6.8.1
 framework = arduino
 ;platform_packages=
-;	framework-arduinoespressif32 @ 3.20011.230801
+;	framework-arduinoespressif32 @ 3.20017.0
 ;	framework-espidf @ 3.50101.0
 lib_deps = 
 	${basedeps.lib_deps}

From 041b550ae9724088fde48698c075f869307fbca4 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 10:00:49 +0100
Subject: [PATCH 09/37] #81: restart HWCDC interrupt after 100ms

---
 lib/serial/GwSerial.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/lib/serial/GwSerial.h b/lib/serial/GwSerial.h
index 75f73a81..83d98a6b 100644
--- a/lib/serial/GwSerial.h
+++ b/lib/serial/GwSerial.h
@@ -4,7 +4,8 @@
 #include "GwLog.h"
 #include "GwBuffer.h"
 #include "GwChannelInterface.h"
-#include "hal/usb_serial_jtag_ll.h"
+
+#define USBCDC_RESTART_TIME 100
 class GwSerialStream;
 class GwSerial : public GwChannelInterface{
     protected:
@@ -71,15 +72,24 @@ template<typename T>
         }
 
         #if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+        /**
+         * issue #81
+         * workaround for the HWCDC beeing stuck at some point in time
+         * with availableForWrite == 0 but the ISR being disabled
+         * we simply give a small delay of 100ms for availableForWrite being 0
+         * and afterwards call isConnected that seems to retrigger the ISR
+        */
         int availableForWrite(HWCDC* c){
             int rt=c->availableForWrite();
             if (rt > 0) {
                 lastWritable=millis();
                 return rt;
             }
-            if (usb_serial_jtag_ll_txfifo_writable() == 1){
-                LOG_INFO("USBserial restart");
-                usb_serial_jtag_ll_ena_intr_mask(USB_SERIAL_JTAG_INTR_SERIAL_IN_EMPTY);
+            unsigned long now=millis();
+            if (now > (lastWritable+USBCDC_RESTART_TIME)){
+                lastWritable=now;
+                LOG_ERROR("***Restart USBCDC***");
+                c->isConnected(); //this seems to retrigger the ISR
             }
             return rt;
         }

From 1d5577a777cbd24ee5755a8ceb911a046d8ad50f Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 10:32:52 +0100
Subject: [PATCH 10/37] #81: directly enable the ISR on HWCDC restart if
 connected

---
 lib/serial/GwSerial.h | 11 ++++++++---
 tools/log.pl          | 28 ++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100755 tools/log.pl

diff --git a/lib/serial/GwSerial.h b/lib/serial/GwSerial.h
index 83d98a6b..f3712ab7 100644
--- a/lib/serial/GwSerial.h
+++ b/lib/serial/GwSerial.h
@@ -4,6 +4,9 @@
 #include "GwLog.h"
 #include "GwBuffer.h"
 #include "GwChannelInterface.h"
+#if CONFIG_IDF_TARGET_ESP32C3 || CONFIG_IDF_TARGET_ESP32S3
+  #include "hal/usb_serial_jtag_ll.h"
+#endif
 
 #define USBCDC_RESTART_TIME 100
 class GwSerialStream;
@@ -77,7 +80,7 @@ template<typename T>
          * workaround for the HWCDC beeing stuck at some point in time
          * with availableForWrite == 0 but the ISR being disabled
          * we simply give a small delay of 100ms for availableForWrite being 0
-         * and afterwards call isConnected that seems to retrigger the ISR
+         * and afterwards retrigger the ISR
         */
         int availableForWrite(HWCDC* c){
             int rt=c->availableForWrite();
@@ -88,8 +91,10 @@ template<typename T>
             unsigned long now=millis();
             if (now > (lastWritable+USBCDC_RESTART_TIME)){
                 lastWritable=now;
-                LOG_ERROR("***Restart USBCDC***");
-                c->isConnected(); //this seems to retrigger the ISR
+                if (c->isConnected()){
+                    //this retriggers the ISR
+                    usb_serial_jtag_ll_ena_intr_mask(USB_SERIAL_JTAG_INTR_SERIAL_IN_EMPTY);
+                }
             }
             return rt;
         }
diff --git a/tools/log.pl b/tools/log.pl
new file mode 100755
index 00000000..fcc5f5e4
--- /dev/null
+++ b/tools/log.pl
@@ -0,0 +1,28 @@
+#! /usr/bin/env perl
+use strict;
+use POSIX qw(strftime);
+my ($dev,$speed)=@ARGV;
+if (not defined $dev){
+    die "usage: $0 dev"
+}
+if (! -e $dev) {
+    die "$dev not found"
+}
+open(my $fh,"<",$dev) or die "unable to open $dev";
+if (defined $speed){
+    print("setting speed $speed");
+    system("stty speed $speed < $dev") == 0 or die "unable to set speed";    
+}
+my $last=0;
+while (<$fh>){
+    my $x=time();
+    if ($last != 0){
+        if ($x > ($last+5)){
+            print("****gap***\n");
+        }
+    }
+    printf strftime("%Y/%m/%d-%H%M%S",localtime($x));
+    printf("[%04.2f]: ",$x-$last); 
+    $last=$x;
+    print $_;
+}
\ No newline at end of file

From 5e592dee508c049e23add0b2979e43caa8002b0b Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 10:50:17 +0100
Subject: [PATCH 11/37] set tty in raw mode for log tool

---
 tools/log.pl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/log.pl b/tools/log.pl
index fcc5f5e4..86cb8e29 100755
--- a/tools/log.pl
+++ b/tools/log.pl
@@ -8,11 +8,12 @@
 if (! -e $dev) {
     die "$dev not found"
 }
-open(my $fh,"<",$dev) or die "unable to open $dev";
 if (defined $speed){
     print("setting speed $speed");
     system("stty speed $speed < $dev") == 0 or die "unable to set speed";    
 }
+system("stty raw < $dev") == 0 or die "unable to set raw mode for $dev";
+open(my $fh,"<",$dev) or die "unable to open $dev";
 my $last=0;
 while (<$fh>){
     my $x=time();

From d0dee367f8da43f0114190928711208737f0922f Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 18:52:47 +0100
Subject: [PATCH 12/37] unify channel handling

---
 lib/channel/GwChannel.h       |   1 +
 lib/channel/GwChannelList.cpp | 233 ++++++++++++++++++++--------------
 2 files changed, 138 insertions(+), 96 deletions(-)

diff --git a/lib/channel/GwChannel.h b/lib/channel/GwChannel.h
index 951290b0..66fb4ae6 100644
--- a/lib/channel/GwChannel.h
+++ b/lib/channel/GwChannel.h
@@ -78,5 +78,6 @@ class GwChannel{
         return (source >= sourceId && source <= maxSourceId);
     }
     String getMode(){return impl->getMode();}
+    int getMinId(){return sourceId;};
 };
 
diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index 072d95f0..56f45983 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -144,10 +144,14 @@ typedef struct {
     const char *preventLog;
     const char *readAct;
     const char *writeAct;
+    const char *sendSeasmart;
     const char *name;
-} SerialParam;
+    int maxId;
+    size_t rxstatus;
+    size_t txstatus;
+} ChannelParam;
 
-static  SerialParam serialParameters[]={
+static  ChannelParam channelParameters[]={
     {
         .id=USB_CHANNEL_ID,
         .baud=GwConfigDefinitions::usbBaud,
@@ -160,7 +164,11 @@ static  SerialParam serialParameters[]={
         .preventLog=GwConfigDefinitions::usbActisense,
         .readAct=GwConfigDefinitions::usbActisense,
         .writeAct=GwConfigDefinitions::usbActSend,
-        .name="USB"
+        .sendSeasmart="",
+        .name="USB",
+        .maxId=-1,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::usbRx),
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::usbTx)
     },
     {
         .id=SERIAL1_CHANNEL_ID,
@@ -174,7 +182,11 @@ static  SerialParam serialParameters[]={
         .preventLog="",
         .readAct="",
         .writeAct="",
-        .name="Serial"
+        .sendSeasmart="",
+        .name="Serial",
+        .maxId=-1,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::serRx),
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::serTx)
     },
     {
         .id=SERIAL2_CHANNEL_ID,
@@ -188,8 +200,49 @@ static  SerialParam serialParameters[]={
         .preventLog="",
         .readAct="",
         .writeAct="",
-        .name="Serial2"
+        .sendSeasmart="",
+        .name="Serial2",
+        .maxId=-1,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::ser2Rx),
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::ser2Tx)
+    },
+    {
+        .id=MIN_TCP_CHANNEL_ID,
+        .baud="",
+        .receive=GwConfigDefinitions::readTCP,
+        .send=GwConfigDefinitions::sendTCP,
+        .direction="",
+        .toN2K=GwConfigDefinitions::tcpToN2k,
+        .readF=GwConfigDefinitions::tcpReadFilter,
+        .writeF=GwConfigDefinitions::tcpWriteFilter,
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
+        .sendSeasmart=GwConfigDefinitions::sendSeasmart,
+        .name="TCPServer",
+        .maxId=MIN_TCP_CHANNEL_ID+10,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::tcpSerRx),
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::tcpSerTx)
+    },
+    {
+        .id=TCP_CLIENT_CHANNEL_ID,
+        .baud="",
+        .receive=GwConfigDefinitions::readTCL,
+        .send=GwConfigDefinitions::sendTCL,
+        .direction="",
+        .toN2K=GwConfigDefinitions::tclToN2k,
+        .readF=GwConfigDefinitions::tclReadFilter,
+        .writeF=GwConfigDefinitions::tclWriteFilter,
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
+        .sendSeasmart=GwConfigDefinitions::tclSeasmart,
+        .name="TCPClient",
+        .maxId=-1,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::tcpClRx),
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::tcpClTx)
     }
+
 };
 
 template<typename T>
@@ -197,9 +250,9 @@ GwSerial* createSerial(GwLog *logger, T* s,int id, bool canRead=true){
     return new GwSerialImpl<T>(logger,s,id,canRead);
 } 
 
-static SerialParam * findSerialParam(int id){
-    SerialParam *param=nullptr;
-    for (auto && p: serialParameters){
+static ChannelParam * findChannelParam(int id){
+    ChannelParam *param=nullptr;
+    for (auto && p: channelParameters){
         if (id == p.id){
             param=&p;
             break;
@@ -208,12 +261,44 @@ static SerialParam * findSerialParam(int id){
     return param;
 }
 
-static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, int idx,int type,int rx,int tx, bool setLog=false){
-    SerialParam *param=findSerialParam(idx);
+static GwSerial * createSerialImpl(GwConfigHandler *config,GwLog *logger, int idx,int rx,int tx, bool setLog=false){
+    LOG_DEBUG(GwLog::DEBUG,"create serial: channel=%d, rx=%d,tx=%d",
+        idx,rx,tx);
+    ChannelParam *param=findChannelParam(idx);
     if (param == nullptr){
         LOG_DEBUG(GwLog::ERROR,"invalid serial channel id %d",idx);
         return nullptr;
     }
+    GwSerial *serialStream=nullptr;
+    GwLog *streamLog=setLog?nullptr:logger;
+    switch(param->id){
+        case USB_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&USBSerial,param->id);
+            break;
+        case SERIAL1_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&Serial1,param->id);
+            break;
+        case SERIAL2_CHANNEL_ID:
+            serialStream=createSerial(streamLog,&Serial2,param->id);
+            break;
+    }
+    if (serialStream == nullptr){
+        LOG_DEBUG(GwLog::ERROR,"invalid serial config with id %d",param->id);
+        return nullptr;
+    }
+    serialStream->begin(config->getInt(param->baud,115200),SERIAL_8N1,rx,tx);
+    if (setLog){
+        logger->setWriter(new GwSerialLog(serialStream,config->getBool(param->preventLog,false)));
+        logger->prefix="GWSERIAL:";
+    }
+    return serialStream;
+}
+static GwChannel * createChannel(GwLog *logger, GwConfigHandler *config, int id,GwChannelInterface *impl, int type=GWSERIAL_TYPE_BI){
+    ChannelParam *param=findChannelParam(id);
+    if (param == nullptr){
+        LOG_DEBUG(GwLog::ERROR,"invalid channel id %d",id);
+        return nullptr;
+    }
     bool canRead=false;
     bool canWrite=false;
     bool validType=false;
@@ -241,37 +326,11 @@ static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, in
         validType=true;
     }
     if (! validType){
-        LOG_DEBUG(GwLog::ERROR,"invalid type for serial channel %d: %d",param->id,type);
+        LOG_DEBUG(GwLog::ERROR,"invalid type for channel %d: %d",param->id,type);
         return nullptr;
     }
-    LOG_DEBUG(GwLog::DEBUG,"serial set up: channel=%d, type=%d,rx=%d,canRead=%d,tx=%d,canWrite=%d",
-        idx,
-        type,rx,(int)canRead,tx,(int)canWrite);
-    GwSerial *serialStream=nullptr;
-    GwLog *streamLog=setLog?nullptr:logger;
-    switch(param->id){
-        case USB_CHANNEL_ID:
-            serialStream=createSerial(streamLog,&USBSerial,param->id);
-            break;
-        case SERIAL1_CHANNEL_ID:
-            serialStream=createSerial(streamLog,&Serial1,param->id);
-            break;
-        case SERIAL2_CHANNEL_ID:
-            serialStream=createSerial(streamLog,&Serial2,param->id);
-            break;
-    }
-    if (serialStream == nullptr){
-        LOG_DEBUG(GwLog::ERROR,"invalid serial config with id %d",param->id);
-        return nullptr;
-    }
-    serialStream->begin(config->getInt(param->baud,115200),SERIAL_8N1,rx,tx);
-    if (setLog){
-        logger->setWriter(new GwSerialLog(serialStream,config->getBool(param->preventLog,false)));
-        logger->prefix="GWSERIAL:";
-    }
-    LOG_DEBUG(GwLog::LOG, "starting serial %d ", param->id);
-    GwChannel *channel = new GwChannel(logger, param->name,param->id);
-    channel->setImpl(serialStream);
+    GwChannel *channel = new GwChannel(logger, param->name,param->id,param->maxId);
+    channel->setImpl(impl);
     channel->begin(
         canRead || canWrite,
         canWrite,
@@ -282,9 +341,11 @@ static GwChannel * createSerialChannel(GwConfigHandler *config,GwLog *logger, in
         config->getBool(param->toN2K),
         config->getBool(param->readAct),
         config->getBool(param->writeAct));
+    LOG_INFO("created channel %s",channel->toString().c_str());
     return channel;
 }
 void GwChannelList::addChannel(GwChannel * channel){
+    if (channel == nullptr) return;
     for (auto &&it:theChannels){
         if (it->overlaps(channel)){
             LOG_DEBUG(GwLog::ERROR,"trying to add channel with overlapping ids %s (%s), ignoring",
@@ -300,7 +361,7 @@ void GwChannelList::preinit(){
     for (auto &&init:serialInits){
         LOG_INFO("serial config found for %d",init.serial);
         if (init.fixedBaud >= 0){
-            SerialParam *param=findSerialParam(init.serial);
+            ChannelParam *param=findChannelParam(init.serial);
             if (! param){
                 LOG_ERROR("invalid serial definition %d found",init.serial)
                 return;
@@ -322,39 +383,39 @@ void GwChannelList::begin(bool fallbackSerial){
     GwChannel *channel=NULL;
     //usb
     if (! fallbackSerial){
-        GwChannel *usb=createSerialChannel(config, logger,USB_CHANNEL_ID,GWSERIAL_TYPE_BI,GWUSB_RX,GWUSB_TX,true);
-        addChannel(usb);
+        GwSerial *usbSerial=createSerialImpl(config, logger,USB_CHANNEL_ID,GWUSB_RX,GWUSB_TX,true);
+        if (usbSerial != nullptr){
+            GwChannel *usbChannel=createChannel(logger,config,USB_CHANNEL_ID,usbSerial,GWSERIAL_TYPE_BI);
+            if (usbChannel != nullptr){
+                addChannel(usbChannel);
+            }
+            else{
+                delete usbSerial;
+            }
+        }
     }
     //TCP server
     sockets=new GwSocketServer(config,logger,MIN_TCP_CHANNEL_ID);
     sockets->begin();
-    channel=new GwChannel(logger,"TCPserver",MIN_TCP_CHANNEL_ID,MIN_TCP_CHANNEL_ID+10);
-    channel->setImpl(sockets);
-    channel->begin(
-        true,
-        config->getBool(config->sendTCP),
-        config->getBool(config->readTCP),
-        config->getString(config->tcpReadFilter),
-        config->getString(config->tcpWriteFilter),
-        config->getBool(config->sendSeasmart),
-        config->getBool(config->tcpToN2k),
-        false,
-        false
-    );
-    addChannel(channel);
+    addChannel(createChannel(logger,config,MIN_TCP_CHANNEL_ID,sockets));
 
     //new serial config handling
     for (auto &&init:serialInits){
         LOG_INFO("creating serial channel %d, rx=%d,tx=%d,type=%d",init.serial,init.rx,init.tx,init.mode);
-        GwChannel *ser=createSerialChannel(config,logger,init.serial,init.mode,init.rx,init.tx);
+        GwSerial *ser=createSerialImpl(config,logger,init.serial,init.rx,init.tx);
         if (ser != nullptr){
-            addChannel(ser);
+            channel=createChannel(logger,config,init.serial,ser,init.mode);
+            if (channel != nullptr){
+                addChannel(channel);
+            }
+            else{
+                delete ser;
+            }
         }
     }
     
     //tcp client
     bool tclEnabled=config->getBool(config->tclEnabled);
-    channel=new GwChannel(logger,"TCPClient",TCP_CLIENT_CHANNEL_ID);
     if (tclEnabled){
         client=new GwTcpClient(logger);
         client->begin(TCP_CLIENT_CHANNEL_ID,
@@ -362,20 +423,8 @@ void GwChannelList::begin(bool fallbackSerial){
             config->getInt(config->remotePort),
             config->getBool(config->readTCL)
         );
-        channel->setImpl(client);
     }
-    channel->begin(
-        tclEnabled,
-        config->getBool(config->sendTCL),
-        config->getBool(config->readTCL),
-        config->getString(config->tclReadFilter),
-        config->getString(config->tclReadFilter),
-        config->getBool(config->tclSeasmart),
-        config->getBool(config->tclToN2k),
-        false,
-        false
-        );
-    addChannel(channel);
+    addChannel(createChannel(logger,config,TCP_CLIENT_CHANNEL_ID,client));
     logger->flush();
 }
 String GwChannelList::getMode(int id){
@@ -412,30 +461,22 @@ GwChannel *GwChannelList::getChannelById(int sourceId){
     return NULL;
 }
 
+/**
+ * slightly tricky generic setter for the API status
+ * we expect all values to be unsigned long
+ * the offsets are always offsetof(GwApi::Status,GwApi::Status::xxx)
+*/
+static void setStatus(GwApi::Status *status,size_t offset,unsigned long v){
+    if (offset == 0) return;
+    *((unsigned long *)(((unsigned char *)status)+offset))=v;
+}
+
 void GwChannelList::fillStatus(GwApi::Status &status){
-    GwChannel *channel=getChannelById(USB_CHANNEL_ID);
-    if (channel){
-        status.usbRx=channel->countRx();
-        status.usbTx=channel->countTx();
-    }
-    channel=getChannelById(SERIAL1_CHANNEL_ID);
-    if (channel){
-        status.serRx=channel->countRx();
-        status.serTx=channel->countTx();
-    }
-    channel=getChannelById(SERIAL2_CHANNEL_ID);
-    if (channel){
-        status.ser2Rx=channel->countRx();
-        status.ser2Tx=channel->countTx();
-    }
-    channel=getChannelById(MIN_TCP_CHANNEL_ID);
-    if (channel){
-        status.tcpSerRx=channel->countRx();
-        status.tcpSerTx=channel->countTx();
-    }
-    channel=getChannelById(TCP_CLIENT_CHANNEL_ID);
-    if (channel){
-        status.tcpClRx=channel->countRx();
-        status.tcpClTx=channel->countTx();
+    for (auto && channel: theChannels){
+        ChannelParam *param=findChannelParam(channel->getMinId());
+        if (param != nullptr){
+            setStatus(&status,param->rxstatus,channel->countRx());
+            setStatus(&status,param->txstatus,channel->countTx());
+        }
     }
 }
\ No newline at end of file

From 490a5b9ba1781e9ea4fef40f6e584eac1b62833d Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 4 Nov 2024 20:21:33 +0100
Subject: [PATCH 13/37] #79: add udp writer

---
 lib/api/GwApi.h                  |  1 +
 lib/channel/GwChannelList.cpp    | 26 ++++++++++++++
 lib/channel/GwChannelList.h      |  1 +
 lib/socketserver/GwUdpWriter.cpp | 59 ++++++++++++++++++++++++++++++++
 lib/socketserver/GwUdpWriter.h   | 28 +++++++++++++++
 web/config.json                  | 41 ++++++++++++++++++++++
 6 files changed, 156 insertions(+)
 create mode 100644 lib/socketserver/GwUdpWriter.cpp
 create mode 100644 lib/socketserver/GwUdpWriter.h

diff --git a/lib/api/GwApi.h b/lib/api/GwApi.h
index d4d07166..00dbc1fc 100644
--- a/lib/api/GwApi.h
+++ b/lib/api/GwApi.h
@@ -95,6 +95,7 @@ class GwApi{
                 unsigned long ser2Tx=0;
                 unsigned long tcpSerRx=0;
                 unsigned long tcpSerTx=0;
+                unsigned long udpwTx=0;
                 int tcpClients=0;
                 unsigned long tcpClRx=0;
                 unsigned long tcpClTx=0;
diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index 56f45983..9b68b07d 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -6,6 +6,7 @@
 #include "GwSocketServer.h"
 #include "GwSerial.h"
 #include "GwTcpClient.h"
+#include "GwUdpWriter.h"
 class SerInit{
     public:
         int serial=-1;
@@ -241,6 +242,24 @@ static  ChannelParam channelParameters[]={
         .maxId=-1,
         .rxstatus=offsetof(GwApi::Status,GwApi::Status::tcpClRx),
         .txstatus=offsetof(GwApi::Status,GwApi::Status::tcpClTx)
+    },
+    {
+        .id=UDPW_CHANNEL_ID,
+        .baud="",
+        .receive="",
+        .send=GwConfigDefinitions::udpwEnabled,
+        .direction="",
+        .toN2K="",
+        .readF="",
+        .writeF=GwConfigDefinitions::udpwWriteFilter,
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
+        .sendSeasmart=GwConfigDefinitions::udpwSeasmart,
+        .name="UDPWriter",
+        .maxId=-1,
+        .rxstatus=0,
+        .txstatus=offsetof(GwApi::Status,GwApi::Status::udpwTx)
     }
 
 };
@@ -425,6 +444,13 @@ void GwChannelList::begin(bool fallbackSerial){
         );
     }
     addChannel(createChannel(logger,config,TCP_CLIENT_CHANNEL_ID,client));
+
+    //udp writer
+    if (config->getBool(GwConfigDefinitions::udpwEnabled)){
+        GwUdpWriter *writer=new GwUdpWriter(config,logger,UDPW_CHANNEL_ID);
+        writer->begin();
+        addChannel(createChannel(logger,config,UDPW_CHANNEL_ID,writer));
+    }
     logger->flush();
 }
 String GwChannelList::getMode(int id){
diff --git a/lib/channel/GwChannelList.h b/lib/channel/GwChannelList.h
index ba45c667..956b786f 100644
--- a/lib/channel/GwChannelList.h
+++ b/lib/channel/GwChannelList.h
@@ -18,6 +18,7 @@
 #define SERIAL2_CHANNEL_ID 3
 #define TCP_CLIENT_CHANNEL_ID 4
 #define MIN_TCP_CHANNEL_ID 5
+#define UDPW_CHANNEL_ID 20
 
 #define MIN_USER_TASK 200
 class GwSocketServer;
diff --git a/lib/socketserver/GwUdpWriter.cpp b/lib/socketserver/GwUdpWriter.cpp
new file mode 100644
index 00000000..51a50371
--- /dev/null
+++ b/lib/socketserver/GwUdpWriter.cpp
@@ -0,0 +1,59 @@
+#include "GwUdpWriter.h"
+#include <ESPmDNS.h>
+#include <errno.h>
+#include "GwBuffer.h"
+#include "GwSocketConnection.h"
+#include "GwSocketHelper.h"
+
+GwUdpWriter::GwUdpWriter(const GwConfigHandler *config, GwLog *logger, int minId)
+{
+    this->config = config;
+    this->logger = logger;
+    this->minId = minId;
+}
+
+void GwUdpWriter::begin()
+{
+    fd=socket(AF_INET,SOCK_DGRAM,IPPROTO_IP);
+    if (fd < 0){
+        LOG_ERROR("unable to create udp socket");
+        return;
+    }
+    int enable = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
+    setsockopt(fd, SOL_SOCKET, SO_BROADCAST, &enable, sizeof(int));
+    port=config->getInt(GwConfigDefinitions::udpwPort);
+    //TODO: check port
+    address=config->getString(GwConfigDefinitions::udpwAddress);
+    LOG_INFO("UDP writer created, address=%s, port=%d",
+            address.c_str(),port);
+    inet_pton(AF_INET, address.c_str(), &destination.sin_addr);
+    destination.sin_family = AF_INET;
+    destination.sin_port = htons(port);
+}
+
+void GwUdpWriter::loop(bool handleRead, bool handleWrite)
+{
+    
+}
+
+void GwUdpWriter::readMessages(GwMessageFetcher *writer)
+{
+    
+}
+size_t GwUdpWriter::sendToClients(const char *buf, int source,bool partial)
+{ 
+    if (source == minId) return 0;
+    size_t len=strlen(buf);
+    ssize_t err = sendto(fd,buf,len,0,(struct sockaddr *)&destination, sizeof(destination));
+    if (err < 0){
+        LOG_DEBUG(GwLog::DEBUG,"UDP writer error sending: %d",errno);
+        return 0;
+    }
+    return err;
+}
+
+
+GwUdpWriter::~GwUdpWriter()
+{
+}
\ No newline at end of file
diff --git a/lib/socketserver/GwUdpWriter.h b/lib/socketserver/GwUdpWriter.h
new file mode 100644
index 00000000..e98ea67e
--- /dev/null
+++ b/lib/socketserver/GwUdpWriter.h
@@ -0,0 +1,28 @@
+#ifndef _GWUDPWRITER_H
+#define _GWUDPWRITER_H
+#include "GWConfig.h"
+#include "GwLog.h"
+#include "GwBuffer.h"
+#include "GwChannelInterface.h"
+#include <memory>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+
+class GwUdpWriter: public GwChannelInterface{
+    private:
+        const GwConfigHandler *config;
+        GwLog *logger;
+        int fd=-1;
+        int minId;
+        int port;
+        String address;
+        struct sockaddr_in destination;
+    public:
+        GwUdpWriter(const GwConfigHandler *config,GwLog *logger,int minId);
+        ~GwUdpWriter();
+        void begin();
+        virtual void loop(bool handleRead=true,bool handleWrite=true);
+        virtual size_t sendToClients(const char *buf,int sourceId, bool partialWrite=false);
+        virtual void readMessages(GwMessageFetcher *writer);
+};
+#endif
\ No newline at end of file
diff --git a/web/config.json b/web/config.json
index 5c21eb51..b4336d6d 100644
--- a/web/config.json
+++ b/web/config.json
@@ -826,6 +826,47 @@
         "description": "send NMEA2000 as seasmart to remote TCP server",
         "category": "TCP client"
     },
+    {
+        "name": "udpwEnabled",
+        "label": "enable",
+        "type": "boolean",
+        "default": "false",
+        "description":"enable the UDP writer",
+        "category":"UDP writer"
+    },
+    {
+        "name": "udpwPort",
+        "label": "remote port",
+        "type": "number",
+        "default": "10110",
+        "description": "the UDP port we send to",
+        "category": "UDP writer"
+    },
+    {
+        "name": "udpwAddress",
+        "label": "remote address",
+        "type": "string",
+        "default": "",
+        "check": "checkIpAddress",
+        "description": "the IP address we connect to in the form 192.168.1.2",
+        "category": "UDP writer"
+    },
+    {
+        "name": "udpwWriteFilter",
+        "label": "NMEA write Filter",
+        "type": "filter",
+        "default": "",
+        "description": "filter for NMEA0183 data when writing to remote UDP server\nselect aison|aisoff, set a whitelist or a blacklist with NMEA sentences like RMC,RMB",
+        "category": "UDP writer"
+    },
+    {
+        "name": "udpwSeasmart",
+        "label": "Seasmart out",
+        "type": "boolean",
+        "default": "false",
+        "description": "send NMEA2000 as seasmart to remote UDP server",
+        "category": "UDP writer"
+    },
     {
         "name": "wifiClient",
         "label": "wifi client",

From a5827e24d8481d7ae6114f98309300d26bf67ab0 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 7 Nov 2024 19:47:27 +0100
Subject: [PATCH 14/37] different modes for UDP writer, allow to select network

---
 lib/socketserver/GwSocketHelper.h |   5 +
 lib/socketserver/GwUdpWriter.cpp  | 184 ++++++++++++++++++++++++++----
 lib/socketserver/GwUdpWriter.h    |  51 ++++++++-
 web/config.json                   |  34 +++++-
 4 files changed, 250 insertions(+), 24 deletions(-)

diff --git a/lib/socketserver/GwSocketHelper.h b/lib/socketserver/GwSocketHelper.h
index 8dea5078..67ba3a6f 100644
--- a/lib/socketserver/GwSocketHelper.h
+++ b/lib/socketserver/GwSocketHelper.h
@@ -17,4 +17,9 @@ class GwSocketHelper{
             if (setsockopt(socket, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) != ESP_OK) return false;
             return true;  
         }
+        static bool isMulticast(const String &addr){
+            in_addr iaddr;
+            if (inet_pton(AF_INET,addr.c_str(),&iaddr) != 1) return false;
+            return IN_MULTICAST(ntohl(iaddr.s_addr));
+        }
 };
\ No newline at end of file
diff --git a/lib/socketserver/GwUdpWriter.cpp b/lib/socketserver/GwUdpWriter.cpp
index 51a50371..c91880e2 100644
--- a/lib/socketserver/GwUdpWriter.cpp
+++ b/lib/socketserver/GwUdpWriter.cpp
@@ -4,36 +4,175 @@
 #include "GwBuffer.h"
 #include "GwSocketConnection.h"
 #include "GwSocketHelper.h"
+#include "GWWifi.h"
+
+GwUdpWriter::WriterSocket::WriterSocket(GwLog *l,int p,const String &src,const String &dst, SourceMode sm) :
+     sourceMode(sm), source(src), destination(dst), port(p),logger(l)
+{
+    if (inet_pton(AF_INET, dst.c_str(), &dstA.sin_addr) != 1)
+    {
+        LOG_ERROR("UDPW: invalid destination ip address %s", dst.c_str());
+        return;
+    }
+    if (sourceMode != SourceMode::S_UNBOUND)
+    {
+        if (inet_pton(AF_INET, src.c_str(), &srcA) != 1)
+        {
+            LOG_ERROR("UDPW: invalid source ip address %s", src.c_str());
+            return;
+        }
+    }
+    dstA.sin_family=AF_INET;
+    dstA.sin_port=htons(port);
+    fd=socket(AF_INET,SOCK_DGRAM,IPPROTO_IP);
+    if (fd < 0){
+        LOG_ERROR("UDPW: unable to create udp socket: %d",errno);
+        return;
+    }
+    int enable = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
+    setsockopt(fd, SOL_SOCKET, SO_BROADCAST, &enable, sizeof(int));
+    switch (sourceMode)
+    {
+    case SourceMode::S_SRC:
+    {
+        sockaddr_in bindA;
+        bindA.sin_family = AF_INET;
+        bindA.sin_port = htons(0); // let system select
+        bindA.sin_addr = srcA;
+        if (bind(fd, (struct sockaddr *)&bindA, sizeof(bindA)) != 0)
+        {
+            LOG_ERROR("UDPW: bind failed for address %s: %d", source.c_str(), errno);
+            ::close(fd);
+            fd = -1;
+            return;
+        }
+    }
+    break;
+    case SourceMode::S_MC:
+    {
+        if (setsockopt(fd,IPPROTO_IP,IP_MULTICAST_IF,&srcA,sizeof(srcA)) != 0){
+            LOG_ERROR("UDPW: unable to set MC source %s: %d",source.c_str(),errno);
+            ::close(fd);
+            fd=-1;
+            return;
+        }
+        int loop=0;
+        setsockopt(fd,IPPROTO_IP,IP_MULTICAST_LOOP,&loop,sizeof(loop));   
+    }
+    break;
+    default:
+      //not bound
+      break;
+    }
+}
+bool GwUdpWriter::WriterSocket::changed(const String &newSrc, const String &newDst){
+    if (newDst != destination) return true;
+    if (sourceMode == SourceMode::S_UNBOUND) return false;
+    return newSrc != source;
+}
+size_t GwUdpWriter::WriterSocket::send(const char *buf,size_t len){
+    if (fd < 0) return 0;
+    ssize_t err = sendto(fd,buf,len,0,(struct sockaddr *)&dstA, sizeof(dstA));
+    if (err < 0){
+        LOG_DEBUG(GwLog::DEBUG,"UDPW %s error sending: %d",destination.c_str(), errno);
+        return 0;
+    }
+    return err;
+}
 
 GwUdpWriter::GwUdpWriter(const GwConfigHandler *config, GwLog *logger, int minId)
 {
     this->config = config;
     this->logger = logger;
     this->minId = minId;
+    port=config->getInt(GwConfigDefinitions::udpwPort);
+    
+}
+void GwUdpWriter::checkStaSocket(){
+    String src;
+    String bc;
+    if (type == T_BCAP || type == T_MCAP || type == T_NORM || type == T_UNKNOWN ) return;
+    bool connected=false;
+    if (WiFi.isConnected()){
+        src=WiFi.localIP().toString();
+        bc=WiFi.broadcastIP().toString();
+        connected=true;
+    }
+    else{
+        if (staSocket == nullptr) return;
+    }
+    String dst;
+    WriterSocket::SourceMode sm=WriterSocket::SourceMode::S_SRC;
+    switch (type){
+        case T_BCALL:
+        case T_BCSTA:
+            sm=WriterSocket::SourceMode::S_SRC;
+            dst=bc;
+            break;
+        case T_MCALL:
+        case T_MCSTA:
+            dst=config->getString(GwConfigDefinitions::udpwMC);
+            sm=WriterSocket::SourceMode::S_MC;
+            break;
+
+    }
+    if (staSocket != nullptr)
+    {
+        if (!connected || staSocket->changed(src, dst))
+        {
+            staSocket->close();
+            delete staSocket;
+            staSocket = nullptr;
+            LOG_INFO("changing/stopping UDPW(sta) socket");
+        }
+    }
+    if (staSocket == nullptr && connected)
+    {
+        LOG_INFO("creating new UDP(sta) socket src=%s, dst=%s", src.c_str(), dst.c_str());
+        staSocket = new WriterSocket(logger, port, src, dst, WriterSocket::SourceMode::S_SRC);
+    }
 }
 
 void GwUdpWriter::begin()
 {
-    fd=socket(AF_INET,SOCK_DGRAM,IPPROTO_IP);
-    if (fd < 0){
-        LOG_ERROR("unable to create udp socket");
-        return;
+    if (type != T_UNKNOWN) return; //already started
+    type=(UType)(config->getInt(GwConfigDefinitions::udpwType));
+    LOG_INFO("UDPW begin, mode=%d",(int)type);
+    String src=WiFi.softAPIP().toString();
+    String dst;
+    WriterSocket::SourceMode sm=WriterSocket::SourceMode::S_UNBOUND;
+    bool createApSocket=false;
+    switch(type){
+        case T_BCALL:
+        case T_BCAP:
+            createApSocket=true;
+            dst=WiFi.softAPBroadcastIP().toString();   
+            sm=WriterSocket::SourceMode::S_SRC;
+            break;
+        case T_MCALL:
+        case T_MCAP:
+            createApSocket=true;
+            dst=config->getString(GwConfigDefinitions::udpwMC);   
+            sm=WriterSocket::SourceMode::S_SRC;
+            break;
+        case T_NORM:
+            createApSocket=true;
+            dst=config->getString(GwConfigDefinitions::udpwAddress);   
+            sm=WriterSocket::SourceMode::S_UNBOUND;
     }
-    int enable = 1;
-    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
-    setsockopt(fd, SOL_SOCKET, SO_BROADCAST, &enable, sizeof(int));
-    port=config->getInt(GwConfigDefinitions::udpwPort);
-    //TODO: check port
-    address=config->getString(GwConfigDefinitions::udpwAddress);
-    LOG_INFO("UDP writer created, address=%s, port=%d",
-            address.c_str(),port);
-    inet_pton(AF_INET, address.c_str(), &destination.sin_addr);
-    destination.sin_family = AF_INET;
-    destination.sin_port = htons(port);
+    if (createApSocket){
+        LOG_INFO("creating new UDPW(ap) socket src=%s, dst=%s", src.c_str(), dst.c_str());
+        apSocket=new WriterSocket(logger,port,src,dst,sm);
+    }
+    checkStaSocket();
 }
 
 void GwUdpWriter::loop(bool handleRead, bool handleWrite)
 {
+    if (handleWrite){
+        checkStaSocket();
+    }
     
 }
 
@@ -45,12 +184,17 @@ size_t GwUdpWriter::sendToClients(const char *buf, int source,bool partial)
 { 
     if (source == minId) return 0;
     size_t len=strlen(buf);
-    ssize_t err = sendto(fd,buf,len,0,(struct sockaddr *)&destination, sizeof(destination));
-    if (err < 0){
-        LOG_DEBUG(GwLog::DEBUG,"UDP writer error sending: %d",errno);
-        return 0;
+    bool hasSent=false;
+    size_t res=0;
+    if (apSocket != nullptr){
+        res=apSocket->send(buf,len);
+        if (res > 0) hasSent=true;
     }
-    return err;
+    if (staSocket != nullptr){
+        res=staSocket->send(buf,len);
+        if (res > 0) hasSent=true;
+    }
+    return hasSent?len:0;
 }
 
 
diff --git a/lib/socketserver/GwUdpWriter.h b/lib/socketserver/GwUdpWriter.h
index e98ea67e..e17a17e8 100644
--- a/lib/socketserver/GwUdpWriter.h
+++ b/lib/socketserver/GwUdpWriter.h
@@ -9,14 +9,59 @@
 #include <arpa/inet.h>
 
 class GwUdpWriter: public GwChannelInterface{
+    public:
+    using UType=enum{
+        T_BCALL=0,
+        T_BCAP=1,
+        T_BCSTA=2,
+        T_NORM=3,
+        T_MCALL=4,
+        T_MCAP=5,
+        T_MCSTA=6,
+        T_UNKNOWN=-1
+    };
     private:
+        class WriterSocket{
+            public:
+            int fd=-1;
+            struct in_addr srcA;
+            struct sockaddr_in dstA;
+            String source;
+            String destination;
+            int port;
+            GwLog *logger;
+            using SourceMode=enum {
+                S_UNBOUND=0,
+                S_MC,
+                S_SRC
+            };
+            SourceMode sourceMode;
+            WriterSocket(GwLog *logger,int p,const String &src,const String &dst, SourceMode sm);
+            void close(){
+                if (fd > 0){
+                    ::close(fd);
+                }
+                fd=-1;
+            }
+            ~WriterSocket(){
+                close();
+            }
+            bool changed(const String &newSrc, const String &newDst);
+            size_t send(const char *buf,size_t len);
+        };
         const GwConfigHandler *config;
         GwLog *logger;
-        int fd=-1;
+        /**
+         * we use fd/address to send to the AP network
+         * and fd2,address2 to send to the station network
+         * for type "normal" we only use fd
+        */
+        WriterSocket *apSocket=nullptr; //also for T_NORM
+        WriterSocket *staSocket=nullptr;
         int minId;
         int port;
-        String address;
-        struct sockaddr_in destination;
+        UType type=T_UNKNOWN;
+        void checkStaSocket();
     public:
         GwUdpWriter(const GwConfigHandler *config,GwLog *logger,int minId);
         ~GwUdpWriter();
diff --git a/web/config.json b/web/config.json
index b4336d6d..3e44a4bc 100644
--- a/web/config.json
+++ b/web/config.json
@@ -842,6 +842,23 @@
         "description": "the UDP port we send to",
         "category": "UDP writer"
     },
+    {
+        "name": "udpwType",
+        "label": "remote address type",
+        "type": "list",
+        "default": "0",
+        "description": "to which networks/addresses do we send\nbc-all: send broadcast to AP and wifi client network\nbc-ap: send broadcast to access point only\nbc-cli: send broadcast to wifi client network\nnormal: normal target address\nmc-all: multicast to AP and wifi client network\nmc-ap:multicast to AP network\nmc-cli: muticast to wifi client network",
+        "list":[
+            {"l":"bc-all","v":"0"},
+            {"l":"bc-ap","v":"1"},
+            {"l":"bc-cli","v":"2"},
+            {"l":"normal","v":"3"},
+            {"l":"mc-all","v":"4"},
+            {"l":"mc-ap","v":"5"},
+            {"l":"mc-cli","v":"6"}
+        ],
+        "category": "UDP writer"
+    },
     {
         "name": "udpwAddress",
         "label": "remote address",
@@ -849,7 +866,22 @@
         "default": "",
         "check": "checkIpAddress",
         "description": "the IP address we connect to in the form 192.168.1.2",
-        "category": "UDP writer"
+        "category": "UDP writer",
+        "condition":{
+            "udpwType":["3"]
+        }
+    },
+    {
+        "name": "udpwMC",
+        "label": "multicast address",
+        "type": "string",
+        "default": "224.0.0.1",
+        "check": "checkMCAddress",
+        "description": "the multicast address we send to 224.0.0.0...239.255.255.255",
+        "category": "UDP writer",
+        "condition":{
+            "udpwType":["4","5","6"]
+        }
     },
     {
         "name": "udpwWriteFilter",

From b0d5e27b5a9aac6a3e99c478c44723e4f32326e5 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 7 Nov 2024 20:16:58 +0100
Subject: [PATCH 15/37] make seasmart working again, only show counter for
 enabled directions

---
 lib/channel/GwChannel.cpp     | 16 ++++++++++------
 lib/channel/GwChannelList.cpp |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/channel/GwChannel.cpp b/lib/channel/GwChannel.cpp
index 069b5ae7..9c799832 100644
--- a/lib/channel/GwChannel.cpp
+++ b/lib/channel/GwChannel.cpp
@@ -58,8 +58,6 @@ GwChannel::GwChannel(GwLog *logger,
     this->name=name;
     this->sourceId=sourceId;
     this->maxSourceId=maxSourceId;
-    this->countIn=new GwCounter<String>(String("count")+name+String("in"));
-    this->countOut=new GwCounter<String>(String("count")+name+String("out"));
     this->impl=NULL;
     this->receiver=new GwChannelMessageReceiver(logger,this);
     this->actisenseReader=NULL;
@@ -100,6 +98,12 @@ void GwChannel::begin(
             actisenseReader->SetReadStream(channelStream);         
         }
     }
+    if (nmeaIn || readActisense){
+        this->countIn=new GwCounter<String>(String("count")+name+String("in"));
+    }
+    if (nmeaOut || seaSmartOut || writeActisense){
+        this->countOut=new GwCounter<String>(String("count")+name+String("out"));
+    }
 }
 void GwChannel::setImpl(GwChannelInterface *impl){
     this->impl=impl;
@@ -135,10 +139,10 @@ void GwChannel::updateCounter(const char *msg, bool out)
     }
     if (key[0] == 0) return;
     if (out){
-        countOut->add(key);
+        if (countOut) countOut->add(key);
     }
     else{
-        countIn->add(key);
+        if (countIn) countIn->add(key);
     }
 }
 
@@ -209,7 +213,7 @@ void GwChannel::parseActisense(N2kHandler handler){
     tN2kMsg N2kMsg;
 
     while (actisenseReader->GetMessageFromStream(N2kMsg)) {
-      countIn->add(String(N2kMsg.PGN));
+      if(countIn) countIn->add(String(N2kMsg.PGN));
       handler(N2kMsg,sourceId);
     }
 }
@@ -220,7 +224,7 @@ void GwChannel::sendActisense(const tN2kMsg &msg, int sourceId){
     //so we can check it here
     if (maxSourceId < 0 && this->sourceId == sourceId) return;
     if (sourceId >= this->sourceId && sourceId <= maxSourceId) return;
-    countOut->add(String(msg.PGN)); 
+    if(countOut) countOut->add(String(msg.PGN)); 
     msg.SendInActisenseFormat(channelStream);
 }
 
diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index 9b68b07d..b08bbfc8 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -356,7 +356,7 @@ static GwChannel * createChannel(GwLog *logger, GwConfigHandler *config, int id,
         canRead,
         config->getString(param->readF),
         config->getString(param->writeF),
-        false,
+        config->getBool(param->sendSeasmart),
         config->getBool(param->toN2K),
         config->getBool(param->readAct),
         config->getBool(param->writeAct));

From 82f5e179875e9cfd4bfa1a5fd9f50c496638fe32 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 8 Nov 2024 21:00:25 +0100
Subject: [PATCH 16/37] intermediate: udp reader

---
 lib/api/GwApi.h                   |   1 +
 lib/channel/GwChannelList.cpp     |  26 +++++
 lib/channel/GwChannelList.h       |   1 +
 lib/queue/GwBuffer.cpp            |  12 ++-
 lib/queue/GwBuffer.h              |   5 +-
 lib/socketserver/GwSocketHelper.h |   3 +
 lib/socketserver/GwUdpReader.cpp  | 158 ++++++++++++++++++++++++++++++
 lib/socketserver/GwUdpReader.h    |  45 +++++++++
 web/config.json                   | 143 ++++++++++++++++++++++++---
 web/index.js                      |  36 +++++--
 10 files changed, 406 insertions(+), 24 deletions(-)
 create mode 100644 lib/socketserver/GwUdpReader.cpp
 create mode 100644 lib/socketserver/GwUdpReader.h

diff --git a/lib/api/GwApi.h b/lib/api/GwApi.h
index 00dbc1fc..5a24e927 100644
--- a/lib/api/GwApi.h
+++ b/lib/api/GwApi.h
@@ -96,6 +96,7 @@ class GwApi{
                 unsigned long tcpSerRx=0;
                 unsigned long tcpSerTx=0;
                 unsigned long udpwTx=0;
+                unsigned long udprRx=0;
                 int tcpClients=0;
                 unsigned long tcpClRx=0;
                 unsigned long tcpClTx=0;
diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index b08bbfc8..4371cef0 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -7,6 +7,7 @@
 #include "GwSerial.h"
 #include "GwTcpClient.h"
 #include "GwUdpWriter.h"
+#include "GwUdpReader.h"
 class SerInit{
     public:
         int serial=-1;
@@ -260,8 +261,27 @@ static  ChannelParam channelParameters[]={
         .maxId=-1,
         .rxstatus=0,
         .txstatus=offsetof(GwApi::Status,GwApi::Status::udpwTx)
+    },
+    {
+        .id=UDPR_CHANNEL_ID,
+        .baud="",
+        .receive=GwConfigDefinitions::udprEnabled,
+        .send="",
+        .direction="",
+        .toN2K=GwConfigDefinitions::udprToN2k,
+        .readF=GwConfigDefinitions::udprReadFilter,
+        .writeF="",
+        .preventLog="",
+        .readAct="",
+        .writeAct="",
+        .sendSeasmart="",
+        .name="UDPReader",
+        .maxId=-1,
+        .rxstatus=offsetof(GwApi::Status,GwApi::Status::udprRx),
+        .txstatus=0
     }
 
+
 };
 
 template<typename T>
@@ -451,6 +471,12 @@ void GwChannelList::begin(bool fallbackSerial){
         writer->begin();
         addChannel(createChannel(logger,config,UDPW_CHANNEL_ID,writer));
     }
+    //udp reader
+    if (config->getBool(GwConfigDefinitions::udprEnabled)){
+        GwUdpReader *reader=new GwUdpReader(config,logger,UDPR_CHANNEL_ID);
+        reader->begin();
+        addChannel(createChannel(logger,config,UDPR_CHANNEL_ID,reader));
+    }
     logger->flush();
 }
 String GwChannelList::getMode(int id){
diff --git a/lib/channel/GwChannelList.h b/lib/channel/GwChannelList.h
index 956b786f..f7247167 100644
--- a/lib/channel/GwChannelList.h
+++ b/lib/channel/GwChannelList.h
@@ -19,6 +19,7 @@
 #define TCP_CLIENT_CHANNEL_ID 4
 #define MIN_TCP_CHANNEL_ID 5
 #define UDPW_CHANNEL_ID 20
+#define UDPR_CHANNEL_ID 21
 
 #define MIN_USER_TASK 200
 class GwSocketServer;
diff --git a/lib/queue/GwBuffer.cpp b/lib/queue/GwBuffer.cpp
index 8e129348..de0f0706 100644
--- a/lib/queue/GwBuffer.cpp
+++ b/lib/queue/GwBuffer.cpp
@@ -21,7 +21,7 @@ GwBuffer::~GwBuffer(){
 }
 void GwBuffer::reset(String reason)
 {
-    LOG_DEBUG(GwLog::LOG,"reseting buffer %s, reason %s",this->name.c_str(),reason.c_str());
+    if (! reason.isEmpty())LOG_DEBUG(GwLog::LOG,"reseting buffer %s, reason %s",this->name.c_str(),reason.c_str());
     writePointer = buffer;
     readPointer = buffer;
     lp("reset");
@@ -33,6 +33,16 @@ size_t GwBuffer::freeSpace()
     }
     return readPointer - writePointer - 1;
 }
+size_t GwBuffer::continousSpace() const{
+    if (readPointer <= writePointer){
+        return bufferSize-offset(writePointer);
+    }
+    return readPointer-writePointer-1;
+}
+void GwBuffer::moveWp(size_t offset){
+    if (offset > continousSpace()) return;
+    writePointer+=offset;
+}
 size_t GwBuffer::usedSpace()
 {
     if (readPointer <= writePointer)
diff --git a/lib/queue/GwBuffer.h b/lib/queue/GwBuffer.h
index 17490ec6..aa67e047 100644
--- a/lib/queue/GwBuffer.h
+++ b/lib/queue/GwBuffer.h
@@ -33,7 +33,7 @@ class GwBuffer{
         uint8_t *buffer;
         uint8_t *writePointer;
         uint8_t *readPointer;
-        size_t offset(uint8_t* ptr){
+        size_t offset(uint8_t* ptr) const{
             return (size_t)(ptr-buffer);
         }
         GwLog *logger;
@@ -54,6 +54,9 @@ class GwBuffer{
          * find the first occurance of x in the buffer, -1 if not found
          */
         int findChar(char x);
+        uint8_t *getWp(){return writePointer;}
+        size_t continousSpace() const; //free space from wp
+        void moveWp(size_t offset); //move the wp forward
 };
 
 #endif
\ No newline at end of file
diff --git a/lib/socketserver/GwSocketHelper.h b/lib/socketserver/GwSocketHelper.h
index 67ba3a6f..f153d450 100644
--- a/lib/socketserver/GwSocketHelper.h
+++ b/lib/socketserver/GwSocketHelper.h
@@ -22,4 +22,7 @@ class GwSocketHelper{
             if (inet_pton(AF_INET,addr.c_str(),&iaddr) != 1) return false;
             return IN_MULTICAST(ntohl(iaddr.s_addr));
         }
+        static bool equals(const in_addr &left, const in_addr &right){
+            return left.s_addr == right.s_addr;
+        }
 };
\ No newline at end of file
diff --git a/lib/socketserver/GwUdpReader.cpp b/lib/socketserver/GwUdpReader.cpp
new file mode 100644
index 00000000..e8b619e9
--- /dev/null
+++ b/lib/socketserver/GwUdpReader.cpp
@@ -0,0 +1,158 @@
+#include "GwUdpReader.h"
+#include <ESPmDNS.h>
+#include <errno.h>
+#include "GwBuffer.h"
+#include "GwSocketConnection.h"
+#include "GwSocketHelper.h"
+#include "GWWifi.h"
+
+
+GwUdpReader::GwUdpReader(const GwConfigHandler *config, GwLog *logger, int minId)
+{
+    this->config = config;
+    this->logger = logger;
+    this->minId = minId;
+    port=config->getInt(GwConfigDefinitions::udprPort);
+    buffer= new GwBuffer(logger,GwBuffer::RX_BUFFER_SIZE,"udprd");
+}
+
+void GwUdpReader::createAndBind(){
+    if (fd >= 0){
+        ::close(fd);
+    }
+    if (currentStationIp.isEmpty() && (type == T_STA || type == T_MCSTA)) return;
+    fd=socket(AF_INET,SOCK_DGRAM,IPPROTO_IP);
+    if (fd < 0){
+        LOG_ERROR("UDPR: unable to create udp socket: %d",errno);
+        return;
+    }
+    int enable = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
+    if (type == T_STA)
+    {
+        if (inet_pton(AF_INET, currentStationIp.c_str(), &listenA.sin_addr) != 1)
+        {
+            LOG_ERROR("UDPR: invalid station ip address %s", currentStationIp.c_str());
+            close(fd);
+            fd = -1;
+            return;
+        }
+    }
+    if (bind(fd,(struct sockaddr *)&listenA,sizeof(listenA)) < 0){
+        LOG_ERROR("UDPR: unable to bind: %d",errno);
+        close(fd);
+        fd=-1;
+        return;
+    }
+    LOG_INFO("UDPR: socket created and bound");
+    if (type != T_MCALL && type != T_MCAP && type != T_MCSTA) {
+        return;
+    }
+    struct ip_mreq mc;
+    String mcAddr=config->getString(GwConfigDefinitions::udprMC);
+    if (inet_pton(AF_INET,mcAddr.c_str(),&mc.imr_multiaddr) != 1){
+        LOG_ERROR("UDPR: invalid multicast addr %s",mcAddr.c_str());
+        ::close(fd);
+        fd=-1;
+        return;
+    }
+    if (type == T_MCALL || type == T_MCAP){
+        mc.imr_interface=apAddr;
+        int res=setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP,&mc,sizeof(mc));
+        if (res != 0){
+            LOG_ERROR("UDPR: unable to add MC membership for AP:%d",errno);
+        }
+        else{
+            LOG_INFO("UDPR: membership for %s for AP",mcAddr.c_str());
+        }
+    }
+    if (!currentStationIp.isEmpty() && (type == T_MCALL || type == T_MCSTA))
+    {
+        mc.imr_interface = staAddr;
+        int res = setsockopt(fd, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mc, sizeof(mc));
+        if (res != 0)
+        {
+            LOG_ERROR("UDPR: unable to add MC membership for STA:%d", errno);
+        }
+        else{
+            LOG_INFO("UDPR: membership for %s for STA %s",mcAddr.c_str(),currentStationIp.c_str());
+        }
+    }
+}
+
+void GwUdpReader::begin()
+{
+    if (type != T_UNKNOWN) return; //already started
+    type=(UType)(config->getInt(GwConfigDefinitions::udprType));
+    LOG_INFO("UDPR begin, mode=%d",(int)type);
+    port=config->getInt(GwConfigDefinitions::udprPort);
+    listenA.sin_family=AF_INET;
+    listenA.sin_port=htons(port);
+    if (type != T_STA){
+        listenA.sin_addr.s_addr=htonl(INADDR_ANY);
+    }
+    String ap=WiFi.softAPIP().toString();
+    if (inet_pton(AF_INET, ap.c_str(), &apAddr) != 1)
+    {
+        LOG_ERROR("UDPR: invalid ap ip address %s", ap.c_str());
+        return;
+    }
+    String sta;
+    if (WiFi.isConnected()) sta=WiFi.localIP().toString();
+    setStationAdd(sta);
+    createAndBind();
+}
+
+bool GwUdpReader::setStationAdd(const String &sta){
+    if (sta == currentStationIp) return false;
+    currentStationIp=sta;
+    if (inet_pton(AF_INET, currentStationIp.c_str(), &staAddr) != 1){
+            LOG_ERROR("UDPR: invalid station ip address %s", currentStationIp.c_str());
+            return false;
+    }
+    LOG_INFO("UDPR: new station IP %s",currentStationIp.c_str());
+    return true;
+}
+void GwUdpReader::loop(bool handleRead, bool handleWrite)
+{
+    if (handleRead){
+        if (type == T_STA || type == T_MCALL || type == T_MCSTA){
+            //only change anything if we considered the station IP
+            String nextStationIp;
+            if (WiFi.isConnected()){
+                String nextStationIp=WiFi.localIP().toString();
+            }
+            if (setStationAdd(nextStationIp)){
+                LOG_INFO("UDPR: wifi client IP changed, restart");
+                createAndBind();
+            }
+        }
+    }
+    
+}
+
+void GwUdpReader::readMessages(GwMessageFetcher *writer)
+{
+    if (fd < 0) return;
+    //we expect one NMEA message in one UDP packet
+    buffer->reset();
+    struct sockaddr_in from;
+    socklen_t fromLen=sizeof(from);
+    ssize_t res=recvfrom(fd,buffer->getWp(),buffer->continousSpace(),MSG_DONTWAIT,
+        (struct sockaddr*)&from,&fromLen);
+    if (res <= 0) return;
+    if (GwSocketHelper::equals(from.sin_addr,apAddr)) return;
+    if (!currentStationIp.isEmpty() && (GwSocketHelper::equals(from.sin_addr,staAddr))) return;
+    buffer->moveWp(res);
+    LOG_DEBUG(GwLog::DEBUG,"UDPR: received %d bytes",res);
+    writer->handleBuffer(buffer);   
+}
+size_t GwUdpReader::sendToClients(const char *buf, int source,bool partial)
+{ 
+    return 0;
+}
+
+
+GwUdpReader::~GwUdpReader()
+{
+}
\ No newline at end of file
diff --git a/lib/socketserver/GwUdpReader.h b/lib/socketserver/GwUdpReader.h
new file mode 100644
index 00000000..08c56bbf
--- /dev/null
+++ b/lib/socketserver/GwUdpReader.h
@@ -0,0 +1,45 @@
+#ifndef _GWUDPREADER_H
+#define _GWUDPREADER_H
+#include "GWConfig.h"
+#include "GwLog.h"
+#include "GwBuffer.h"
+#include "GwChannelInterface.h"
+#include <memory>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+
+class GwUdpReader: public GwChannelInterface{
+    public:
+    using UType=enum{
+        T_ALL=0,
+        T_AP=1,
+        T_STA=2,
+        T_MCALL=4,
+        T_MCAP=5,
+        T_MCSTA=6,
+        T_UNKNOWN=-1
+    };
+    private:
+        const GwConfigHandler *config;
+        GwLog *logger;
+        int minId;
+        int port;
+        int fd=-1;
+        struct sockaddr_in listenA;
+        String listenIp;
+        String currentStationIp;
+        struct in_addr apAddr;
+        struct in_addr staAddr;
+        UType type=T_UNKNOWN;
+        void createAndBind();
+        bool setStationAdd(const String &sta);
+        GwBuffer *buffer=nullptr;
+    public:
+        GwUdpReader(const GwConfigHandler *config,GwLog *logger,int minId);
+        ~GwUdpReader();
+        void begin();
+        virtual void loop(bool handleRead=true,bool handleWrite=true);
+        virtual size_t sendToClients(const char *buf,int sourceId, bool partialWrite=false);
+        virtual void readMessages(GwMessageFetcher *writer);
+};
+#endif
\ No newline at end of file
diff --git a/web/config.json b/web/config.json
index 3e44a4bc..0b75d272 100644
--- a/web/config.json
+++ b/web/config.json
@@ -691,6 +691,7 @@
         "label": "TCP port",
         "type": "number",
         "default": "10110",
+        "check":"checkPort",
         "description": "the TCP port we listen on",
         "category": "TCP server"
     },
@@ -766,8 +767,12 @@
         "label": "remote port",
         "type": "number",
         "default": "10110",
+        "check":"checkPort",
         "description": "the TCP port we connect to",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "remoteAddress",
@@ -776,7 +781,10 @@
         "default": "",
         "check": "checkIpAddress",
         "description": "the IP address we connect to in the form 192.168.1.2\nor an MDNS name like ESP32NMEA2K.local",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "sendTCL",
@@ -784,7 +792,10 @@
         "type": "boolean",
         "default": "true",
         "description": "send out NMEA data to remote TCP server",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "readTCL",
@@ -792,7 +803,10 @@
         "type": "boolean",
         "default": "true",
         "description": "receive NMEA data from remote TCP server",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "tclToN2k",
@@ -800,7 +814,10 @@
         "type": "boolean",
         "default": "true",
         "description": "convert NMEA0183 from remote TCP server to NMEA2000",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "tclReadFilter",
@@ -808,7 +825,10 @@
         "type": "filter",
         "default": "",
         "description": "filter for NMEA0183 data when reading from remote TCP server\nselect aison|aisoff, set a whitelist or a blacklist with NMEA sentences like RMC,RMB",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "tclWriteFilter",
@@ -816,7 +836,10 @@
         "type": "filter",
         "default": "",
         "description": "filter for NMEA0183 data when writing to remote TCP server\nselect aison|aisoff, set a whitelist or a blacklist with NMEA sentences like RMC,RMB",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "tclSeasmart",
@@ -824,7 +847,10 @@
         "type": "boolean",
         "default": "false",
         "description": "send NMEA2000 as seasmart to remote TCP server",
-        "category": "TCP client"
+        "category": "TCP client",
+        "condition":{
+            "tclEnabled":"true"
+        }
     },
     {
         "name": "udpwEnabled",
@@ -840,7 +866,11 @@
         "type": "number",
         "default": "10110",
         "description": "the UDP port we send to",
-        "category": "UDP writer"
+        "check":"checkPort",
+        "category": "UDP writer",
+        "condition":{
+            "udpwEnabled":"true"
+        }
     },
     {
         "name": "udpwType",
@@ -857,7 +887,10 @@
             {"l":"mc-ap","v":"5"},
             {"l":"mc-cli","v":"6"}
         ],
-        "category": "UDP writer"
+        "category": "UDP writer",
+        "condition":{
+            "udpwEnabled":"true"
+        }
     },
     {
         "name": "udpwAddress",
@@ -868,7 +901,8 @@
         "description": "the IP address we connect to in the form 192.168.1.2",
         "category": "UDP writer",
         "condition":{
-            "udpwType":["3"]
+            "udpwType":["3"],
+            "udpwEnabled":"true"
         }
     },
     {
@@ -880,7 +914,8 @@
         "description": "the multicast address we send to 224.0.0.0...239.255.255.255",
         "category": "UDP writer",
         "condition":{
-            "udpwType":["4","5","6"]
+            "udpwType":["4","5","6"],
+            "udpwEnabled":"true"
         }
     },
     {
@@ -889,7 +924,10 @@
         "type": "filter",
         "default": "",
         "description": "filter for NMEA0183 data when writing to remote UDP server\nselect aison|aisoff, set a whitelist or a blacklist with NMEA sentences like RMC,RMB",
-        "category": "UDP writer"
+        "category": "UDP writer",
+        "condition":{
+            "udpwEnabled":"true"
+        }
     },
     {
         "name": "udpwSeasmart",
@@ -897,7 +935,84 @@
         "type": "boolean",
         "default": "false",
         "description": "send NMEA2000 as seasmart to remote UDP server",
-        "category": "UDP writer"
+        "category": "UDP writer",
+        "condition":{
+            "udpwEnabled":"true"
+        }
+    },
+    {
+        "name": "udprEnabled",
+        "label": "enable",
+        "type": "boolean",
+        "default": "false",
+        "description":"enable the UDP reader",
+        "category":"UDP reader"
+    },
+    {
+        "name": "udprPort",
+        "label": "local port",
+        "type": "number",
+        "default": "10110",
+        "check":"checkPort",
+        "description": "the UDP port we listen on",
+        "category": "UDP reader",
+        "condition":{
+            "udprEnabled":"true"
+        }
+    },
+    {
+        "name": "udprType",
+        "label": "local address type",
+        "type": "list",
+        "default": "0",
+        "description": "to which networks/addresses do we listen\nall: listen on AP and wifi client network\nap: listen in access point network only\ncli: listen in wifi client network\nmc-all: receive multicast from AP and wifi client network\nmc-ap:receive multicast from AP network\nmc-cli: receive muticast wifi client network",
+        "list":[
+            {"l":"all","v":"0"},
+            {"l":"ap","v":"1"},
+            {"l":"cli","v":"2"},
+            {"l":"mc-all","v":"4"},
+            {"l":"mc-ap","v":"5"},
+            {"l":"mc-cli","v":"6"}
+        ],
+        "category": "UDP reader",
+        "condition":{
+            "udprEnabled":"true"
+        }
+    },
+    {
+        "name": "udprToN2k",
+        "label": "to NMEA2000",
+        "type": "boolean",
+        "default": "true",
+        "description": "convert NMEA0183 from UDP to NMEA2000",
+        "category": "UDP reader",
+        "condition":{
+            "udprEnabled":"true"
+        }
+    },
+    {
+        "name": "udprMC",
+        "label": "multicast address",
+        "type": "string",
+        "default": "224.0.0.1",
+        "check": "checkMCAddress",
+        "description": "the multicast address we listen on 224.0.0.0...239.255.255.255",
+        "category": "UDP reader",
+        "condition":{
+            "udprType":["4","5","6"],
+            "udprEnabled":"true"
+        }
+    },
+    {
+        "name": "udprReadFilter",
+        "label": "NMEA read Filter",
+        "type": "filter",
+        "default": "",
+        "description": "filter for NMEA0183 data when receiving\nselect aison|aisoff, set a whitelist or a blacklist with NMEA sentences like RMC,RMB",
+        "category": "UDP reader",
+        "condition":{
+            "udprEnabled":"true"
+        }
     },
     {
         "name": "wifiClient",
diff --git a/web/index.js b/web/index.js
index 337c21cb..6962506b 100644
--- a/web/index.js
+++ b/web/index.js
@@ -181,6 +181,12 @@
         }
     }
 
+    checkers.checkPort=function(v,allValues,def){
+        let parsed=parseInt(v);
+        if (isNaN(parsed)) return "must be a number";
+        if (parsed <1 || parsed >= 65536) return "port must be in the range 1..65536"; 
+    }
+
     checkers.checkSystemName=function(v) {
         //2...32 characters for ssid
         let allowed = v.replace(/[^a-zA-Z0-9]*/g, '');
@@ -213,13 +219,20 @@
     }
 
     checkers.checkIpAddress=function(v, allValues, def) {
-        if (allValues.tclEnabled != "true") return;
         if (!v) return "cannot be empty";
         if (!v.match(/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/)
             && !v.match(/.*\.local/))
             return "must be either in the form 192.168.1.1 or xxx.local";
     }
+    checkers.checkMCAddress=function(v, allValues, def) {
+        if (!v) return "cannot be empty";
+        if (!v.match(/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/))
+            return "must be in the form 224.0.0.1";
+        let parts=v.split(".");
+        let o1=parseInt(v[0]);
+        if (o1 < 224 || o1 > 239) return "mulicast address must be in the range 224.0.0.0 to 239.255.255.255"
 
+    }
     checkers.checkXDR=function(v, allValues) {
         if (!v) return;
         let parts = v.split(',');
@@ -264,21 +277,22 @@
                 continue;
             }
             let check = v.getAttribute('data-check');
-            if (check) {
+            if (check && conditionOk(name)) {
+                let cfgDef=getConfigDefition(name);
                 let checkFunction=checkers[check];
                 if (typeof (checkFunction) === 'function') {
                     if (! loggedChecks[check]){
                         loggedChecks[check]=true;
                         //console.log("check:"+check);
                     }
-                    let res = checkFunction(v.value, allValues, getConfigDefition(name));
+                    let res = checkFunction(v.value, allValues, cfgDef);
                     if (res) {
                         let value = v.value;
                         if (v.type === 'password') value = "******";
                         let label = v.getAttribute('data-label');
                         if (!label) label = v.getAttribute('name');
                         v.classList.add("error");
-                        alert("invalid config for " + label + "(" + value + "):\n" + res);
+                        alert("invalid config for "+cfgDef.category+":" + label + "(" + value + "):\n" + res);
                         return;
                     }
                 }
@@ -472,10 +486,10 @@
         if (!(condition instanceof Array)) condition = [condition];
         return condition;
     }
-    function checkCondition(element) {
-        let name = element.getAttribute('name');
+
+    function conditionOk(name){
         let condition = getConditions(name);
-        if (!condition) return;
+        if (!condition) return true;
         let visible = false;
         if (!condition instanceof Array) condition = [condition];
         condition.forEach(function (cel) {
@@ -493,7 +507,13 @@
                 }
             }
             if (lvis) visible = true;
-        });
+        }); 
+        return visible;   
+    }
+
+    function checkCondition(element) {
+        let name = element.getAttribute('name');
+        let visible=conditionOk(name);
         let row = closestParent(element, 'row');
         if (!row) return;
         if (visible) row.classList.remove('hidden');

From 4bded7bbb4e40f049b3c51280b4d46fcbf24c954 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 11 Nov 2024 19:52:44 +0100
Subject: [PATCH 17/37] make sta udp receiver working

---
 lib/socketserver/GwUdpReader.cpp | 2 +-
 web/index.js                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/socketserver/GwUdpReader.cpp b/lib/socketserver/GwUdpReader.cpp
index e8b619e9..d7efbb90 100644
--- a/lib/socketserver/GwUdpReader.cpp
+++ b/lib/socketserver/GwUdpReader.cpp
@@ -120,7 +120,7 @@ void GwUdpReader::loop(bool handleRead, bool handleWrite)
             //only change anything if we considered the station IP
             String nextStationIp;
             if (WiFi.isConnected()){
-                String nextStationIp=WiFi.localIP().toString();
+                nextStationIp=WiFi.localIP().toString();
             }
             if (setStationAdd(nextStationIp)){
                 LOG_INFO("UDPR: wifi client IP changed, restart");
diff --git a/web/index.js b/web/index.js
index 6962506b..a82f79e2 100644
--- a/web/index.js
+++ b/web/index.js
@@ -229,7 +229,7 @@
         if (!v.match(/[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*/))
             return "must be in the form 224.0.0.1";
         let parts=v.split(".");
-        let o1=parseInt(v[0]);
+        let o1=parseInt(parts[0]);
         if (o1 < 224 || o1 > 239) return "mulicast address must be in the range 224.0.0.0 to 239.255.255.255"
 
     }

From b4eaad4dbfbd0c47bd4bb385669fc63993c5ce17 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 11 Nov 2024 20:07:22 +0100
Subject: [PATCH 18/37] bind mc udp receiver to mc address

---
 lib/socketserver/GwUdpReader.cpp | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/lib/socketserver/GwUdpReader.cpp b/lib/socketserver/GwUdpReader.cpp
index d7efbb90..b88848cf 100644
--- a/lib/socketserver/GwUdpReader.cpp
+++ b/lib/socketserver/GwUdpReader.cpp
@@ -49,13 +49,7 @@ void GwUdpReader::createAndBind(){
         return;
     }
     struct ip_mreq mc;
-    String mcAddr=config->getString(GwConfigDefinitions::udprMC);
-    if (inet_pton(AF_INET,mcAddr.c_str(),&mc.imr_multiaddr) != 1){
-        LOG_ERROR("UDPR: invalid multicast addr %s",mcAddr.c_str());
-        ::close(fd);
-        fd=-1;
-        return;
-    }
+    mc.imr_multiaddr=listenA.sin_addr;
     if (type == T_MCALL || type == T_MCAP){
         mc.imr_interface=apAddr;
         int res=setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP,&mc,sizeof(mc));
@@ -63,7 +57,7 @@ void GwUdpReader::createAndBind(){
             LOG_ERROR("UDPR: unable to add MC membership for AP:%d",errno);
         }
         else{
-            LOG_INFO("UDPR: membership for %s for AP",mcAddr.c_str());
+            LOG_INFO("UDPR: membership for for AP");
         }
     }
     if (!currentStationIp.isEmpty() && (type == T_MCALL || type == T_MCSTA))
@@ -75,7 +69,7 @@ void GwUdpReader::createAndBind(){
             LOG_ERROR("UDPR: unable to add MC membership for STA:%d", errno);
         }
         else{
-            LOG_INFO("UDPR: membership for %s for STA %s",mcAddr.c_str(),currentStationIp.c_str());
+            LOG_INFO("UDPR: membership for STA %s",currentStationIp.c_str());
         }
     }
 }
@@ -88,15 +82,27 @@ void GwUdpReader::begin()
     port=config->getInt(GwConfigDefinitions::udprPort);
     listenA.sin_family=AF_INET;
     listenA.sin_port=htons(port);
-    if (type != T_STA){
-        listenA.sin_addr.s_addr=htonl(INADDR_ANY);
-    }
+    listenA.sin_addr.s_addr=htonl(INADDR_ANY); //default
     String ap=WiFi.softAPIP().toString();
     if (inet_pton(AF_INET, ap.c_str(), &apAddr) != 1)
     {
         LOG_ERROR("UDPR: invalid ap ip address %s", ap.c_str());
         return;
     }
+    if (type == T_MCALL || type == T_MCAP || type == T_MCSTA){
+        String mcAddr=config->getString(GwConfigDefinitions::udprMC);
+        if (inet_pton(AF_INET, mcAddr.c_str(), &listenA.sin_addr) != 1)
+        {
+            LOG_ERROR("UDPR: invalid mc address %s", mcAddr.c_str());
+            close(fd);
+            fd = -1;
+            return;
+        }
+        LOG_INFO("UDPR: using multicast address %s",mcAddr.c_str());
+    }
+    if (type == T_AP){
+        listenA.sin_addr=apAddr;
+    }
     String sta;
     if (WiFi.isConnected()) sta=WiFi.localIP().toString();
     setStationAdd(sta);

From 098b9ba55826990c9fc6e57da7a5835765a258ea Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Mon, 11 Nov 2024 20:28:57 +0100
Subject: [PATCH 19/37] use GwBuffer::fillData for udp receive

---
 lib/queue/GwBuffer.cpp           | 10 ----------
 lib/queue/GwBuffer.h             |  5 +----
 lib/socketserver/GwUdpReader.cpp | 19 +++++++++++--------
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/lib/queue/GwBuffer.cpp b/lib/queue/GwBuffer.cpp
index de0f0706..66a8d754 100644
--- a/lib/queue/GwBuffer.cpp
+++ b/lib/queue/GwBuffer.cpp
@@ -33,16 +33,6 @@ size_t GwBuffer::freeSpace()
     }
     return readPointer - writePointer - 1;
 }
-size_t GwBuffer::continousSpace() const{
-    if (readPointer <= writePointer){
-        return bufferSize-offset(writePointer);
-    }
-    return readPointer-writePointer-1;
-}
-void GwBuffer::moveWp(size_t offset){
-    if (offset > continousSpace()) return;
-    writePointer+=offset;
-}
 size_t GwBuffer::usedSpace()
 {
     if (readPointer <= writePointer)
diff --git a/lib/queue/GwBuffer.h b/lib/queue/GwBuffer.h
index aa67e047..145984fc 100644
--- a/lib/queue/GwBuffer.h
+++ b/lib/queue/GwBuffer.h
@@ -18,9 +18,9 @@ class GwMessageFetcher{
  * buffer to safely inserte data if it fits
  * and to write out data if possible
  */
-typedef size_t (*GwBufferHandleFunction)(uint8_t *buffer, size_t len, void *param);
 class GwBuffer{
     public:
+        using GwBufferHandleFunction=std::function<size_t(uint8_t *buffer, size_t len, void *param)>;
         static const size_t TX_BUFFER_SIZE=1620; // app. 20 NMEA messages
         static const size_t RX_BUFFER_SIZE=600;  // enough for 1 NMEA message or actisense message or seasmart message
         typedef enum {
@@ -54,9 +54,6 @@ class GwBuffer{
          * find the first occurance of x in the buffer, -1 if not found
          */
         int findChar(char x);
-        uint8_t *getWp(){return writePointer;}
-        size_t continousSpace() const; //free space from wp
-        void moveWp(size_t offset); //move the wp forward
 };
 
 #endif
\ No newline at end of file
diff --git a/lib/socketserver/GwUdpReader.cpp b/lib/socketserver/GwUdpReader.cpp
index b88848cf..612eb10e 100644
--- a/lib/socketserver/GwUdpReader.cpp
+++ b/lib/socketserver/GwUdpReader.cpp
@@ -142,15 +142,18 @@ void GwUdpReader::readMessages(GwMessageFetcher *writer)
     if (fd < 0) return;
     //we expect one NMEA message in one UDP packet
     buffer->reset();
-    struct sockaddr_in from;
-    socklen_t fromLen=sizeof(from);
-    ssize_t res=recvfrom(fd,buffer->getWp(),buffer->continousSpace(),MSG_DONTWAIT,
+    size_t rd=buffer->fillData(buffer->freeSpace(),
+    [this](uint8_t *rcvb,size_t rcvlen,void *param)->size_t{
+        struct sockaddr_in from;
+        socklen_t fromLen=sizeof(from);
+        ssize_t res=recvfrom(fd,rcvb,rcvlen,MSG_DONTWAIT,
         (struct sockaddr*)&from,&fromLen);
-    if (res <= 0) return;
-    if (GwSocketHelper::equals(from.sin_addr,apAddr)) return;
-    if (!currentStationIp.isEmpty() && (GwSocketHelper::equals(from.sin_addr,staAddr))) return;
-    buffer->moveWp(res);
-    LOG_DEBUG(GwLog::DEBUG,"UDPR: received %d bytes",res);
+        if (res <= 0) return 0;
+        if (GwSocketHelper::equals(from.sin_addr,apAddr)) return 0;
+        if (!currentStationIp.isEmpty() && (GwSocketHelper::equals(from.sin_addr,staAddr))) return 0;
+        return res;
+    },this);
+    if (buffer->usedSpace() > 0)(GwLog::DEBUG,"UDPR: received %d bytes",buffer->usedSpace());
     writer->handleBuffer(buffer);   
 }
 size_t GwUdpReader::sendToClients(const char *buf, int source,bool partial)

From 4504b8832f262f8c2816929de61031f1d8a48024 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Wed, 13 Nov 2024 10:59:00 +0100
Subject: [PATCH 20/37] #66: pick minimal changes of latest version from
 NMEA0183-AIS

---
 lib/nmea2ktoais/NMEA0183AISMessages.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/nmea2ktoais/NMEA0183AISMessages.cpp b/lib/nmea2ktoais/NMEA0183AISMessages.cpp
index 081a1b64..a0f9ec01 100644
--- a/lib/nmea2ktoais/NMEA0183AISMessages.cpp
+++ b/lib/nmea2ktoais/NMEA0183AISMessages.cpp
@@ -219,7 +219,7 @@ bool SetAISClassBMessage18(tNMEA0183AISMsg &NMEA0183AISMsg, uint8_t MessageID, u
 bool SetAISClassBMessage24PartA(tNMEA0183AISMsg &NMEA0183AISMsg, uint8_t MessageID, uint8_t Repeat, uint32_t UserID, char *Name) {
 
   bool found = false;
-  for (int i = 0; i < vships.size(); i++) {
+  for (size_t i = 0; i < vships.size(); i++) {
     if ( vships[i]->_userID == UserID ) {
       found = true;
       break;

From 7ea500bad650b17bc4210fb94422e3ffd5674d00 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Wed, 13 Nov 2024 15:08:27 +0100
Subject: [PATCH 21/37] #87: include system name in export file name

---
 web/index.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/web/index.js b/web/index.js
index a82f79e2..eabd250b 100644
--- a/web/index.js
+++ b/web/index.js
@@ -1071,8 +1071,12 @@
     function formatDateForFilename(usePrefix, d) {
         let rt = "";
         if (usePrefix) {
+            let hdl= document.getElementById('headline');
+            if (hdl){
+                rt=hdl.textContent+"_";
+            }
             let fwt = document.querySelector('.status-fwtype');
-            if (fwt) rt = fwt.textContent;
+            if (fwt) rt += fwt.textContent;
             rt += "_";
         }
         if (!d) d = new Date();

From 2c87be78db93550762ddc4cf9e7ed251cd119c3a Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Wed, 13 Nov 2024 17:05:17 +0100
Subject: [PATCH 22/37] make changing timeouts working correctly

---
 lib/boatData/GwBoatData.cpp | 62 ++++++++++++++++++++++---------------
 lib/boatData/GwBoatData.h   | 19 +++++++++---
 src/main.cpp                |  1 +
 3 files changed, 52 insertions(+), 30 deletions(-)

diff --git a/lib/boatData/GwBoatData.cpp b/lib/boatData/GwBoatData.cpp
index 4be14505..39d22c8f 100644
--- a/lib/boatData/GwBoatData.cpp
+++ b/lib/boatData/GwBoatData.cpp
@@ -48,12 +48,36 @@ GwBoatItemBase::GwBoatItemBase(String name, String format, GwBoatItemBase::TOTyp
     this->type = 0;
     this->lastUpdateSource = -1;
 }
-void GwBoatItemBase::setInvalidTime(unsigned long it, bool force){
-    if (toType != TOType::user || force ){
-        invalidTime=it;
+void GwBoatItemBase::setInvalidTime(GwConfigHandler *cfg){
+    if (toType != TOType::user){
+        unsigned long timeout=GwBoatItemBase::INVALID_TIME;
+        switch(getToType()){
+            case GwBoatItemBase::TOType::ais:
+                timeout=cfg->getInt(GwConfigDefinitions::timoAis);
+                break;
+            case GwBoatItemBase::TOType::def:
+                timeout=cfg->getInt(GwConfigDefinitions::timoDefault);
+                break;
+            case GwBoatItemBase::TOType::lng:
+                timeout=cfg->getInt(GwConfigDefinitions::timoLong);
+                break;
+            case GwBoatItemBase::TOType::sensor:
+                timeout=cfg->getInt(GwConfigDefinitions::timoSensor);
+                break;
+            case GwBoatItemBase::TOType::keep:
+                timeout=0;
+                break;
+        }
+        invalidTime=timeout;
     }
 }
 size_t GwBoatItemBase::getJsonSize() { return JSON_OBJECT_SIZE(10); }
+
+void GwBoatItemBase::GwBoatItemMap::add(const String &name,GwBoatItemBase *item){
+    boatData->setInvalidTime(item);
+    (*this)[name]=item;
+}
+
 #define STRING_SIZE 40
 GwBoatItemBase::StringWriter::StringWriter()
 {
@@ -127,7 +151,7 @@ GwBoatItem<T>::GwBoatItem(String name, String formatInfo, unsigned long invalidT
     this->type = GwBoatItemTypes::getType(dummy);
     if (map)
     {
-        (*map)[name] = this;
+        map->add(name,this);
     }
 }
 template <class T>
@@ -137,7 +161,7 @@ GwBoatItem<T>::GwBoatItem(String name, String formatInfo, GwBoatItemBase::TOType
     this->type = GwBoatItemTypes::getType(dummy);
     if (map)
     {
-        (*map)[name] = this;
+        map->add(name,this);
     }
 }
 
@@ -322,28 +346,12 @@ void GwBoatDataSatList::toJsonDoc(GwJsonDocument *doc, unsigned long minTime)
 GwBoatData::GwBoatData(GwLog *logger, GwConfigHandler *cfg)
 {
     this->logger = logger;
+    this->config = cfg;
+}
+void GwBoatData::begin(){
     for (auto &&it : values){
-        unsigned long timeout=GwBoatItemBase::INVALID_TIME;
-        switch(it.second->getToType()){
-            case GwBoatItemBase::TOType::ais:
-                timeout=cfg->getInt(GwConfigDefinitions::timoAis);
-                break;
-            case GwBoatItemBase::TOType::def:
-                timeout=cfg->getInt(GwConfigDefinitions::timoDefault);
-                break;
-            case GwBoatItemBase::TOType::lng:
-                timeout=cfg->getInt(GwConfigDefinitions::timoLong);
-                break;
-            case GwBoatItemBase::TOType::sensor:
-                timeout=cfg->getInt(GwConfigDefinitions::timoSensor);
-                break;
-            case GwBoatItemBase::TOType::keep:
-                timeout=0;
-                break;
-        }
-        it.second->setInvalidTime(timeout);
+        it.second->setInvalidTime(config);
     }
-
 }
 GwBoatData::~GwBoatData()
 {
@@ -456,6 +464,10 @@ double GwBoatData::getDoubleValue(String name, double defaultv)
         return defaultv;
     return it->second->getDoubleValue();
 }
+
+void GwBoatData::setInvalidTime(GwBoatItemBase *item){
+    if (config != nullptr) item->setInvalidTime(config);
+}
 double formatCourse(double cv)
 {
     double rt = cv * 180.0 / M_PI;
diff --git a/lib/boatData/GwBoatData.h b/lib/boatData/GwBoatData.h
index 18a5f083..a4df3b4e 100644
--- a/lib/boatData/GwBoatData.h
+++ b/lib/boatData/GwBoatData.h
@@ -14,6 +14,8 @@
 #define ROT_WA_FACTOR 60
 
 class GwJsonDocument;
+class GwBoatData;
+
 class GwBoatItemBase{
     public:
         using TOType=enum{
@@ -56,7 +58,6 @@ class GwBoatItemBase{
         GWSC(formatRot);
         GWSC(formatDate);
         GWSC(formatTime);
-        typedef std::map<String,GwBoatItemBase*> GwBoatItemMap;
     protected:
         int type;
         unsigned long lastSet=0;
@@ -93,10 +94,15 @@ class GwBoatItemBase{
         virtual double getDoubleValue()=0;
         String getName(){return name;}
         const String & getFormat() const{return format;}
-        virtual void setInvalidTime(unsigned long it, bool force=true);
+        virtual void setInvalidTime(GwConfigHandler *cfg);
         TOType getToType(){return toType;}
+        class GwBoatItemMap : public std::map<String,GwBoatItemBase*>{
+            GwBoatData *boatData;
+            public:
+            GwBoatItemMap(GwBoatData *bd):boatData(bd){}
+            void add(const String &name,GwBoatItemBase *item);
+        };
 };
-class GwBoatData;
 template<class T> class GwBoatItem : public GwBoatItemBase{
     protected:
         T data;
@@ -186,8 +192,9 @@ class GwBoatItemNameProvider
     clazz *name=new clazz(#name,GwBoatItemBase::fmt,toType,&values) ;
 class GwBoatData{
     private:
-        GwLog *logger;
-        GwBoatItemBase::GwBoatItemMap values;
+        GwLog *logger=nullptr;
+        GwConfigHandler *config=nullptr;
+        GwBoatItemBase::GwBoatItemMap values{this};
     public:
 
     GWBOATDATA(double,COG,formatCourse) // course over ground
@@ -231,9 +238,11 @@ class GwBoatData{
     public:
         GwBoatData(GwLog *logger, GwConfigHandler *cfg);
         ~GwBoatData();
+        void begin();
         template<class T> GwBoatItem<T> *getOrCreate(T initial,GwBoatItemNameProvider *provider);
         template<class T> bool update(T value,int source,GwBoatItemNameProvider *provider);
         template<class T> T getDataWithDefault(T defaultv, GwBoatItemNameProvider *provider);
+        void setInvalidTime(GwBoatItemBase *item);
         bool isValid(String name);
         double getDoubleValue(String name,double defaultv);
         GwBoatItemBase *getBase(String name);
diff --git a/src/main.cpp b/src/main.cpp
index 714c0a5a..90e56917 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -784,6 +784,7 @@ void setup() {
     logger.prefix="FALLBACK:";
   logger.setWriter(new DefaultLogWriter());
 #endif
+  boatData.begin();
   userCodeHandler.startInitTasks(MIN_USER_TASK);
   channels.preinit();
   config.stopChanges();

From a8a0df4b70c42e05ba0ac04b6e7317fe733d4947 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Wed, 13 Nov 2024 18:07:50 +0100
Subject: [PATCH 23/37] #50: fix handling of GSV messages (completely wrong sat
 info)

---
 lib/nmea0183ton2k/NMEA0183DataToN2K.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/nmea0183ton2k/NMEA0183DataToN2K.cpp b/lib/nmea0183ton2k/NMEA0183DataToN2K.cpp
index be01cacd..bd593d5f 100644
--- a/lib/nmea0183ton2k/NMEA0183DataToN2K.cpp
+++ b/lib/nmea0183ton2k/NMEA0183DataToN2K.cpp
@@ -860,7 +860,7 @@ class NMEA0183DataToN2KFunctions : public NMEA0183DataToN2K
             LOG_DEBUG(GwLog::DEBUG,"GSV invalid current %u %s",current,msg.line);
             return;
         }
-        for (int idx=2;idx < msg.FieldCount();idx+=4){
+        for (int idx=3;idx < msg.FieldCount();idx+=4){
             if (msg.FieldLen(idx) < 1 ||
                 msg.FieldLen(idx+1) < 1 ||
                 msg.FieldLen(idx+2) < 1 ||

From f73390c9aef827e187e05d0a48aefeaaf8df1bfb Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 15:20:13 +0100
Subject: [PATCH 24/37] enable channel if only seasmart or actisense is
 configured

---
 lib/channel/GwChannelList.cpp | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/channel/GwChannelList.cpp b/lib/channel/GwChannelList.cpp
index 4371cef0..7861f3d2 100644
--- a/lib/channel/GwChannelList.cpp
+++ b/lib/channel/GwChannelList.cpp
@@ -369,17 +369,20 @@ static GwChannel * createChannel(GwLog *logger, GwConfigHandler *config, int id,
         return nullptr;
     }
     GwChannel *channel = new GwChannel(logger, param->name,param->id,param->maxId);
+    bool sendSeaSmart=config->getBool(param->sendSeasmart);
+    bool readAct=config->getBool(param->readAct);
+    bool writeAct=config->getBool(param->writeAct);
     channel->setImpl(impl);
     channel->begin(
-        canRead || canWrite,
+        canRead || canWrite || readAct || writeAct|| sendSeaSmart,
         canWrite,
         canRead,
         config->getString(param->readF),
         config->getString(param->writeF),
-        config->getBool(param->sendSeasmart),
+        sendSeaSmart,
         config->getBool(param->toN2K),
-        config->getBool(param->readAct),
-        config->getBool(param->writeAct));
+        readAct,
+        writeAct);
     LOG_INFO("created channel %s",channel->toString().c_str());
     return channel;
 }

From 048a16d9391bf3a97910f03aa562c2335b697df8 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 16:16:40 +0100
Subject: [PATCH 25/37] add listeners for status handling

---
 web/index.js | 106 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 59 insertions(+), 47 deletions(-)

diff --git a/web/index.js b/web/index.js
index eabd250b..538a51a2 100644
--- a/web/index.js
+++ b/web/index.js
@@ -76,47 +76,49 @@
         }
         getJson('/api/status')
             .then(function (jsonData) {
+                if (jsonData.salt !== undefined) {
+                    lastSalt=jsonData.salt;
+                    delete jsonData.salt;
+                }
+                if (jsonData.minUser !== undefined){
+                    minUser=jsonData.minUser;
+                    delete jsonData.minUser;
+                }
+                callListeners(api.EVENTS.status,jsonData);
                 let statusPage = document.getElementById('statusPageContent');
                 let even = true; //first counter
-                for (let k in jsonData) {
-                    if (k == "salt") {
-                        lastSalt = jsonData[k];
-                        continue;
-                    }
-                    if (k == "minUser") {
-                        minUser = parseInt(jsonData[k]);
-                        continue;
-                    }
-                    if (!statusPage) continue;
-                    if (typeof (jsonData[k]) === 'object') {
-                        if (k.indexOf('count') == 0) {
-                            createCounterDisplay(statusPage, k.replace("count", "").replace(/in$/, " in").replace(/out$/, " out"), k, even);
-                            even = !even;
-                            for (let sk in jsonData[k]) {
-                                let key = k + "." + sk;
-                                if (typeof (jsonData[k][sk]) === 'object') {
-                                    //msg details
-                                    updateMsgDetails(key, jsonData[k][sk]);
-                                }
-                                else {
-                                    let el = document.getElementById(key);
-                                    if (el) el.textContent = jsonData[k][sk];
+                if (statusPage){
+                    for (let k in jsonData) {
+                        if (typeof (jsonData[k]) === 'object') {
+                            if (k.indexOf('count') == 0) {
+                                createCounterDisplay(statusPage, k.replace("count", "").replace(/in$/, " in").replace(/out$/, " out"), k, even);
+                                even = !even;
+                                for (let sk in jsonData[k]) {
+                                    let key = k + "." + sk;
+                                    if (typeof (jsonData[k][sk]) === 'object') {
+                                        //msg details
+                                        updateMsgDetails(key, jsonData[k][sk]);
+                                    }
+                                    else {
+                                        let el = document.getElementById(key);
+                                        if (el) el.textContent = jsonData[k][sk];
+                                    }
                                 }
                             }
+                            if (k.indexOf("ch") == 0) {
+                                //channel def
+                                let name = k.substring(2);
+                                channelList[name] = jsonData[k];
+                            }
                         }
-                        if (k.indexOf("ch") == 0) {
-                            //channel def
-                            let name = k.substring(2);
-                            channelList[name] = jsonData[k];
+                        else {
+                            let el = document.getElementById(k);
+                            if (el) el.textContent = jsonData[k];
+                            forEl('.status-' + k, function (el) {
+                                el.textContent = jsonData[k];
+                            });
                         }
                     }
-                    else {
-                        let el = document.getElementById(k);
-                        if (el) el.textContent = jsonData[k];
-                        forEl('.status-' + k, function (el) {
-                            el.textContent = jsonData[k];
-                        });
-                    }
                 }
                 lastUpdate = (new Date()).getTime();
                 if (reloadConfig) {
@@ -382,6 +384,7 @@
                 icon.classList.add('icon-less');
             }
         });
+        callListeners(api.EVENTS.counterDisplayCreated,row);
     }
     function validKey(key) {
         if (!key) return;
@@ -1996,8 +1999,16 @@
         hideDashboardItem(name); //will recreate it on next data receive
     }
     const api= {
-        registerListener: function (callback) {
-            listeners.push(callback);
+        registerListener: function (callback,opt_event) {
+            if (opt_event === undefined){
+                listeners.push(callback);
+            }
+            else{
+                listeners.push({
+                    event:opt_event,
+                    callback:callback
+                })
+            }
         },
         /**
          * helper for creating dom elements
@@ -2058,11 +2069,13 @@
         parseBoatDataLine: parseBoatDataLine,
         EVENTS: {
             init: 0, //called when capabilities are loaded, data is capabilities
-            tab: 1, //tab page activated data is the id of the tab page
-            config: 2, //data is the config object
-            boatData: 3, //data is the list of boat Data items
+            tab: 1, //tab page activated, data is the id of the tab page
+            config: 2, //called when the config data is loaded,data is the config object
+            boatData: 3, //called when boatData is received, data is the list of boat Data items
             dataItemCreated: 4, //data is an object with
                                 // name: the item name, element: the frame item of the boat data display
+            status: 5, //status received, data is the status object
+            counterDisplayCreated: 6 //data is the row for the display
         }
     };
     function callListeners(event,data){
@@ -2070,6 +2083,13 @@
             if (typeof(listener) === 'function'){
                 listener(event,data);
             }
+            else if (typeof(listener) === 'object'){
+                if (listener.event === event){
+                    if (typeof(listener.callback) === 'function'){
+                        listener.callback(event,data);
+                    }
+                }
+            }
         })
     }
     window.esp32nmea2k = api;
@@ -2120,14 +2140,6 @@
                 });
             }
         } catch (e) { }
-        let statusPage = document.getElementById('statusPageContent');
-        /*if (statusPage){
-            let even=true;
-            for (let c in counters){
-                createCounterDisplay(statusPage,counters[c],c,even);
-                even=!even;
-            }
-        }*/
         forEl('#uploadFile', function (el) {
             el.addEventListener('change', function (ev) {
                 if (ev.target.files.length < 1) return;

From 506dd7ea9f11d5876c68e8a42c1431ed8abb2c8d Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 16:16:51 +0100
Subject: [PATCH 26/37] add example listener

---
 lib/exampletask/index.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/exampletask/index.js b/lib/exampletask/index.js
index a85fe10d..da098e09 100644
--- a/lib/exampletask/index.js
+++ b/lib/exampletask/index.js
@@ -11,6 +11,11 @@
     const infoUrl='https://github.com/wellenvogel/esp32-nmea2000/tree/master/lib/exampletask';
     let boatItemName;
     let boatItemElement;
+    api.registerListener((id,data)=>{
+        if (isActive){
+            console.log("exampletask status listener",data);
+        }
+    },api.EVENTS.status)
     api.registerListener((id,data)=>{
         if (id === api.EVENTS.init){
             //data is capabilities

From 538f643fbffb83db65c5cc52bf5c3a7cf855c9ed Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 18:15:12 +0100
Subject: [PATCH 27/37] add registerRequestHandler to the API with examples

---
 lib/api/GwApi.h                   | 16 +++++++++++++
 lib/exampletask/GwExampleTask.cpp | 37 +++++++++++++++++++++++++++++++
 lib/exampletask/Readme.md         | 20 +++++++++++++++++
 lib/exampletask/index.js          | 15 +++++++++++++
 lib/gwwebserver/GwWebServer.h     |  3 ++-
 lib/queue/GwSynchronized.h        |  4 ++--
 lib/usercode/GwUserCode.cpp       | 30 +++++++++++++++++++++++++
 lib/usercode/GwUserCode.h         |  2 ++
 src/main.cpp                      | 10 ++++++++-
 9 files changed, 133 insertions(+), 4 deletions(-)

diff --git a/lib/api/GwApi.h b/lib/api/GwApi.h
index 5a24e927..9d79ef3c 100644
--- a/lib/api/GwApi.h
+++ b/lib/api/GwApi.h
@@ -6,7 +6,9 @@
 #include "GWConfig.h"
 #include "GwBoatData.h"
 #include "GwXDRMappings.h"
+#include "GwSynchronized.h"
 #include <map>
+#include <ESPAsyncWebServer.h>
 class GwApi;
 typedef void (*GwUserTaskFunction)(GwApi *);
 //API to be used for additional tasks
@@ -171,6 +173,20 @@ class GwApi{
         virtual void remove(int idx){}
         virtual TaskInterfaces * taskInterfaces()=0;
 
+        /**
+         * register handler for web URLs
+         * Please be aware that this handler function will always be called from a separate
+         * task. So you must ensure proper synchronization!
+        */
+        using HandlerFunction=std::function<void(AsyncWebServerRequest *)>;
+        /**
+         * @param url: the url of that will trigger the handler.
+         *             it will be prefixed with /api/user/<taskname>
+         *             taskname is the name that you used in addUserTask
+         * @param handler: the handler function (see remark above about thread synchronization)
+        */
+        virtual void registerRequestHandler(const String &url,HandlerFunction handler)=0;
+
         /**
          * only allowed during init methods
         */
diff --git a/lib/exampletask/GwExampleTask.cpp b/lib/exampletask/GwExampleTask.cpp
index 340b7851..9b4cbcdc 100644
--- a/lib/exampletask/GwExampleTask.cpp
+++ b/lib/exampletask/GwExampleTask.cpp
@@ -7,6 +7,7 @@
 #include <vector>
 #include "N2kMessages.h"
 #include "GwXdrTypeMappings.h"
+
 /**
  * INVALID!!! - the next interface declaration will not work
  *              as it is not in the correct header file
@@ -144,6 +145,26 @@ String formatValue(GwApi::BoatValue *value){
     return String(buffer);
 }
 
+class ExampleWebData{
+    SemaphoreHandle_t lock;
+    int data=0;
+    public:
+    ExampleWebData(){
+        lock=xSemaphoreCreateMutex();
+    }
+    ~ExampleWebData(){
+        vSemaphoreDelete(lock);
+    }
+    void set(int v){
+        GWSYNCHRONIZED(&lock);
+        data=v;
+    }
+    int get(){
+        GWSYNCHRONIZED(&lock);
+        return data;
+    }
+};
+
 void exampleTask(GwApi *api){
     GwLog *logger=api->getLogger();
     //get some configuration data
@@ -172,8 +193,24 @@ void exampleTask(GwApi *api){
     LOG_DEBUG(GwLog::LOG,"exampleNotWorking update returned %d",(int)nwrs);
     String voltageTransducer=api->getConfig()->getString(GwConfigDefinitions::exTransducer);
     int voltageInstance=api->getConfig()->getInt(GwConfigDefinitions::exInstanceId);
+    ExampleWebData webData;
+    /**
+     * an example web request handler
+     * it uses a synchronized data structure as it gets called from a different thread
+     * be aware that you must not block for longer times here!
+    */
+    api->registerRequestHandler("data",[&webData](AsyncWebServerRequest *request){
+        int data=webData.get();
+        char buffer[30];
+        snprintf(buffer,29,"%d",data);
+        buffer[29]=0;
+        request->send(200,"text/plain",buffer); 
+    });
+    int loopcounter=0;
     while(true){
         delay(1000);
+        loopcounter++;
+        webData.set(loopcounter);
         /*
         * getting values from the internal data store (boatData) requires some special handling
         * our tasks runs (potentially) at some time on a different core then the main code
diff --git a/lib/exampletask/Readme.md b/lib/exampletask/Readme.md
index 965e1773..2a7e42e3 100644
--- a/lib/exampletask/Readme.md
+++ b/lib/exampletask/Readme.md
@@ -32,6 +32,26 @@ Files
      This file allows to add some config definitions that are needed for our task. For the possible options have a look at the global [config.json](../../web/config.json). Be careful not to overwrite config defitions from the global file. A good practice wood be to prefix the names of definitions with parts of the library name. Always put them in a separate category so that they do not interfere with the system ones.
      The defined config items can later be accessed in the code (see the example in [GwExampleTask.cpp](GwExampleTask.cpp)).
 
+   * [index.js](index.js)<br>
+     You can add javascript code that will contribute to the UI of the system. The WebUI provides a small API that allows you to "hook" into some functions to include your own parts of the UI. This includes adding new tabs, modifying/replacing the data display items, modifying the status display or accessing the config items.
+     For the API refer to [../../web/index.js](../../web/index.js#L2001).
+     To start interacting just register for some events like api.EVENTS.init. You can check the capabilities you have defined to see if your task is active.
+     By registering an own formatter [api.addUserFormatter](../../web/index.js#L2054) you can influence the way boat data items are shown.
+     You can even go for an own display by registering for the event *dataItemCreated* and replace the dom element content with your own html. By additionally having added a user formatter you can now fill your own html with the current value.
+     By using [api.addTabPage](../../web/index.js#L2046) you can add new tabs that you can populate with your own code. Or you can link to an external URL.<br>
+     Please be aware that your js code is always combined with the code from the core into one js file.<br>
+     For fast testing there is a small python script that allow you to test the UI without always flushing each change.
+     Just run it with
+     ```
+     tools/testServer.py nnn http://x.x.x.x/api
+     ```
+     with nnn being the local port and x.x.x.x the address of a running system. Open `http://localhost:nnn` in your browser.<br>
+     After a change just start the compilation and reload the page.
+
+   * [index.css](index.css)<br>
+     You can add own css to influence the styling of the display.
+
+
  Interfaces
  ----------
  The task init function and the task function interact with the core using an [API](../api/GwApi.h) that they get when started.
diff --git a/lib/exampletask/index.js b/lib/exampletask/index.js
index da098e09..155991cc 100644
--- a/lib/exampletask/index.js
+++ b/lib/exampletask/index.js
@@ -28,6 +28,21 @@
                 //you can use the helper addEl to create elements
                 let page=api.addTabPage(tabName,"Example");
                 api.addEl('div','hdg',page,"this is a test tab");
+                let vrow=api.addEl('div','row',page);
+                api.addEl('span','label',vrow,'loops: ');
+                let lcount=api.addEl('span','value',vrow,'0');
+                //query the loop count
+                window.setInterval(()=>{
+                    fetch('/api/user/exampleTask/data')
+                        .then((res)=>{
+                            if (! res.ok) throw Error("server error: "+res.status);
+                            return res.text();
+                        })
+                        .then((txt)=>{
+                            lcount.textContent=txt;
+                        })
+                        .catch((e)=>console.log("rq:",e));
+                },1000);
                 api.addEl('button','',page,'Info').addEventListener('click',function(ev){
                     window.open(infoUrl,'info');
                 })
diff --git a/lib/gwwebserver/GwWebServer.h b/lib/gwwebserver/GwWebServer.h
index 84c265a5..c2b48d89 100644
--- a/lib/gwwebserver/GwWebServer.h
+++ b/lib/gwwebserver/GwWebServer.h
@@ -4,6 +4,7 @@
 #include <functional>
 #include "GwMessage.h"
 #include "GwLog.h"
+#include "GwApi.h"
 class GwWebServer{
     private:
         AsyncWebServer *server;
@@ -11,7 +12,7 @@ class GwWebServer{
         GwLog *logger;
     public:
         typedef GwRequestMessage *(RequestCreator)(AsyncWebServerRequest *request);
-        using HandlerFunction=std::function<void(AsyncWebServerRequest *)>;
+        using HandlerFunction=GwApi::HandlerFunction;
         GwWebServer(GwLog *logger, GwRequestQueue *queue,int port);
         ~GwWebServer();
         void begin();
diff --git a/lib/queue/GwSynchronized.h b/lib/queue/GwSynchronized.h
index 53241db9..786b5f06 100644
--- a/lib/queue/GwSynchronized.h
+++ b/lib/queue/GwSynchronized.h
@@ -7,10 +7,10 @@ class GwSynchronized{
     public:
         GwSynchronized(SemaphoreHandle_t *locker){
             this->locker=locker;
-            xSemaphoreTake(*locker, portMAX_DELAY);
+            if (locker != nullptr) xSemaphoreTake(*locker, portMAX_DELAY);
         }
         ~GwSynchronized(){
-            xSemaphoreGive(*locker);
+            if (locker != nullptr) xSemaphoreGive(*locker);
         }
 };
 
diff --git a/lib/usercode/GwUserCode.cpp b/lib/usercode/GwUserCode.cpp
index 4522c926..211eda90 100644
--- a/lib/usercode/GwUserCode.cpp
+++ b/lib/usercode/GwUserCode.cpp
@@ -191,6 +191,7 @@ class TaskApi : public GwApiInternal
     SemaphoreHandle_t *mainLock;
     SemaphoreHandle_t localLock;
     std::map<int,GwCounter<String>> counter;
+    std::map<String,GwApi::HandlerFunction> webHandlers;
     String name;
     bool counterUsed=false;
     int counterIdx=0;
@@ -315,6 +316,10 @@ class TaskApi : public GwApiInternal
     virtual bool addXdrMapping(const GwXDRMappingDef &def){
         return api->addXdrMapping(def);
     }
+    virtual void registerRequestHandler(const String &url,HandlerFunction handler){
+        GWSYNCHRONIZED(&localLock);
+        webHandlers[url]=handler;
+    }
     virtual void addCapability(const String &name, const String &value){
         if (! isInit) return;
         userCapabilities[name]=value;
@@ -335,6 +340,16 @@ class TaskApi : public GwApiInternal
     virtual void setCalibrationValue(const String &name, double value){
         api->setCalibrationValue(name,value);
     }
+    virtual bool handleWebRequest(const String &url,AsyncWebServerRequest *req){
+        GWSYNCHRONIZED(&localLock);
+        auto it=webHandlers.find(url);
+        if (it == webHandlers.end()){
+            api->getLogger()->logDebug(GwLog::LOG,"no web handler task=%s url=%s",name.c_str(),url.c_str());
+            return false;
+        }
+        it->second(req);
+        return true;
+    }
 
 };
 
@@ -404,4 +419,19 @@ int GwUserCode::getJsonSize(){
         }
     }
     return rt;
+}
+void GwUserCode::handleWebRequest(const String &url,AsyncWebServerRequest *req){
+    int sep1=url.indexOf('/');
+    String tname;
+    if (sep1 > 0){
+        tname=url.substring(0,sep1);
+        for (auto &&it:userTasks){
+            if (it.api && it.name == tname){
+                if (it.api->handleWebRequest(url.substring(sep1+1),req)) return;
+                break;
+            }
+        }
+    }
+    LOG_DEBUG(GwLog::DEBUG,"no task found for web request %s[%s]",url.c_str(),tname.c_str());
+    req->send(404, "text/plain", "not found");
 }
\ No newline at end of file
diff --git a/lib/usercode/GwUserCode.h b/lib/usercode/GwUserCode.h
index a218bc91..94e745d1 100644
--- a/lib/usercode/GwUserCode.h
+++ b/lib/usercode/GwUserCode.h
@@ -11,6 +11,7 @@ class GwApiInternal : public GwApi{
     ~GwApiInternal(){}
     virtual void fillStatus(GwJsonDocument &status){};
     virtual int getJsonSize(){return 0;};
+    virtual bool handleWebRequest(const String &url,AsyncWebServerRequest *req){return false;}
 };
 class GwUserTask{
     public:
@@ -50,5 +51,6 @@ class GwUserCode{
         Capabilities *getCapabilities();
         void fillStatus(GwJsonDocument &status);
         int getJsonSize();
+        void handleWebRequest(const String &url,AsyncWebServerRequest *);
 };
 #endif
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index 90e56917..ae8f3d52 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -348,6 +348,8 @@ class ApiImpl : public GwApiInternal
     }
     return xdrMappings.addFixedMapping(mapping);
   }
+  virtual void registerRequestHandler(const String &url,HandlerFunction handler){
+  }
   virtual void addCapability(const String &name, const String &value){}
   virtual bool addUserTask(GwUserTaskFunction task,const String Name, int stackSize=2000){
     return false;
@@ -768,6 +770,7 @@ void loopFunction(void *){
     //delay(1);
   }
 }
+const String USERPREFIX="/api/user/";
 void setup() {
   mainLock=xSemaphoreCreateMutex();
   uint8_t chipid[6];
@@ -845,7 +848,12 @@ void setup() {
                               snprintf(buffer,29,"%g",value);
                               buffer[29]=0;
                               request->send(200,"text/plain",buffer);    
-  });                                                        
+  });
+  webserver.registerHandler((USERPREFIX+"*").c_str(),[&USERPREFIX](AsyncWebServerRequest *req){
+                              String turl=req->url().substring(USERPREFIX.length());
+                              logger.logDebug(GwLog::DEBUG,"user web request for %s",turl.c_str());
+                              userCodeHandler.handleWebRequest(turl,req);
+  });
 
   webserver.begin();
   xdrMappings.begin();

From c292b82635755acd5bffa9c17d339e6a32b67729 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 18:33:18 +0100
Subject: [PATCH 28/37] correct some js errors in example

---
 lib/exampletask/index.js | 173 +++++++++++++++++++--------------------
 1 file changed, 83 insertions(+), 90 deletions(-)

diff --git a/lib/exampletask/index.js b/lib/exampletask/index.js
index 155991cc..b2c6f9db 100644
--- a/lib/exampletask/index.js
+++ b/lib/exampletask/index.js
@@ -5,104 +5,97 @@
     //on our case this is "testboard"
     //so we only start any action when we receive the init event
     //and we successfully checked that our requested capability is there
-    let isActive=false;
     const tabName="example";
     const configName="exampleBDSel";
     const infoUrl='https://github.com/wellenvogel/esp32-nmea2000/tree/master/lib/exampletask';
     let boatItemName;
     let boatItemElement;
-    api.registerListener((id,data)=>{
-        if (isActive){
-            console.log("exampletask status listener",data);
-        }
-    },api.EVENTS.status)
-    api.registerListener((id,data)=>{
-        if (id === api.EVENTS.init){
-            //data is capabilities
-            //check if our requested capability is there (see GwExampleTask.h)
-            if (data.testboard) isActive=true;
-            if (isActive){
-                //add a simple additional tab page
-                //you will have to build the content of the page dynamically
-                //using normal dom manipulation methods
-                //you can use the helper addEl to create elements
-                let page=api.addTabPage(tabName,"Example");
-                api.addEl('div','hdg',page,"this is a test tab");
-                let vrow=api.addEl('div','row',page);
-                api.addEl('span','label',vrow,'loops: ');
-                let lcount=api.addEl('span','value',vrow,'0');
-                //query the loop count
-                window.setInterval(()=>{
-                    fetch('/api/user/exampleTask/data')
-                        .then((res)=>{
-                            if (! res.ok) throw Error("server error: "+res.status);
-                            return res.text();
-                        })
-                        .then((txt)=>{
-                            lcount.textContent=txt;
-                        })
-                        .catch((e)=>console.log("rq:",e));
-                },1000);
-                api.addEl('button','',page,'Info').addEventListener('click',function(ev){
-                    window.open(infoUrl,'info');
+    api.registerListener((id, data) => {
+        //data is capabilities
+        //check if our requested capability is there (see GwExampleTask.h)
+        if (!data.testboard) return; //do nothing if we are not active
+        //add a simple additional tab page
+        //you will have to build the content of the page dynamically
+        //using normal dom manipulation methods
+        //you can use the helper addEl to create elements
+        let page = api.addTabPage(tabName, "Example");
+        api.addEl('div', 'hdg', page, "this is a test tab");
+        let vrow = api.addEl('div', 'row', page);
+        api.addEl('span', 'label', vrow, 'loops: ');
+        let lcount = api.addEl('span', 'value', vrow, '0');
+        //query the loop count
+        window.setInterval(() => {
+            fetch('/api/user/exampleTask/data')
+                .then((res) => {
+                    if (!res.ok) throw Error("server error: " + res.status);
+                    return res.text();
                 })
-                //add a tab for an external URL
-                api.addTabPage('exhelp','Info',infoUrl);
+                .then((txt) => {
+                    //set the text content of our value element with what we received
+                    lcount.textContent = txt;
+                })
+                .catch((e) => console.log("rq:", e));
+        }, 1000);
+        api.addEl('button', '', page, 'Info').addEventListener('click', function (ev) {
+            window.open(infoUrl, 'info');
+        })
+        //add a tab for an external URL
+        api.addTabPage('exhelp', 'Info', infoUrl);
+        //now as we know we are active - register all the listeners we need
+        api.registerListener((id, data) => {
+            console.log("exampletask status listener", data);
+        }, api.EVENTS.status)
+        api.registerListener((id, data) => {
+            if (data === tabName) {
+                //maybe we need some activity when our page is being activated
+                console.log("example tab activated");
             }
-        }
-        if (isActive){
-            //console.log("exampletask listener",id,data);
-            if (id === api.EVENTS.tab){
-                if (data === tabName){
-                    //maybe we need some activity when our page is being activated
-                    console.log("example tab activated");
-                }
+        }, api.EVENTS.tab);
+
+        api.registerListener((id, data) => {
+            //we have a configuration that
+            //gives us the name of a boat data item we would like to 
+            //handle special
+            //in our case we just use an own formatter and add some
+            //css to the display field
+            //as this item can change we need to keep track of the 
+            //last item we handled
+            let nextboatItemName = data[configName];
+            console.log("value of " + configName, nextboatItemName);
+            if (nextboatItemName) {
+                //register a user formatter that will be called whenever
+                //there is a new valid value
+                //we simply add an "X:" in front
+                api.addUserFormatter(nextboatItemName, "m(x)", function (v, valid) {
+                    if (!valid) return;
+                    return "X:" + v;
+                })
+                //after this call the item will be recreated
             }
-            if (id == api.EVENTS.config){
-                //we have a configuration that
-                //gives us the name of a boat data item we would like to 
-                //handle special
-                //in our case we just use an own formatter and add some
-                //css to the display field
-                //as this item can change we need to keep track of the 
-                //last item we handled
-                let nextboatItemName=data[configName];
-                console.log("value of "+configName,nextboatItemName);
-                if (nextboatItemName){
-                    //register a user formatter that will be called whenever
-                    //there is a new valid value
-                    //we simply add an "X:" in front
-                    api.addUserFormatter(nextboatItemName,"m(x)",function(v,valid){
-                        if (!valid) return;
-                        return "X:"+v;
-                    })
-                    //after this call the item will be recreated
-                }
-                if (boatItemName !== undefined && boatItemName != nextboatItemName){
-                    //if the boat item that we handle has changed, remove
-                    //the previous user formatter (this will recreate the item)
-                    api.removeUserFormatter(boatItemName);
-                }
-                boatItemName=nextboatItemName;
-                boatItemElement=undefined;
+            if (boatItemName !== undefined && boatItemName != nextboatItemName) {
+                //if the boat item that we handle has changed, remove
+                //the previous user formatter (this will recreate the item)
+                api.removeUserFormatter(boatItemName);
             }
-            if (id == api.EVENTS.dataItemCreated){
-                //this event is called whenever a data item has
-                //been created (or recreated)
-                //if this is the item we handle, we just add a css class
-                //we could also completely rebuild the dom below the element
-                //and use our formatter to directly write/draw the data
-                //avoid direct manipulation of the element (i.e. changing the classlist)
-                //as this element remains there all the time
-                if (boatItemName && boatItemName == data.name){
-                    boatItemElement=data.element;
-                    //use the helper forEl to find elements within the dashboard item
-                    //the value element has the class "dashValue"
-                    api.forEl(".dashValue",function(el){
-                        el.classList.add("examplecss");
-                    },boatItemElement);
-                }
+            boatItemName = nextboatItemName;
+            boatItemElement = undefined;
+        }, api.EVENTS.config);
+        api.registerListener((id, data) => {
+            //this event is called whenever a data item has
+            //been created (or recreated)
+            //if this is the item we handle, we just add a css class
+            //we could also completely rebuild the dom below the element
+            //and use our formatter to directly write/draw the data
+            //avoid direct manipulation of the element (i.e. changing the classlist)
+            //as this element remains there all the time
+            if (boatItemName && boatItemName == data.name) {
+                boatItemElement = data.element;
+                //use the helper forEl to find elements within the dashboard item
+                //the value element has the class "dashValue"
+                api.forEl(".dashValue", function (el) {
+                    el.classList.add("examplecss");
+                }, boatItemElement);
             }
-        }
-    })
+        }, api.EVENTS.dataItemCreated);
+    }, api.EVENTS.init);
 })();

From 13efceeef703ea1d6ee32ddf48771428f8a63db2 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 19:47:10 +0100
Subject: [PATCH 29/37] extend doc for new api functions and networking options

---
 Readme.md                 |  5 ++-
 doc/network.md            | 80 +++++++++++++++++++++++++++++++++++++++
 lib/exampletask/Readme.md |  3 +-
 3 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 doc/network.md

diff --git a/Readme.md b/Readme.md
index 6de6c6ab..ebcd08b0 100644
--- a/Readme.md
+++ b/Readme.md
@@ -47,10 +47,13 @@ Hardware
 --------
 The software is prepared to run on different kinds of ESP32 based modules and accessoirs. For some of them prebuild binaries are available that only need to be flashed, others would require to add some definitions of the used PINs and features and to build the binary.
 For the list of hardware set ups refer to [Hardware](doc/Hardware.md).
-For details of the usage of serial devices and the USB connection refer to [Serial and USB](doc/serial-usb.md).
 
 There is a couple of prebuild binaries that you can directly flash to your device. For other combinations of hardware there is an [online build service](doc/BuildService.md) that will allow you to select your hardware and trigger a build.
 
+Connectivity
+------------
+For details of the usage of serial devices and the USB connection refer to [Serial and USB](doc/serial-usb.md).<br>
+For details on the networking capabilities refer to [Networking](doc/network.md).
 
 Installation
 ------------
diff --git a/doc/network.md b/doc/network.md
new file mode 100644
index 00000000..6c92692c
--- /dev/null
+++ b/doc/network.md
@@ -0,0 +1,80 @@
+Networking
+==========
+The gateway includes a couple of network functions to send and receive NMEA data on network connections and to use a WebBrowser for configuration and status display.
+
+To understand the networking functions you always have to consider 2 parts:
+
+The "physical" connection
+-------------------------
+For the gateway this is always a Wifi connection.
+It provides the ability to use it as an access point and/or to connect to another wifi network (wifi client).
+
+Access Point
+************
+When starting up it will create an own Wifi network (access point or AP) with the name of the system. You connect to this network like to any Wifi Hotspot.
+The initial password is esp32nmea2k. You should change this password on the configuration page in the Web UI.
+When you connect the gateway will provide you with an IP address (DHCP) - and it will also have an own IP address in this range (default: 192.168.15.1). If this IP address (range) conflicts with other devices (especially if you run multiple gateways and connect them) you can change the range at the system tab on the configuration page.<br>
+If you do not need the access point during normal operation you can set "stopApTime" so that it will shut down after this time (in minutes) if no device is connected. This will also reduce the power consumption.
+
+Wifi Client
+***********
+Beside the own access point the gateway can also connect to another Wifi network. You need to configure the SSID and password at the "wifi client" tab.
+On the status page you can see if the gateway is connected and which address it got within the network.
+
+You can use both networks to connect to the gateway. It announces it's services via [bonjour](https://developer.apple.com/bonjour/). So you normally should be able to reach your system using Name.local (name being the system name, default ESP32NMEA2K). Or you can use an app that allows for bonjour browsing.
+
+The "logical" connection
+------------------------
+After you connected a device to the gateway on either the access point or by using the same Wifi network you can easily access the Web UI with a browser - e.g. using the Name.local url.
+
+To send or receive NMEA data you additionally need to configure some connection between the gateway and your device(s).
+The gateway provides the following options for NMEA connections:
+
+TCP Server
+**********
+When using the TCP server you must configure a connection to the gateway _on the device_ that you would like to connect. The gateway listens at port 10110 (can be changed at the TCP server tab). So on your device you need to configure the address of the gateway and this port. The address depends on the network that you use to connect your device - either the address from the gateway access point (default: 192.168.15.1) - or the address that the gateway was given in the Wifi client network (check this on the status page).<br>
+If your device supports this you can also use the Name.local for the address - or let the device browse for a bonjour service.<br>
+The TCP server has a limit for the number of connections (can be configured, default: 6). As for any channel you can define what it will write and read on it's connections and apply filters.
+If you want to send NMEA2000 data via TCP you can enable "Seasmart out".
+
+TCP Client
+**********
+With this settings you can configure the gateway to establish a connection to another device that provides data via TCP. You need to know the address and port for that device. If the other device also uses bonjour (like e.g. a second gateway device) you can also use this name instead of the address.
+Like for the TCP server you can define what should be send or received with filters.
+
+UDP Reader
+**********
+UDP is distinct from TCP in that it does not establish a connection directly. Instead in sends/receives messages - without checking if they have been received by someone at all. Therefore it is also not able to ensure that really all messages are reaching their destination. But as the used protocols (NMEA0183, NMEA2000) are prepared for unreliable connections any way (for serial connections you normally have no idea if the data arrives) UDP is still a valid way of connecting devices.<br>
+One advantage of UDP is that you can send out messages as broadcast or multicast - thus allowing for multiple receivers.
+
+Small hint for __multicast__:<br>
+Normally in the environment the gateway will work you will not use multicast. If you want to send to multiple devices you will use broadcast. The major difference between them are 2 points:<br>
+  1. broadcast messages will not pass a real router (but they will be available to all devices connected to one access point)
+  2. broadcast messages will always be send to all devices - regardless whether they would like to receive them or not. This can create a bit more network traffic.
+
+Multicast requires that receivers announce their interest in receiving messages (by "joining" a so called multicast group) and this way only interested devices will receive such messages - and it is possible to configure routers in a way that they route multicast messages.
+
+To use the gateway UDP reader you must select from where you would like to receive messages. In any case you need to set up a port (default: 10110). Afterwards you need to decide on the allowed sources:
+  * _all_ (default): accept messages from both the access point and the wifi client network - both broadcast messages and directly addressed ones
+  * _ap_: only accept messages from devices that are connected to the access point
+  * _cli_: only accept messages from devices on the Wifi client network
+  * _mp-all_: you need to configure the multicast address(group) you would like to join and will receive multicast from both the access point and the wifi client network
+  * _mp-ap_: multicast only from the access point network
+  * _mp-cli_: multicast only from the wifi client network
+
+UDP Writer
+**********
+The UDP writer basically is the counterpart for the UDP reader.
+You also have to select where do you want the UDP messages to be sent to.
+  * _bc-all_ (default): Send the messages as broadcast to devices connected to the own access point and to devices on the wifi client network
+  * _bc-ap_: send the messages as broadcast only to the access point network
+  * _bc-cli_: send the messages as broadcast to the wifi client network
+  * _normal_: you need to configure a destination address (one device) that you want the messages to be send to
+  * _mc-all_: send messages as multicast to both the access point network and the wifi client network. _Hint_: Only devices that configured the same multicast address will receive such messages.
+  * _mc-ap_: multicast only to the access point network
+  * _mc-cli_: multicast only to the wifi client network.
+
+With the combination of UDP writer and UDP reader you can easily connect multiple gateway devices without a lot of configuration. Just configure one device as UDP writer (with the default settings) and configure other devices as UDP reader (also with default settings) - this way it does not matter how you connect the devices - all devices will receive the data that is sent by the first one.<br>
+__Remark:__ be carefull not to create loops when you would like to send data in both directions between 2 devices using UDP. Either use filters - or use TCP connections as they are able to send data in both directions on one connection (without creating a loop).
+
+If you want to forward NMEA2000 data from one gateway device to another, just enable "Seasmart out" at the sender side. This will encapsulate the NMEA2000 data in a NMEA0183 compatible format. The receiver will always automatically detect this format and handle the data correctly.
diff --git a/lib/exampletask/Readme.md b/lib/exampletask/Readme.md
index 2a7e42e3..4c51d92f 100644
--- a/lib/exampletask/Readme.md
+++ b/lib/exampletask/Readme.md
@@ -70,7 +70,8 @@ Files
  * add capabilities (since 20231105 - as an alternative to a static DECLARE_CAPABILITY )
  * add a user task (since 20231105 - as an alternative to a static DECLARE_USERTASK)
  * store or read task interface data (see below)
-
+ * add a request handler for web requests (since 202411xx) - see registerRequestHandler in the API
+ 
 
  __Interfacing between Task__
 

From 5adf48322088205a42584780680c3e3584e5c435 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Thu, 14 Nov 2024 20:01:51 +0100
Subject: [PATCH 30/37] add release doc 20241114

---
 Readme.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/Readme.md b/Readme.md
index ebcd08b0..08ac3e3d 100644
--- a/Readme.md
+++ b/Readme.md
@@ -170,6 +170,28 @@ For details refer to the [example description](lib/exampletask/Readme.md).
 
 Changelog
 ---------
+[20241114](../../releases/tag/20241114)
+**********
+* UDP writer and reader - [#79](../../issues/79)
+* extensions for [user tasks](lib/exampletask/Readme.md)
+  * extend the Web UI with js and css
+  * register handler for Web Requests
+* Naming of the config file [#87](../../issues/87)
+* MTW from PGN130311 [#83](../../issues/83)
+* USB connection on S3 stops [#81](../../issues/81)
+* remove invalid true wind calc, allow to configure some mapping - partly [#75](../../issues/75)
+* correctly parse GSV messages [#50](../../issues/50)
+* minor adaptations from new version [#66](../../issues/66) 
+* new platform version 6.8.1
+* internally restructure the channel handling
+* add docs for [networking](doc/network.md) and [serial/USB](doc/serial-usb.md)
+* allow to configure the timeout(s) for the data display
+* new library versions - nmea2000 4.22.0, nmea0183 1.10.1
+* allow for builds completely without FastLED
+* wipe the nvs partition on factory reset (to handle corrupted config)
+* do not store the wifi settings in nvs on the system level [#78](../../issues/78)
+* rename of data: HDG->HDT, MHDG->HDM
+* adapt crash decoder tool to s3
 [20240428](../../releases/tag/20240428)
 **********
 * fix build error with M5 gps kit

From bf03de68ac4e570c79c932d416e264f33ab1d10c Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 16:52:11 +0100
Subject: [PATCH 31/37] call requestHandler in user code outside of the API
 lock

---
 lib/usercode/GwUserCode.cpp | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/lib/usercode/GwUserCode.cpp b/lib/usercode/GwUserCode.cpp
index 211eda90..bf4502be 100644
--- a/lib/usercode/GwUserCode.cpp
+++ b/lib/usercode/GwUserCode.cpp
@@ -340,17 +340,23 @@ class TaskApi : public GwApiInternal
     virtual void setCalibrationValue(const String &name, double value){
         api->setCalibrationValue(name,value);
     }
-    virtual bool handleWebRequest(const String &url,AsyncWebServerRequest *req){
-        GWSYNCHRONIZED(&localLock);
-        auto it=webHandlers.find(url);
-        if (it == webHandlers.end()){
-            api->getLogger()->logDebug(GwLog::LOG,"no web handler task=%s url=%s",name.c_str(),url.c_str());
-            return false;
+    virtual bool handleWebRequest(const String &url, AsyncWebServerRequest *req)
+    {
+        GwApi::HandlerFunction handler;
+        {
+            GWSYNCHRONIZED(&localLock);
+            auto it = webHandlers.find(url);
+            if (it == webHandlers.end())
+            {
+                api->getLogger()->logDebug(GwLog::LOG, "no web handler task=%s url=%s", name.c_str(), url.c_str());
+                return false;
+            }
+            handler = it->second;
         }
-        it->second(req);
+        if (handler)
+            handler(req);
         return true;
     }
-
 };
 
 GwUserCode::GwUserCode(GwApiInternal *api,SemaphoreHandle_t *mainLock){

From 22976f08749ab222c47ca69c1daf63ddcbd159d1 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 16:52:25 +0100
Subject: [PATCH 32/37] add USB defines to readme

---
 Readme.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Readme.md b/Readme.md
index 08ac3e3d..db52113f 100644
--- a/Readme.md
+++ b/Readme.md
@@ -192,6 +192,8 @@ Changelog
 * do not store the wifi settings in nvs on the system level [#78](../../issues/78)
 * rename of data: HDG->HDT, MHDG->HDM
 * adapt crash decoder tool to s3
+* allow to set the Pins used for the USB connection (not S3 with integrated PHY) using GWUSB_TX and GWUSB_RX defines
+
 [20240428](../../releases/tag/20240428)
 **********
 * fix build error with M5 gps kit

From abe15281a192328ee305efe37397eae513549de0 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 18:39:38 +0100
Subject: [PATCH 33/37] #71: add BMP280 and env IV to cibuild

---
 lib/hardware/GwM5Grove.in  | 46 ++++++++++++++++++++++--------------
 lib/iictask/GwBMP280.cpp   |  3 +--
 lib/iictask/platformio.ini | 12 +++++++++-
 webinstall/build.yaml      | 48 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 89 insertions(+), 20 deletions(-)

diff --git a/lib/hardware/GwM5Grove.in b/lib/hardware/GwM5Grove.in
index 75dad36a..9c05fbf0 100644
--- a/lib/hardware/GwM5Grove.in
+++ b/lib/hardware/GwM5Grove.in
@@ -63,6 +63,17 @@
   #define _GWQMP6988
 #endif
 
+#GROVE
+#ifdef M5_ENV4$GS$
+  #ifndef M5_GROOVEIIC$GS$
+    #define M5_GROOVEIIC$GS$
+  #endif
+  GROOVE_IIC(SHT3X,$Z$,1)
+  GROOVE_IIC(BMP280,$Z$,1)
+  #define _GWSHT3X
+  #define _BMP280
+#endif
+
 #GROVE
 //example: -DSHT3XG1_A : defines STH3Xn1 on grove A - x depends on the other devices
 #ifdef GWSHT3XG1$GS$
@@ -118,6 +129,24 @@
   #define _GWBME280
 #endif
 
+#GROVE
+#ifdef GWBMP280G1$GS$
+  #ifndef M5_GROOVEIIC$GS$
+    #define M5_GROOVEIIC$GS$
+  #endif
+  GROOVE_IIC(BMP280,$Z$,1)
+  #define _GWBMP280
+#endif
+
+#GROVE
+#ifdef GWBMP280G2$GS$
+  #ifndef M5_GROOVEIIC$GS$
+    #define M5_GROOVEIIC$GS$
+  #endif
+  GROOVE_IIC(BMP280,$Z$,2)
+  #define _GWBMP280
+#endif
+
 #GROVE
 //select up to 2 IIC devices for grove usage
 #ifdef M5_GROOVEIIC$GS$
@@ -144,21 +173,4 @@
   #endif
 #endif
 
-#GROVE
-#ifdef GWBMP280G1$GS$
-  #ifndef M5_GROOVEIIC$GS$
-    #define M5_GROOVEIIC$GS$
-  #endif
-  GROOVE_IIC(BMP280,$Z$,1)
-  #define _GWBMP280
-#endif
-
-#GROVE
-#ifdef GWBMP280G2$GS$
-  #ifndef M5_GROOVEIIC$GS$
-    #define M5_GROOVEIIC$GS$
-  #endif
-  GROOVE_IIC(BMP280,$Z$,2)
-  #define _GWBMP280
-#endif
 
diff --git a/lib/iictask/GwBMP280.cpp b/lib/iictask/GwBMP280.cpp
index 11878bfd..a9efe480 100644
--- a/lib/iictask/GwBMP280.cpp
+++ b/lib/iictask/GwBMP280.cpp
@@ -39,8 +39,7 @@ class BMP280Config : public IICSensorBase{
         }
     virtual bool isActive(){return prAct||tmAct;}
     virtual bool initDevice(GwApi *api,TwoWire *wire){
-        GwLog *logger=api->getLogger();
-	//Wire.begin(GWIIC_SDA,GWIIC_SCL);        
+        GwLog *logger=api->getLogger();  
         device= new Adafruit_BMP280(wire);
         if (! device->begin(addr)){
             LOG_DEBUG(GwLog::ERROR,"unable to initialize %s at %d",prefix.c_str(),addr);
diff --git a/lib/iictask/platformio.ini b/lib/iictask/platformio.ini
index a7857972..09151617 100644
--- a/lib/iictask/platformio.ini
+++ b/lib/iictask/platformio.ini
@@ -11,6 +11,17 @@ build_flags=
     -D M5_CAN_KIT
     ${env.build_flags}
 
+[env:m5stack-atom-env4]
+extends = sensors
+board = m5stack-atom
+lib_deps = 
+    ${env.lib_deps}
+    ${sensors.lib_deps}
+build_flags=
+    -D M5_ENV4
+    -D M5_CAN_KIT
+    ${env.build_flags}
+
 [env:m5stack-atom-bme280]
 extends = sensors
 board = m5stack-atom
@@ -43,7 +54,6 @@ lib_deps =
     ${env.lib_deps}
     ${sensors.lib_deps}
 build_flags=
-    #-D M5_ENVBPS
     -D GWBMP280
     -D M5_GROOVEIIC
     -D M5_CAN_KIT
diff --git a/webinstall/build.yaml b/webinstall/build.yaml
index 569c95cd..176dc6f2 100644
--- a/webinstall/build.yaml
+++ b/webinstall/build.yaml
@@ -51,6 +51,16 @@ types:
           - value: M5_ENV3#grv#
             key: true
             resource: qmp69881#grv#1,sht3x#grv#1
+      - label: "M5 ENV4"
+        type: checkbox
+        key: m5env4#grv#
+        target: define
+        url: "https://docs.m5stack.com/en/unit/ENV%E2%85%A3%20Unit"
+        description: "M5 sensor module temperature, humidity, pressure"
+        values:
+          - value: M5_ENV4#grv#
+            key: true
+            resource: bmp280#grv#1,sht3x#grv#1
       - type: checkbox
         label: SHT3X-1
         description: "SHT30 temperature and humidity sensor 0x44"
@@ -111,6 +121,26 @@ types:
           - key: true
             value: GWBME280G2#grv#
             resource: bme280#grv#2
+      - type: checkbox
+        label: BMP280-1
+        description: "BMP280 temperature/humidity/pressure sensor 0x76"
+        key: bmp2801g1
+        target: define
+        url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
+        values: 
+          - key: true
+            value: GWBMP280G1#grv#
+            resource: bmp280#grv#1
+      - type: checkbox
+        label: BMP280-2
+        description: "BMP280 temperature/humidity/pressure sensor 0x77"
+        key: bmp2802
+        target: define
+        url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
+        values: 
+          - key: true
+            value: GWBMP280G2#grv#
+            resource: bmp280#grv#2
   
 
   - &m5groovecan
@@ -498,6 +528,24 @@ types:
             values: 
               - key: true
                 value: GWBME280#busname#2
+          - type: checkbox
+            label: BMP280-#busname#-1
+            description: "BMP280 temperature/humidity/pressure sensor 0x76"
+            key: bmp2801
+            target: define
+            url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
+            values: 
+              - key: true
+                value: GWBMP280#busname#1
+          - type: checkbox
+            label: BMP280-#busname#-2
+            description: "BME280 temperature/humidity/pressure sensor 0x77"
+            key: bmp2802
+            target: define
+            url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
+            values: 
+              - key: true
+                value: GWBMP280#busname#2
   
   - &spisensors
     type: checkbox

From d86619db1a94d770ae67ab24626a6495ca147872 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 19:34:58 +0100
Subject: [PATCH 34/37] correct descriptions for BMP280 in cibuild

---
 webinstall/build.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/webinstall/build.yaml b/webinstall/build.yaml
index 176dc6f2..7af26792 100644
--- a/webinstall/build.yaml
+++ b/webinstall/build.yaml
@@ -123,7 +123,7 @@ types:
             resource: bme280#grv#2
       - type: checkbox
         label: BMP280-1
-        description: "BMP280 temperature/humidity/pressure sensor 0x76"
+        description: "BMP280 temperature/pressure sensor 0x76"
         key: bmp2801g1
         target: define
         url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
@@ -133,7 +133,7 @@ types:
             resource: bmp280#grv#1
       - type: checkbox
         label: BMP280-2
-        description: "BMP280 temperature/humidity/pressure sensor 0x77"
+        description: "BMP280 temperature/pressure sensor 0x77"
         key: bmp2802
         target: define
         url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
@@ -530,7 +530,7 @@ types:
                 value: GWBME280#busname#2
           - type: checkbox
             label: BMP280-#busname#-1
-            description: "BMP280 temperature/humidity/pressure sensor 0x76"
+            description: "BMP280 temperature/pressure sensor 0x76"
             key: bmp2801
             target: define
             url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"
@@ -539,7 +539,7 @@ types:
                 value: GWBMP280#busname#1
           - type: checkbox
             label: BMP280-#busname#-2
-            description: "BME280 temperature/humidity/pressure sensor 0x77"
+            description: "BMP280 temperature/pressure sensor 0x77"
             key: bmp2802
             target: define
             url: "https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf"

From 757ef93db4e32138adecbd13515ec00cb5e13f65 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 21:35:38 +0100
Subject: [PATCH 35/37] correct defines for BMP280

---
 lib/hardware/GwM5Grove.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/hardware/GwM5Grove.in b/lib/hardware/GwM5Grove.in
index 9c05fbf0..e0b634f2 100644
--- a/lib/hardware/GwM5Grove.in
+++ b/lib/hardware/GwM5Grove.in
@@ -71,7 +71,7 @@
   GROOVE_IIC(SHT3X,$Z$,1)
   GROOVE_IIC(BMP280,$Z$,1)
   #define _GWSHT3X
-  #define _BMP280
+  #define _GWBMP280
 #endif
 
 #GROVE

From 7a6eba4b86791234d5d341ca56b0cca29b7f1b66 Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 21:37:15 +0100
Subject: [PATCH 36/37] use groove def for env bps

---
 lib/iictask/platformio.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/iictask/platformio.ini b/lib/iictask/platformio.ini
index 09151617..609ef5a2 100644
--- a/lib/iictask/platformio.ini
+++ b/lib/iictask/platformio.ini
@@ -54,7 +54,7 @@ lib_deps =
     ${env.lib_deps}
     ${sensors.lib_deps}
 build_flags=
-    -D GWBMP280
+    -D GWBMP280G1
     -D M5_GROOVEIIC
     -D M5_CAN_KIT
     ${env.build_flags}

From 70fb1b363363cd4518d64f8f0cab0a846bd77c2c Mon Sep 17 00:00:00 2001
From: andreas <andreas@wellenvogel.de>
Date: Fri, 15 Nov 2024 21:39:35 +0100
Subject: [PATCH 37/37] make BMP280 working with groove config

---
 lib/iictask/GwBME280.cpp | 1 +
 lib/iictask/GwBMP280.cpp | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/iictask/GwBME280.cpp b/lib/iictask/GwBME280.cpp
index cc241134..6eb48477 100644
--- a/lib/iictask/GwBME280.cpp
+++ b/lib/iictask/GwBME280.cpp
@@ -140,6 +140,7 @@ class BME280Config : public IICSensorBase{
             busId = 2;
             addr = 0x77;
             CFG280(BME28022);
+            ok=true;
         }
         intv *= 1000;
     }
diff --git a/lib/iictask/GwBMP280.cpp b/lib/iictask/GwBMP280.cpp
index a9efe480..c1b1729e 100644
--- a/lib/iictask/GwBMP280.cpp
+++ b/lib/iictask/GwBMP280.cpp
@@ -2,8 +2,6 @@
 #ifdef _GWIIC
     #if defined(GWBMP280) || defined(GWBMP28011) || defined(GWBMP28012)|| defined(GWBMP28021)|| defined(GWBMP28022)
         #define _GWBMP280
-    #else
-        #undef _GWBMP280
     #endif
 #else
     #undef _GWBMP280
@@ -126,6 +124,7 @@ class BMP280Config : public IICSensorBase{
             busId = 2;
             addr = 0x77;
             CFGBMP280(BMP28022);
+            ok=true;
         }
         intv *= 1000;
     }