From 12e444a2aa40602c606f4e7d93f7ddc8a499aee1 Mon Sep 17 00:00:00 2001 From: "Wong, Vincent Por Yin" Date: Mon, 12 Oct 2020 16:40:46 +0800 Subject: [PATCH] Initial commit of the new IOTG TSN Reference Software This is the first commit, comprising of over 2 years of hard work, thanks to the many individuals listed below. This commit introduces 3 separate C-applications and a lot of new scripts to enable users to try it out quickly. As with the older version, this reference software is specific to certain Intel IOTG platforms only - which currently does not include Apollo Lake I and the i210 Ethernet controller. As this is an initial commit, some README and instructions may be incomplete, and near-future commits will address them. Co-authored-by: Noor Azura Ahmad Tarmizi Co-authored-by: Khor, Isaac Shi Yan Co-authored-by: Singh, Sartaj Signed-off-by: Ong, Boon Leong Signed-off-by: Wong, Vee Khee Signed-off-by: Kweh Hock Leong Signed-off-by: Weifeng Voon Signed-off-by: Song, Yoong Siang --- .gitignore | 33 ++ LICENSE.txt | 30 + README.md | 105 ++++ README_json.md | 184 ++++++ README_sh.md | 118 ++++ build.sh | 58 ++ json/ehl/opcua-pkt1a-tsn.json.i | 64 +++ json/ehl/opcua-pkt1a.json.i | 25 + json/ehl/opcua-pkt1b-tsn.json.i | 31 + json/ehl/opcua-pkt1b.json.i | 25 + json/ehl2/opcua-pkt2a-tsn.json.i | 78 +++ json/ehl2/opcua-pkt2a.json.i | 38 ++ json/ehl2/opcua-pkt2b-tsn.json.i | 77 +++ json/ehl2/opcua-pkt2b.json.i | 38 ++ json/ehl2/opcua-pkt3a-tsn.json.i | 85 +++ json/ehl2/opcua-pkt3a.json.i | 38 ++ json/ehl2/opcua-pkt3b-tsn.json.i | 84 +++ json/ehl2/opcua-pkt3b.json.i | 38 ++ json/ehl2/opcua-xdp2a-tsn.json.i | 101 ++++ json/ehl2/opcua-xdp2a.json.i | 38 ++ json/ehl2/opcua-xdp2b-tsn.json.i | 101 ++++ json/ehl2/opcua-xdp2b.json.i | 38 ++ json/ehl2/opcua-xdp3a-tsn.json.i | 108 ++++ json/ehl2/opcua-xdp3a.json.i | 38 ++ json/ehl2/opcua-xdp3b-tsn.json.i | 108 ++++ json/ehl2/opcua-xdp3b.json.i | 38 ++ json/gen_setup.py | 417 ++++++++++++++ json/helpers.py | 39 ++ json/helpers.sh | 445 ++++++++++++++ json/opcua-run.sh | 236 ++++++++ json/run.sh | 53 ++ json/tgl/opcua-pkt1a-tsn.json.i | 60 ++ json/tgl/opcua-pkt1a.json.i | 25 + json/tgl/opcua-pkt1b-tsn.json.i | 30 + json/tgl/opcua-pkt1b.json.i | 25 + run.sh | 117 ++++ scripts/clock-setup.sh | 71 +++ scripts/enable_extts.sh | 45 ++ scripts/enable_pps.sh | 52 ++ scripts/gPTP.cfg | 22 + scripts/helpers.sh | 301 ++++++++++ scripts/iperf3-bg-client.sh | 51 ++ scripts/iperf3-bg-server.sh | 47 ++ scripts/irq_affinity_4c_4tx_4rx.map | 6 + scripts/irq_affinity_4c_8tx_8rx-multi.map | 6 + scripts/irq_affinity_4c_8tx_8rx.map | 6 + scripts/latency_dual.gnu | 109 ++++ scripts/latency_single.gnu | 91 +++ scripts/liveplot.gnu | 36 ++ scripts/setup-tsq1a.sh | 47 ++ scripts/setup-tsq1b.sh | 46 ++ scripts/setup-vs1a.sh | 51 ++ scripts/setup-vs1b.sh | 51 ++ scripts/setup_etf.sh | 54 ++ scripts/setup_mqprio.sh | 40 ++ scripts/setup_taprio.sh | 71 +++ scripts/setup_vlanrx.sh | 45 ++ scripts/tsq1a.sh | 80 +++ scripts/tsq1b.sh | 61 ++ scripts/vs1a.sh | 101 ++++ scripts/vs1b.sh | 108 ++++ src/Makefile | 29 + src/opcua-tsn/CMakeLists.txt | 25 + src/opcua-tsn/json_helper.c | 186 ++++++ src/opcua-tsn/json_helper.h | 52 ++ src/opcua-tsn/multicallback_server.c | 230 ++++++++ src/opcua-tsn/opcua_common.c | 334 +++++++++++ src/opcua-tsn/opcua_common.h | 113 ++++ src/opcua-tsn/opcua_custom.c | 319 ++++++++++ src/opcua-tsn/opcua_custom.h | 171 ++++++ src/opcua-tsn/opcua_datasource.c | 356 ++++++++++++ src/opcua-tsn/opcua_datasource.h | 79 +++ src/opcua-tsn/opcua_publish.c | 256 +++++++++ src/opcua-tsn/opcua_publish.h | 45 ++ src/opcua-tsn/opcua_subscribe.c | 206 +++++++ src/opcua-tsn/opcua_subscribe.h | 41 ++ src/opcua-tsn/opcua_utils.h | 60 ++ src/tsq.c | 671 ++++++++++++++++++++++ src/txrx-afpkt.c | 579 +++++++++++++++++++ src/txrx-afpkt.h | 48 ++ src/txrx-afxdp.c | 630 ++++++++++++++++++++ src/txrx-afxdp.h | 52 ++ src/txrx.c | 417 ++++++++++++++ src/txrx.h | 162 ++++++ 84 files changed, 9825 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 README_json.md create mode 100644 README_sh.md create mode 100755 build.sh create mode 100644 json/ehl/opcua-pkt1a-tsn.json.i create mode 100644 json/ehl/opcua-pkt1a.json.i create mode 100644 json/ehl/opcua-pkt1b-tsn.json.i create mode 100644 json/ehl/opcua-pkt1b.json.i create mode 100644 json/ehl2/opcua-pkt2a-tsn.json.i create mode 100644 json/ehl2/opcua-pkt2a.json.i create mode 100644 json/ehl2/opcua-pkt2b-tsn.json.i create mode 100644 json/ehl2/opcua-pkt2b.json.i create mode 100644 json/ehl2/opcua-pkt3a-tsn.json.i create mode 100644 json/ehl2/opcua-pkt3a.json.i create mode 100644 json/ehl2/opcua-pkt3b-tsn.json.i create mode 100644 json/ehl2/opcua-pkt3b.json.i create mode 100644 json/ehl2/opcua-xdp2a-tsn.json.i create mode 100644 json/ehl2/opcua-xdp2a.json.i create mode 100644 json/ehl2/opcua-xdp2b-tsn.json.i create mode 100644 json/ehl2/opcua-xdp2b.json.i create mode 100644 json/ehl2/opcua-xdp3a-tsn.json.i create mode 100644 json/ehl2/opcua-xdp3a.json.i create mode 100644 json/ehl2/opcua-xdp3b-tsn.json.i create mode 100644 json/ehl2/opcua-xdp3b.json.i create mode 100644 json/gen_setup.py create mode 100644 json/helpers.py create mode 100644 json/helpers.sh create mode 100755 json/opcua-run.sh create mode 100755 json/run.sh create mode 100644 json/tgl/opcua-pkt1a-tsn.json.i create mode 100644 json/tgl/opcua-pkt1a.json.i create mode 100644 json/tgl/opcua-pkt1b-tsn.json.i create mode 100644 json/tgl/opcua-pkt1b.json.i create mode 100755 run.sh create mode 100755 scripts/clock-setup.sh create mode 100755 scripts/enable_extts.sh create mode 100755 scripts/enable_pps.sh create mode 100644 scripts/gPTP.cfg create mode 100644 scripts/helpers.sh create mode 100755 scripts/iperf3-bg-client.sh create mode 100755 scripts/iperf3-bg-server.sh create mode 100644 scripts/irq_affinity_4c_4tx_4rx.map create mode 100644 scripts/irq_affinity_4c_8tx_8rx-multi.map create mode 100644 scripts/irq_affinity_4c_8tx_8rx.map create mode 100755 scripts/latency_dual.gnu create mode 100755 scripts/latency_single.gnu create mode 100644 scripts/liveplot.gnu create mode 100755 scripts/setup-tsq1a.sh create mode 100755 scripts/setup-tsq1b.sh create mode 100755 scripts/setup-vs1a.sh create mode 100755 scripts/setup-vs1b.sh create mode 100755 scripts/setup_etf.sh create mode 100755 scripts/setup_mqprio.sh create mode 100755 scripts/setup_taprio.sh create mode 100755 scripts/setup_vlanrx.sh create mode 100755 scripts/tsq1a.sh create mode 100755 scripts/tsq1b.sh create mode 100755 scripts/vs1a.sh create mode 100755 scripts/vs1b.sh create mode 100644 src/Makefile create mode 100644 src/opcua-tsn/CMakeLists.txt create mode 100644 src/opcua-tsn/json_helper.c create mode 100644 src/opcua-tsn/json_helper.h create mode 100644 src/opcua-tsn/multicallback_server.c create mode 100644 src/opcua-tsn/opcua_common.c create mode 100644 src/opcua-tsn/opcua_common.h create mode 100644 src/opcua-tsn/opcua_custom.c create mode 100644 src/opcua-tsn/opcua_custom.h create mode 100644 src/opcua-tsn/opcua_datasource.c create mode 100644 src/opcua-tsn/opcua_datasource.h create mode 100644 src/opcua-tsn/opcua_publish.c create mode 100644 src/opcua-tsn/opcua_publish.h create mode 100644 src/opcua-tsn/opcua_subscribe.c create mode 100644 src/opcua-tsn/opcua_subscribe.h create mode 100644 src/opcua-tsn/opcua_utils.h create mode 100644 src/tsq.c create mode 100644 src/txrx-afpkt.c create mode 100644 src/txrx-afpkt.h create mode 100644 src/txrx-afxdp.c create mode 100644 src/txrx-afxdp.h create mode 100644 src/txrx.c create mode 100644 src/txrx.h diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f3cbed0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Editing and running artifacts +*.plot +*.eps +*.log +*.o +*.a +tags +*.vim +*.swp +*.un +*~ +*.pcap + +# Build artifacts +tsq +txrx-tsn + +# Skip result files +results-*/ +*.txt +*.png + +# Ignore convenience symlink +a5 +a6 + +# We only save json.i +*.json + +opcua-server +src/build-opcua/ +iperf3-gen-cmd.sh +json/__pycache__/ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..54866a9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2020, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd4549c --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +# IOTG TSN Reference Software for Linux + +## Overview + +This project is a set of scripts and C-applications used to showcase specific +__Ethernet-TSN__ features and workloads. Its many components can be used individually +in a variety of ways but this project will use them specifically for the +below 3 applications and their dedicated/respective examples (aka configs): + +* **TSQ** [Time Synchronization Quality Measurement] + * __tsq1__ showcases the nanosecond-precision time synchronization between platforms + using pulse-per-second(PPS) output & auxiliary timestamping(AUX_TS) input pins. + +* **TXRX-TSN** [AF_PACKET & AF_XDP socket-based application] + * __vs1__ showcases the bounded low-latency transmission/reception achievable via AF_PACKET + & AF_XDP standard linux socket interfaces, while leveraging various + device-specific Ethernet-TSN features. + +* **OPCUA-SERVER** [AF_PACKET & AF_XDP OPCUA-based application] + * __opcua-pkt1__ showcases the bounded low-latency + transmission/reception achievable via AF_PACKET over libopen62541 + (an OPC-UA-based library) APIs, while leveraging various device-specific + Ethernet-TSN features over a single-ethernet connection. This application + uses JSON files as an input. + * __opcua-pkt2__, __opcua-pkt3__, __opcua-xdp2__ & __opcua-xdp3__ showcases + the bounded low-latency transmission/reception achievable via AF_PACKET & + AF_XDP over libopen62541 (an OPC-UA-based library) APIs, while leveraging + various device-specific Ethernet-TSN features over **two** ethernet + connection.This application uses JSON files as an input and require platforms + with at least 2 Ethernet-TSN ports (e.g. EHL). Users can choose either __opcua-*2__ + or __opcua-*3__ to enable/disable background (iperf3) traffic. + +Refer to the full documentation for details as this README serves as a +high-level overview only. Many specifics will not be present here due to the +project's complexity. + +## Disclaimers + +This project serves as an example to demonstrate TSN functionality only on the +supported platforms. This project is not for intended for production use. + +Users are responsible for their own products functionality and performance. +Refer to LICENSE.txt for the license used by this project. + +Each example defaults to a set of parameter values that has been tested on the +supported platforms to achieve ideal bounded-latency for each packet while the +platform is running on a preempt-rt kernel. + +## Dependencies + +This project is designed to work on specific Intel hardware and their +respective bundled software (e.g. IFWI, BSP, preempt-rt kernel). These examples +may not perform as expected when run on other platforms or configurations. +Currently supported platforms are: + +* Elkhart Lake +* Tiger Lake UP3 + +These packages are required to be installed for the scripts to run. (Note: If +you are running from an Intel-provided Yocto BSP, these dependencies would have +already been installed): + +* linuxptp - clock synchronization utilties (e.g. ptp4l & phc2sys) +* gnuplot +* python (3.7.7) +* libopen62541 +* libbpf +* libelf +* librt +* libpthread +* libjson +* iperf3 + +### IceWM shortcuts: + +A GUI/window manager should be used while executing these applications. The Yocto BSP +uses IceWM as its default window manager. Here are some keyboard shortcuts, +in case a mouse is not available. + +Notation: C is control, M is meta/alt, S is shift, s is super + +1. `` to open xterm +2. `` to resize window (using arrow keys) +3. `` to maximise vertical space +4. `` to move window +5. `` to move to the left, rightarrow for right +6. `` to show window list +7. `` to show window actions menu + +## Build the project + +In a Yocto BSP with IOTG TSN Ref Sw included: + +1. To build all 3 applications: + +```bash +cd /usr/share/iotg-tsn-ref-sw/ +./build.sh +``` + +## Run the examples + +For instructions to run TSQ or TXRX-TSN,refer to [README_sh.md](README_sh.md) + +For instructions to run OPCUA-SERVER, refer to [README_json.md](README_json.md) diff --git a/README_json.md b/README_json.md new file mode 100644 index 0000000..f4a4bad --- /dev/null +++ b/README_json.md @@ -0,0 +1,184 @@ +# Running examples using JSON + +All examples are run with 2 units of the same platform. Mind the notation +"[Board A or B]". The following steps assumes both platforms are connected +to each other via an Ethernet connection and user has a terminal open in the +/usr/share/iotg-tsn-ref-sw directory - with the C-applications already built. + +## Role of run.sh & JSON + +Unlike TSQ and TXRX-TSN which are regular C-application that takes in input via +parameter lists. OPCUA-SERVER uses JSON files for input. This is to allow OPCUA-SERVER +to have more customizable parameters and enable pre-written example configurations +in the form of JSON. To make it even easier, all Linux configurations are also +customizable via JSON file. + +Run.sh serves only as a shortcut to call the desired scripts and pass them +the JSON files. Each pre-defined example has either a *.json.i or *-tsn.json.i +file for either Board A or Board B. + +Important: when using run.sh, users should modify the corresponding +**.json.i** file NOT the generated .json file. **.json.i** denotes incomplete or +interim json file as it has fields intentionally left blank for run.sh to fill in +automatically. + +When a user executes run.sh with a command (init/setup/run), the following scripts +are called respectively: + +```bash + +#Init only +run.sh [iface2] init + |__ json/opcua-run.sh #create a json using correct iface(s) + |__ json/helpers.sh #set ip/mac address, single/dual port + +#Setup only +run.sh [iface2] setup + |__ json/opcua-run.sh #create a json using correct iface(s) + |__ json/gen_setup.py #parse json and generate setup scripts + |__ generated_setup.sh #set up clocks, different TCs + |__ ptp4l + |__ phc2sys + |__ tc + +#Run only +run.sh [iface2] run + |__ json/opcua-run.sh #create a json using correct iface(s) + |__ ./iperf-gen-cmd.sh #run iperf if required + |__ ./opcua-server #use new json opcua parameters to execute + |__ gnuplot & save results +``` + +## OPCUA-SERVER - AF_PACKET & AF_XDP OPCUA-based application + +Refer to the full documentation for details as this README will serve as an +overview only. + +OPCUA-SERVER is a C-application that can transmit and receive ETH_UADP packets using +AF_PACKET or AF_XDP sockets using libopen62541 (OPCUA-based library) APIs. It +accepts only 1 .json.i file as input and supported .json.i entries are as follows +(kindly refer to the examples in json// for examples of its structure tree): + +| Category | Field | Type | Min | Max | +| ----------------- | ---------------------------- | ------- | ------ | -------- | +| Mandatory/General | publisher_interface | String | | | +| Mandatory/General | subscriber_interface | String | | | +| Mandatory/General | use_xdp | Boolean | | | +| Mandatory/General | packet_count | Int | 1 | 10000000 | +| Mandatory/General | cycle_time_ns | Int | 100000 | 5000000 | +| Mandatory/General | polling_duration_ns | Int | 0 | 10000000 | +| Publisher | url | String | | | +| Publisher | pub_id | String | | | +| Publisher | dataset_writer_id | Int | 0 | 99999 | +| Publisher | writer_group_id | Int | 0 | 99999 | +| Publisher | early_offset_ns | Int | 0 | 1000000 | +| Publisher | publish_offset_ns | Int | 0 | 10000000 | +| Publisher | socket_prio | Int | 0 | 7 | +| Publisher | two_way_data | Boolean | | | +| Publisher | cpu_affinity | Int | 0 | 3 | +| Publisher | xdp_queue | Int | 1 | 3 | +| Subscriber | url | String | | | +| Subscriber | sub_id | String | 0 | 99999 | +| Subscriber | subscribed_pub_id | Int | 0 | 99999 | +| Subscriber | subscribed_dataset_writer_id | Int | 0 | 99999 | +| Subscriber | subscribed_writer_group_id | Int | 0 | 99999 | +| Subscriber | offset_ns | Int | 0 | 10000000 | +| Subscriber | subscriber_output_file | String | | | +| Subscriber | two_way_data | Boolean | | | +| Subscriber | cpu_affinity | Int | 0 | 3 | +| Subscriber | xdp_queue | Int | 1 | 3 | + +### Single-port example + +Similar to TSQ & TXRX-TSN examples, 2 platforms are required to be connected to +each other via a single-ethernet connection. + +1. [Board A] Run the setup script to initialize IP and MAC address, then start + clock synchronization and setup TAPRIO + ETF qdiscs. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE opcua-pkt1a init + ./run.sh $IFACE opcua-pkt1a setup + ``` + +2. [Board A] Run the setup script to initialize IP and MAC address, then start + clock synchronization and setup ingress qdisc. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE opcua-pkt1b init + ./run.sh $IFACE opcua-pkt1b setup + ``` + +3. [Board B] Start listening for packets. + + ```bash + ./run.sh $IFACE opcua-pkt1b run + ``` + +4. [Board B] Immediately after step 3, start transmitting packets. The + application will terminate itself after completion. Note that AF_PACKET or + opcua-pkt* configurations will take longer as they use much higher packet + intervals compared to AF_XDP or opuca-xdp* configurations. + + ```bash + ./run.sh $IFACE opcua-pkt1a run + ``` + +5. [Board B] A gnuplot window should appear showing the time taken for each + packet to traverse from the application in Board A to Board B. + + Each run will enumerate and store its raw timestamps/data & plot images in the + results- folder for easy reference, repeat steps 3-4 as needed. + +### Dual-port example + +In this example, 2 platforms are required to be connected to each other via +**two** ethernet connections. Meaning there is a total of four +ethernet interfaces (as opposed to the usual two). + +A simplified explanation of the data flow: +1. Board A's first interface ($IFACE1) is used to transmit +2. Board B's first interface ($IFACE1) is used to receive +3. Board B's second interface ($IFACE2) is used to transmit back to Board A +4. Board A's second interface ($IFACE2) is used to receive + +When using dual-ports, the should be replaced with "ehl2". (TGL not supported) + +1. [Board A] Run the setup script to initialize IP and MAC address, then start + clock synchronization and setup TAPRIO + ETF qdiscs. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE $IFACE2 opcua-pkt3a init + ./run.sh $IFACE $IFACE2 opcua-pkt3a setup + ``` + +2. [Board B] Run the setup script to initialize IP and MAC address, then start + clock synchronization and setup ingress qdisc. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE $IFACE2 opcua-pkt3b init + ./run.sh $IFACE $IFACE2 opcua-pkt3b setup + ``` + +3. [Board B] Start listening for packets. + + ```bash + ./run.sh $IFACE $IFACE2 opcua-pkt3b run + ``` + +4. [Board A] Immediately after step 3, start transmitting packets. + + ```bash + ./run.sh $IFACE opcua-pkt3a run + ``` + +5. [Board A] A gnuplot window should appear showing the time taken for each + packet to traverse from the application in Board A to Board B and back to + Board A. + + Each run will enumerate and store its raw timestamps/data & plot images in the + results- folder for easy reference, repeat steps 3-4 as needed. diff --git a/README_sh.md b/README_sh.md new file mode 100644 index 0000000..5b7a256 --- /dev/null +++ b/README_sh.md @@ -0,0 +1,118 @@ +# Run an example + +All examples are run with 2 units of the same platform. Mind the notation +"[Board A or B]". The following steps assumes both platforms are connected +to each other via an Ethernet connection and user has a terminal open in the +/usr/share/iotg-tsn-ref-sw directory - with the C-applications already built. + +## Role of run.sh + +Run.sh serves only as a shortcut to call the desired scripts. +The following section shows how to use it to run TSQ and TXRX-TSN examples. + +Each example has a corresponding setup and run script for either +Board A or Board B. Curious users can review these scripts on what commands +are actually being run. + +## TSQ - Time Synchronization Quality Measurement + +Refer to the full documentation for hardware setup as this example requires +multiple pin header connections for (pulse-per-second) PPS and (auxiliary +timestamping) AUXTS. + +The TSQ C-application retrieves externally-triggered hardware timestamps (by +PPS) and calculates the difference between the timestamps collected from two +platforms - which have their PTP clocks synchronized by PTP4L. To see what other +parameters can be used when calling TSQ, refer to: ./tsq --help + +Note that TSQ is just a C-application and features such as AUXTS and PPS require +shell commands to enable/disable. setup-tsq1* & tsq1 scripts is used to perform +both the setting up and execution of the TSQ application. + +1. [Board A] Run the setup script to configure IP and MAC address, start ptp4l + , enable pulse-per-second and external timestamping. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE setup-tsq1a + ``` + +2. [Board B] Run the setup script to configure IP and MAC address, start ptp4l + and enable external timestamping. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE setup-tsq1b + ``` + +3. [Board A] Start the talker and listener. + + ```bash + ./run.sh $IFACE tsq1a + ``` + +4. [Board B] Immediately after step 3, start the talker. + + ```bash + ./run.sh $IFACE tsq1b + ``` + +5. [Board A] A gnuplot window should appear indicating the difference between + the external timestamps obtained from Board A & B. In an ideal environment, + the difference should be 0 nanoseconds all the time. With ptp4l, the + difference should be within the +/- 50 nanoseconds range. Both Board A & + B's TSQ application will stop after 50 seconds. + +## TXRX-TSN - AF_PACKET & AF_XDP socket-based application + +Refer to the full documentation for details as this README will serve as an +overview only. + +TXRX-TSN is a simple C-application that can transmit and receive packets using +AF_PACKET or AF_XDP sockets. To see what parameters can be used when calling TXRX-TSN, refer to: ./txrx-tsn --help + +Note that TXRX-TSN is just a C-application and setup-vs1* & vs1* scripts are +used in this example, to perform both the setting up and execution of TXRX-TSN. + +Specifically in this example, TXRX-TSN is used to transmit and receive packets using AF_PACKET +first, and then repeat using AF_XDP - to show the time taken for each packet to traverse from the +application in Board A to Board B when IPERF3 is running in the background and TSN features enabled. + +1. [Board A] Run the setup script to configure IP and MAC address, start clock + synchronization and setup TAPRIO qdisc. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE setup-vs1a + ``` + +2. [Board B] Run the setup script to configure IP and MAC address, start clock + synchronization and setup ingress qdiscs. + + ```bash + cd /usr/share/iotg-tsn-ref-sw/ + ./run.sh $IFACE setup-vs1b + ``` + +3. [Board B] Start listening for packets using AF_PACKET, it will automatically + re-start the application using AF_XDP socket after it has completed. This + duration will be printed on the shell line: "Phase xxx (N-duration seconds)" + + ```bash + ./run.sh $IFACE vs1b + ``` + +4. [Board A] Immediately after step 3, start transmitting packets using AF_PACKET. + it will automatically re-start the application and trasnmit using AF_XDP + socket after 30 seconds. + + ```bash + ./run.sh $IFACE vs1a + ``` + +5. [Board B] A gnuplot window should appear showing the time taken for each + packet to traverse from the application in Board A to Board B. AF_XDP sockets + should show an improvement in both overall average and jitter. + + Each run will enumerate and store its raw timestamps/data & plot images in the + results- folder for easy reference, repeat steps 3-4 as needed. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..343d89a --- /dev/null +++ b/build.sh @@ -0,0 +1,58 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +make -C src/ clean + +make -C src/ + +mv src/txrx-tsn . +if [ $? -ne 0 ]; then + echo "txrx-tsn build failed" && exit +fi + +mv src/tsq . +if [ $? -ne 0 ]; then + echo "tsq build failed" && exit +fi + +mkdir -p src/build-opcua +rm -rf src/build-opcua/* + +cd src/build-opcua/ +cmake ../opcua-tsn +make -j$(nproc) + +test -e opcua-server +if [ $? -ne 0 ]; then + echo "opcua-server build failed" && exit +fi +mv opcua-server ../../ diff --git a/json/ehl/opcua-pkt1a-tsn.json.i b/json/ehl/opcua-pkt1a-tsn.json.i new file mode 100644 index 0000000..ea3891a --- /dev/null +++ b/json/ehl/opcua-pkt1a-tsn.json.i @@ -0,0 +1,64 @@ +{ + "ptp": { + "interface": "_PREPROCESS_STR_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "phc2sys": { + "interface": "_PREPROCESS_STR_interface", + "clock": "CLOCK_REALTIME", + "ignore_existing": true + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 7, + "queues": "1@0 1@1 1@2 1@3 1@4 1@5 1@6", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3, + "p4": 4, + "p5": 5, + "p6": 6 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 500000 + }, + { + "gate_mask": "42", + "duration": 500000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 300000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} diff --git a/json/ehl/opcua-pkt1a.json.i b/json/ehl/opcua-pkt1a.json.i new file mode 100644 index 0000000..83d5363 --- /dev/null +++ b/json/ehl/opcua-pkt1a.json.i @@ -0,0 +1,25 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": false, + "packet_count": 1000000, + "cycle_time_ns": 1000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.6", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 800000, + "publish_offset_ns": 100, + "socket_prio": 6, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": {} + }, +} diff --git a/json/ehl/opcua-pkt1b-tsn.json.i b/json/ehl/opcua-pkt1b-tsn.json.i new file mode 100644 index 0000000..c44e2d0 --- /dev/null +++ b/json/ehl/opcua-pkt1b-tsn.json.i @@ -0,0 +1,31 @@ +{ + "ptp": { + "interface": "_PREPROCESS_STR_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "phc2sys": { + "interface": "_PREPROCESS_STR_interface", + "clock": "CLOCK_REALTIME", + "ignore_existing": true + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} diff --git a/json/ehl/opcua-pkt1b.json.i b/json/ehl/opcua-pkt1b.json.i new file mode 100644 index 0000000..ae70b61 --- /dev/null +++ b/json/ehl/opcua-pkt1b.json.i @@ -0,0 +1,25 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": false, + "packet_count": 1000000, + "cycle_time_ns": 1000000, + "polling_duration_ns": 0, + "publishers": {}, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 50000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + } + }, +} diff --git a/json/ehl2/opcua-pkt2a-tsn.json.i b/json/ehl2/opcua-pkt2a-tsn.json.i new file mode 100644 index 0000000..0f85d8e --- /dev/null +++ b/json/ehl2/opcua-pkt2a-tsn.json.i @@ -0,0 +1,78 @@ +{ + "custom_sync_a": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 7, + "queues": "1@0 1@1 1@2 1@3 1@4 1@5 1@6", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3, + "p4": 4, + "p5": 5, + "p6": 6 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 1000000 + }, + { + "gate_mask": "42", + "duration": 1000000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "etf": [ + { + "delta": 400000, + "queue": 6, + "offload": true + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} + diff --git a/json/ehl2/opcua-pkt2a.json.i b/json/ehl2/opcua-pkt2a.json.i new file mode 100644 index 0000000..4f6a5bb --- /dev/null +++ b/json/ehl2/opcua-pkt2a.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": false, + "packet_count": 500000, + "cycle_time_ns": 2000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.6", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 700000, + "publish_offset_ns": 100, + "socket_prio": 6, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://aa-11-aa-11-aa-11", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 1050000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": true, + "cpu_affinity": 3, + "xdp_queue": -1 + } + } + }, +} diff --git a/json/ehl2/opcua-pkt2b-tsn.json.i b/json/ehl2/opcua-pkt2b-tsn.json.i new file mode 100644 index 0000000..d5b35ff --- /dev/null +++ b/json/ehl2/opcua-pkt2b-tsn.json.i @@ -0,0 +1,77 @@ +{ + "custom_sync_b": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 7, + "queues": "1@0 1@1 1@2 1@3 1@4 1@5 1@6", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3, + "p4": 4, + "p5": 5, + "p6": 6 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 1000000 + }, + { + "gate_mask": "42", + "duration": 1000000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 400000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} diff --git a/json/ehl2/opcua-pkt2b.json.i b/json/ehl2/opcua-pkt2b.json.i new file mode 100644 index 0000000..5a44b3a --- /dev/null +++ b/json/ehl2/opcua-pkt2b.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "subscriber_interface": "_PREPROCESS_STR_interface", + "publisher_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": false, + "packet_count": 500000, + "cycle_time_ns": 2000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.6", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 700000, + "publish_offset_ns": 1000100, + "socket_prio": 6, + "two_way_data": true, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 50000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 3, + "xdp_queue": -1 + } + } + }, +} diff --git a/json/ehl2/opcua-pkt3a-tsn.json.i b/json/ehl2/opcua-pkt3a-tsn.json.i new file mode 100644 index 0000000..c2d4f74 --- /dev/null +++ b/json/ehl2/opcua-pkt3a-tsn.json.i @@ -0,0 +1,85 @@ +{ + "custom_sync_a": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "iperf3": { + "cpu_affinity": 0, + "run_server": true, + "client_target_address": "169.254.1.22", + "client_runtime_in_sec": 5000, + "client_bandwidth_in_mbps": 200 + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 7, + "queues": "1@0 1@1 1@2 1@3 1@4 1@5 1@6", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3, + "p4": 4, + "p5": 5, + "p6": 6 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 1000000 + }, + { + "gate_mask": "42", + "duration": 1000000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "etf": [ + { + "delta": 400000, + "queue": 6, + "offload": true + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} + diff --git a/json/ehl2/opcua-pkt3a.json.i b/json/ehl2/opcua-pkt3a.json.i new file mode 100644 index 0000000..5138a13 --- /dev/null +++ b/json/ehl2/opcua-pkt3a.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": false, + "packet_count": 500000, + "cycle_time_ns": 2000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.6", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 800000, + "publish_offset_ns": 100, + "socket_prio": 6, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://aa-11-aa-11-aa-11", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 1150000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": true, + "cpu_affinity": 3, + "xdp_queue": -1 + } + } + }, +} diff --git a/json/ehl2/opcua-pkt3b-tsn.json.i b/json/ehl2/opcua-pkt3b-tsn.json.i new file mode 100644 index 0000000..f4324ad --- /dev/null +++ b/json/ehl2/opcua-pkt3b-tsn.json.i @@ -0,0 +1,84 @@ +{ + "custom_sync_b": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "iperf3": { + "cpu_affinity": 0, + "run_server": true, + "client_target_address": "169.254.2.11", + "client_runtime_in_sec": 5000, + "client_bandwidth_in_mbps": 200 + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 7, + "queues": "1@0 1@1 1@2 1@3 1@4 1@5 1@6", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3, + "p4": 4, + "p5": 5, + "p6": 6 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 1000000 + }, + { + "gate_mask": "42", + "duration": 1000000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 400000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + } + ] +} diff --git a/json/ehl2/opcua-pkt3b.json.i b/json/ehl2/opcua-pkt3b.json.i new file mode 100644 index 0000000..fe4a4cf --- /dev/null +++ b/json/ehl2/opcua-pkt3b.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "subscriber_interface": "_PREPROCESS_STR_interface", + "publisher_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": false, + "packet_count": 500000, + "cycle_time_ns": 2000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.6", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 800000, + "publish_offset_ns": 1000100, + "socket_prio": 6, + "two_way_data": true, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 150000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 3, + "xdp_queue": -1 + } + } + }, +} diff --git a/json/ehl2/opcua-xdp2a-tsn.json.i b/json/ehl2/opcua-xdp2a-tsn.json.i new file mode 100644 index 0000000..f48df98 --- /dev/null +++ b/json/ehl2/opcua-xdp2a-tsn.json.i @@ -0,0 +1,101 @@ +{ + "custom_sync_a": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "tc_group" : [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 100000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "run_sh": [ + "ethtool -K _PREPROCESS_STR_interface rxvlan off", + "ethtool -K _PREPROCESS_STR_2nd_interface rxvlan off" + ] + } + ] +} diff --git a/json/ehl2/opcua-xdp2a.json.i b/json/ehl2/opcua-xdp2a.json.i new file mode 100644 index 0000000..e08dc5b --- /dev/null +++ b/json/ehl2/opcua-xdp2a.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": true, + "packet_count": 1000000, + "cycle_time_ns": 400000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.2", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 100000, + "publish_offset_ns": 100, + "socket_prio": 2, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": 2 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.2", + "sub_id": 11, + "subscribed_pub_id": 2235, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 200000, + "subscriber_output_file": "afxdp-rxtstamps.txt", + "two_way_data": true, + "cpu_affinity": 3, + "xdp_queue": 2 + } + } + }, +} diff --git a/json/ehl2/opcua-xdp2b-tsn.json.i b/json/ehl2/opcua-xdp2b-tsn.json.i new file mode 100644 index 0000000..de52bbf --- /dev/null +++ b/json/ehl2/opcua-xdp2b-tsn.json.i @@ -0,0 +1,101 @@ +{ + "custom_sync_b": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "tc_group" : [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 100000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "run_sh": [ + "ethtool -K _PREPROCESS_STR_interface rxvlan off", + "ethtool -K _PREPROCESS_STR_2nd_interface rxvlan off" + ] + } + ] +} diff --git a/json/ehl2/opcua-xdp2b.json.i b/json/ehl2/opcua-xdp2b.json.i new file mode 100644 index 0000000..93a3f56 --- /dev/null +++ b/json/ehl2/opcua-xdp2b.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_2nd_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": true, + "packet_count": 1000000, + "cycle_time_ns": 400000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.2", + "pub_id": 2235, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 50000, + "publish_offset_ns": 200000, + "socket_prio": 2, + "two_way_data": true, + "cpu_affinity": 2, + "xdp_queue": 2 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.2", + "sub_id": 22, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 5000, + "subscriber_output_file": "afxdp-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 3, + "xdp_queue": 2 + } + } + }, +} diff --git a/json/ehl2/opcua-xdp3a-tsn.json.i b/json/ehl2/opcua-xdp3a-tsn.json.i new file mode 100644 index 0000000..78ddd5d --- /dev/null +++ b/json/ehl2/opcua-xdp3a-tsn.json.i @@ -0,0 +1,108 @@ +{ + "custom_sync_a": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "iperf3": { + "cpu_affinity": 0, + "run_server": true, + "client_target_address": "169.254.1.22", + "client_runtime_in_sec": 5000, + "client_bandwidth_in_mbps": 200 + }, + "tc_group" : [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 100000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "run_sh": [ + "ethtool -K _PREPROCESS_STR_interface rxvlan off", + "ethtool -K _PREPROCESS_STR_2nd_interface rxvlan off" + ] + } + ] +} diff --git a/json/ehl2/opcua-xdp3a.json.i b/json/ehl2/opcua-xdp3a.json.i new file mode 100644 index 0000000..e08dc5b --- /dev/null +++ b/json/ehl2/opcua-xdp3a.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_2nd_interface", + "use_xdp": true, + "packet_count": 1000000, + "cycle_time_ns": 400000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.2", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 100000, + "publish_offset_ns": 100, + "socket_prio": 2, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": 2 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.2", + "sub_id": 11, + "subscribed_pub_id": 2235, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 200000, + "subscriber_output_file": "afxdp-rxtstamps.txt", + "two_way_data": true, + "cpu_affinity": 3, + "xdp_queue": 2 + } + } + }, +} diff --git a/json/ehl2/opcua-xdp3b-tsn.json.i b/json/ehl2/opcua-xdp3b-tsn.json.i new file mode 100644 index 0000000..4235fae --- /dev/null +++ b/json/ehl2/opcua-xdp3b-tsn.json.i @@ -0,0 +1,108 @@ +{ + "custom_sync_b": { + "interface": "_PREPROCESS_STR_interface.vlan", + "interface2": "_PREPROCESS_STR_2nd_interface.vlan", + "ignore_existing": true, + "socket_prio": 1 + }, + "iperf3": { + "cpu_affinity": 0, + "run_server": true, + "client_target_address": "169.254.2.11", + "client_runtime_in_sec": 5000, + "client_bandwidth_in_mbps": 200 + }, + "tc_group" : [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ] + }, + { + "interface": "_PREPROCESS_STR_2nd_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 3, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "43", + "duration": 200000 + }, + { + "gate_mask": "42", + "duration": 200000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 100000, + "queue": 6, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 6, + "rx_hw_q": 2 + } + ], + "run_sh": [ + "ethtool -K _PREPROCESS_STR_interface rxvlan off", + "ethtool -K _PREPROCESS_STR_2nd_interface rxvlan off" + ] + } + ] +} diff --git a/json/ehl2/opcua-xdp3b.json.i b/json/ehl2/opcua-xdp3b.json.i new file mode 100644 index 0000000..93a3f56 --- /dev/null +++ b/json/ehl2/opcua-xdp3b.json.i @@ -0,0 +1,38 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_2nd_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": true, + "packet_count": 1000000, + "cycle_time_ns": 400000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://aa-11-aa-11-aa-11:3.2", + "pub_id": 2235, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 50000, + "publish_offset_ns": 200000, + "socket_prio": 2, + "two_way_data": true, + "cpu_affinity": 2, + "xdp_queue": 2 + } + }, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.2", + "sub_id": 22, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 5000, + "subscriber_output_file": "afxdp-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 3, + "xdp_queue": 2 + } + } + }, +} diff --git a/json/gen_setup.py b/json/gen_setup.py new file mode 100644 index 0000000..80954af --- /dev/null +++ b/json/gen_setup.py @@ -0,0 +1,417 @@ +import argparse +import os.path +import re +import sys +import time +import os +import json +import math + +import helpers as h + +# Dry run or not +# Make this a global so we don't have to pass it everywhere +# IS_DRY = False + +def get_show_qdisc_cmd(interface): + return 'tc qdisc show dev {}'.format(interface) + +def delete_qdiscs(iface): + #print('Deleting existing qdiscs') + cmd = 'tc qdisc del dev {} root'.format(iface) + proc = h.sh_run(cmd) + h.sh_run('sleep 5') + +def init_ingress(iface, show_qdisc_cmd): + clear_rx_cmd = "tc qdisc del dev {} parent ffff:".format(iface) + add_ingress_cmd = "tc qdisc add dev {} ingress".format(iface) + + output = h.sh_run(clear_rx_cmd) + output = h.sh_run(add_ingress_cmd) + +def set_vlanrx(iface, config, show_qdisc_cmd): + + vlan_priority = str(config['vlan_priority']) + rx_hw_q = str(config['rx_hw_q']) + + set_vlanrx_cmd = "tc filter add dev {} ".format(iface) + set_vlanrx_cmd += "parent ffff: protocol 802.1Q flower vlan_prio {} ".format(vlan_priority) + set_vlanrx_cmd += "hw_tc {} ".format(rx_hw_q) + + output = h.sh_run(set_vlanrx_cmd) + + +def set_taprio(iface, maps, config, basetime, clkid): + schedules = "" + for each in config["schedule"]: + schedules += "sched-entry S {} {} ".format(each['gate_mask'], str(each['duration'])) + + num_tc = 0 + queues = 0 + handle = 0 + for each in config.keys(): + if each == "num_tc": + num_tc = config["num_tc"] + if each == "queues": + queues = config["queues"] + if each == "handle": + handle = config["handle"] + + if not num_tc: + #print( "num_tc not defined" ) + exit(1) + if not queues: + #print( "queues not defined" ) + exit(1) + if not handle: + #print ( "handle not defined" ) + exit(1) + + # Final taprio command to run + set_taprio_cmd = "tc -d qdisc replace dev {} ".format(iface) + set_taprio_cmd += "parent root handle {} ".format(handle) + set_taprio_cmd += "taprio num_tc {} map {} ".format(num_tc, maps) + set_taprio_cmd += "queues {} base-time {} ".format(queues, basetime) + set_taprio_cmd += "{} flags 0x2".format(schedules) + + # Parse hardware offload option + # It's specified in json as "offload": true|false + if 'offload' in config and config.get('offload'): + set_taprio_cmd += ' offload 1' + + output = h.sh_run(set_taprio_cmd) + +def set_mqprio(iface, maps, config): + num_tc = 0 + queues = 0 + handle = 0 + for each in config.keys(): + if each == "num_tc": + num_tc = config["num_tc"] + if each == "queues": + queues = config["queues"] + if each == "handle": + handle = config["handle"] + + if not num_tc: + #print("num_tc not defined") + exit(1) + if not queues: + #print("queues not defined") + exit(1) + if not handle: + #print("handle not defined") + exit(1) + + # Final mqprio command to run + set_mqprio_cmd = "tc qdisc add dev {} ".format(iface) + set_mqprio_cmd += "parent root handle {} ".format(handle) + set_mqprio_cmd +="mqprio num_tc {} ".format(num_tc) + set_mqprio_cmd +="map {} queues {} hw 0".format(maps,queues) + output = h.sh_run(set_mqprio_cmd) + +def set_etf(iface, clkid, config, show_cmd, use_taprio): + # unpack config variable + delta = str(config['delta']) + + # naming convention for queue is : + # since naming starts from 1, we add 1 to queue. + queue = str(config['queue'] + 1) + + # deadline_mode and offload is turn off by default + deadline_mode = False + offload = False + + # find the parent for specific qdisc + cmd = 'HANDLE_ID="$(tc qdisc show dev {} | tr -d \':\' | awk \'NR==1{{print $3}}\')"'.format(iface) + h.sh_run(cmd) + + # check if config specify deadline_mode or offload + for each in config.keys(): + if each == 'deadline_mode': + deadline_mode = config['deadline_mode'] + if each == 'offload': + offload = config['offload'] + + # generate etf command + etf_default_cmd = 'tc qdisc replace dev {} parent $HANDLE_ID:{} etf '.format(iface, queue) + etf_default_cmd += ' clockid {}'.format(clkid) + etf_default_cmd += ' delta {}'.format(delta) + if offload == True: + etf_default_cmd += ' offload' + if deadline_mode == True: + etf_default_cmd += ' deadline_mode' + + # #print('Adding etf qdisc on queue {}...'.format(queue)) + output = h.sh_run(etf_default_cmd) + # h.sh_run('sleep 2') + +def set_mapping(scenario_config, use_mapping): + maps = [] + maps.append(str(scenario_config[use_mapping]["mapping"]["default"])) + maps = maps * 16 + # apply mapping for specific queue + for each in scenario_config[use_mapping]["mapping"]: + if each == 'default': + continue + priority = int(each[1:]) + queue = str(scenario_config[use_mapping]["mapping"][each]) + maps[priority] = queue + maps = ' '.join(maps) + return maps + +def set_cbs(iface, scenario_config): + handle = scenario_config["handle"] + parent = scenario_config["parent"] + queue = scenario_config["queue"] + 1 + sendslope = scenario_config["sendslope"] + idleslope = scenario_config["idleslope"] + hicredit = scenario_config["locredit"] + locredit = scenario_config["locredit"] + offload= scenario_config["offload"] + + # generate cbs command + etf_default_cmd = 'tc qdisc replace dev {} '.format(iface) + etf_default_cmd += 'handle {} parent {}:{} '.format(handle,parent,queue) + etf_default_cmd += 'cbs idleslope {} sendslope {} '.format(idleslope,sendslope) + etf_default_cmd += 'hicredit {} locredit {} '.format(hicredit,locredit) + etf_default_cmd += 'offload {} '.format(offload) + + #print('Adding cbs qdisc on queue {}...'.format(queue)) + output = h.sh_run(etf_default_cmd) + h.sh_run('sleep 5') + +def process_tc_data(data): + # If file is empty then we do nothing + if len(data) is 0: + return + + if not 'interface' in data: + h.err_exit('Interface not found in tc config') + + interface = data.get('interface') + + delete_qdiscs(interface) + + # Moved from existing script + use_scheduler = '' + is_using_taprio = False + if 'mqprio' in data: use_scheduler = 'mqprio' + if 'taprio' in data: + is_using_taprio = True + use_scheduler = 'taprio' + + clock_id = 'CLOCK_TAI' + if use_scheduler: + #print("Setting up {} qdisc".format(use_scheduler)) + maps = set_mapping(data, use_scheduler) + + if use_scheduler == 'taprio': + time_elapsed = data['taprio']['time_elapsed'] + base_time = '$(expr $(date +%s) + 5)000000000' + #print('Base time set to {}s from now'.format(time_elapsed)) + set_taprio(interface, maps, data["taprio"], base_time, clock_id) + + elif use_scheduler == 'mqprio': + set_mqprio(interface, maps, data["mqprio"]) + + show_qdisc_cmd = get_show_qdisc_cmd(interface) + + if 'cbs' in data: + #print('Setup CBS') + set_cbs(interface, data["cbs"]) + + if 'etf' in data: + #print('Setup ETF') + for each_config in data["etf"]: + set_etf(interface, clock_id, each_config, show_qdisc_cmd, + is_using_taprio) + + if 'vlanrx' in data: + #print('Setup VLAN RX steering') + init_ingress(interface, show_qdisc_cmd) + for each_config in data["vlanrx"]: + set_vlanrx(interface, each_config, show_qdisc_cmd) + + if 'run_sh' in data: + #print('Running raw shell commands, this shouldn\'t happen in prod') + for cmd in data.get('run_sh'): + h.sh_run(cmd) + + # #print(h.sh_run(show_qdisc_cmd).decode('utf8')) + +def process_iperf3(obj): + # kill all currently running iperf3 + h.sh_run('pkill iperf3') + run_server = obj.get('run_server', False) + client_target_add = str(obj.get('client_target_address')) + cpu_aff = str(obj.get('cpu_affinity', 3)) + client_runtime = str(obj.get('client_runtime_in_sec', 5000)) + iperf3_log = "/var/log/iperf3_client.log" + client_bw = str(obj.get('client_bandwidth_in_mbps', 1)) + + # run the iperf3 server if server is enabled + if run_server: + iperf3_com = 'iperf3 -s -D --affinity {} &'.format(cpu_aff) + h.sh_run(iperf3_com) + + # create the iperf3 client(udp) bash script if target server IP is given + if client_target_add: + client_iperf3_com = 'iperf3 -c {} --affinity {} -u -t {} --logfile {} -b {}M '.format(client_target_add, cpu_aff, client_runtime, iperf3_log, client_bw) + f = open('iperf3-gen-cmd.sh','w') + f.write(str(client_iperf3_com)) + f.close() + +def process_ptp(obj): + h.ensure_keys_exist('ptp', obj, 'interface') + iface = obj.get('interface') + ignore_existing = obj.get('ignore_existing', False) + socket_prio = str(obj.get('socket_prio', 2)) + + ## TODO sock prio or net prio + if not ignore_existing: return + h.sh_run('pkill ptp4l') + + # ptp4l -mP2Hi eth0 -f scripts/gPTP.cfg --step_threshold=2 --socket_priority 1 + arglist = ['taskset', '-c', '1', 'ptp4l', '-mP2Hi', iface, '-f', 'scripts/gPTP.cfg', '--step_threshold=2'] + arglist += ['--socket_priority', socket_prio] + h.run_with_out(arglist, '/var/log/ptp4l.log') + h.sh_run('sleep 30') + +def process_phc2sys(obj): + h.ensure_keys_exist('phc2sys', obj, 'clock', 'interface') + clock = obj.get('clock') + iface = obj.get('interface') + ignore_existing = obj.get('ignore_existing', False) + + if not ignore_existing: return + h.sh_run('pkill phc2sys') + h.sh_run('sleep 2') + + # pmc -ub 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248 \ + # clockAccuracy 0xfe offsetScaledLogVariance 0xffff \ + # currentUtcOffset 37 leap61 0 leap59 0 currentUtcOffsetValid 1 \ + # ptpTimescale 1 timeTraceable 1 frequencyTraceable 0 timeSource 0xa0 + arglist = ['pmc', '-u', '-b', '0', '-t', '1', '"', + 'SET GRANDMASTER_SETTINGS_NP clockClass 248' + ' clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 37' + ' leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable' + ' 1 frequencyTraceable 0 timeSource 0xa0', '"'] + h.run_with_out(arglist, '/var/log/pmc.log') + + h.sh_run('sleep 2') + + # phc2sys -c CLOCK_REALTIME --step_threshold=1 -s eth0 \ + # --transportSpecific=1 -O 0 -w -ml 7 + arglist = ['taskset', '-c', '1', 'phc2sys', '-c', 'CLOCK_REALTIME', '--step_threshold=1', '-s', + iface, '--transportSpecific=1', '-O', '0', '-w', '-ml', '7'] + h.run_with_out(arglist, '/var/log/phc2sys.log') + +def process_custom_a(obj): + h.ensure_keys_exist(obj, 'interface2', 'interface') + iface = obj.get('interface') + iface2 = obj.get('interface2') + ignore_existing = obj.get('ignore_existing', False) + socket_prio = str(obj.get('socket_prio', 1)) + + if not ignore_existing: return + h.sh_run('pkill phc2sys') + h.sh_run('pkill ptp4l') + + # ptp4l -mP2Hi eth0 -f scripts/gPTP.cfg --step_threshold=2 --socket_priority 1 + arglist = ['taskset', '-c', '1', 'ptp4l', '-mP2Hi', iface, '-f', 'scripts/gPTP.cfg', '--step_threshold=2'] + arglist += ['--socket_priority', '1'] + h.run_with_out(arglist, '/var/log/ptp4l.log') + #print("Give 30 secs for ptp to sync") + if not h.IS_DRY: + h.sh_run('sleep 30') + + # pmc -ub 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248 \ + # clockAccuracy 0xfe offsetScaledLogVariance 0xffff \ + # currentUtcOffset 37 leap61 0 leap59 0 currentUtcOffsetValid 1 \ + # ptpTimescale 1 timeTraceable 1 frequencyTraceable 0 timeSource 0xa0 + arglist = ['pmc', '-u', '-b', '0', '-t', '1', '"', + 'SET GRANDMASTER_SETTINGS_NP clockClass 248' + ' clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 37' + ' leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable' + ' 1 frequencyTraceable 0 timeSource 0xa0', '"'] + h.run_with_out(arglist, '/var/log/pmc.log') + + # phc2sys -c CLOCK_REALTIME --step_threshold=1 -s eth0 \ + # --transportSpecific=1 -O 0 -w -ml 7 + arglist = ['taskset', '-c', '1', 'phc2sys', '-c', 'CLOCK_REALTIME', '--step_threshold=1', '-s', + iface, '--transportSpecific=1', '-O', '0', '-w', '-ml', '7'] + h.run_with_out(arglist, '/var/log/phc2sys.log') + +def process_custom_b(obj): + h.ensure_keys_exist(obj, 'interface2', 'interface') + iface = obj.get('interface') + iface2 = obj.get('interface2') + ignore_existing = obj.get('ignore_existing', False) + socket_prio = str(obj.get('socket_prio', 1)) + + if not ignore_existing: return + h.sh_run('pkill phc2sys') + h.sh_run('pkill ptp4l') + + # ptp4l -mP2Hi eth0 -i eth2 -f scripts/gPTP.cfg --step_threshold=2 \ + # --socket_priority 1 --boundary_clock_jbod=1 + arglist = ['taskset', '-c', '1', 'ptp4l', '-mP2Hi', iface, '-i', iface2 , '-f', 'scripts/gPTP.cfg', '--step_threshold=2'] + arglist += ['--socket_priority', '1'] + arglist += ['--boundary_clock_jbod=1'] + h.run_with_out(arglist, '/var/log/ptp4l.log') + #print("Give 30 secs for ptp to sync") + if not h.IS_DRY: + h.sh_run('sleep 30') + + # pmc -ub 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248 \ + # clockAccuracy 0xfe offsetScaledLogVariance 0xffff \ + # currentUtcOffset 37 leap61 0 leap59 0 currentUtcOffsetValid 1 \ + # ptpTimescale 1 timeTraceable 1 frequencyTraceable 0 timeSource 0xa0 + arglist = ['pmc', '-u', '-b', '0', '-t', '1', '"', + 'SET GRANDMASTER_SETTINGS_NP clockClass 248' + ' clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 0' + ' leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable' + ' 1 frequencyTraceable 0 timeSource 0xa0', '"'] + h.run_with_out(arglist, '/var/log/pmc.log') + + # phc2sys -arrml 7 -f scripts/gPTP.cfg + arglist = ['taskset', '-c', '1', 'phc2sys', '-arrml', '7', '-f', 'scripts/gPTP.cfg'] + h.run_with_out(arglist, '/var/log/phc2sys.log') + +def main(): + parser = argparse.ArgumentParser(description='Configures the interface') + parser.add_argument('config_file', help='Path to config file') + parser.add_argument('-d', '--dry-run', dest='dry', default=False, + action='store_true', help='Display commands without running them') + args = parser.parse_args() + cfg_path = args.config_file + + h.initialize() + h.IS_DRY = args.dry + + if not os.path.isfile(cfg_path): + #print('File {} not found'.format(cfg_path)) + exit(1) + + with open(cfg_path, 'r') as f: + data = json.loads(f.read()) + + if 'ptp' in data: + process_ptp(data['ptp']) + + if 'phc2sys' in data: process_phc2sys(data['phc2sys']) + + if 'custom_sync_a' in data: process_custom_a(data['custom_sync_a']) + if 'custom_sync_b' in data: process_custom_b(data['custom_sync_b']) + + for each_tc in data["tc_group"]: + process_tc_data(each_tc) + + if 'iperf3' in data: + process_iperf3(data['iperf3']) + + h.teardown() + +if __name__ == '__main__': + main() diff --git a/json/helpers.py b/json/helpers.py new file mode 100644 index 0000000..67ca2c0 --- /dev/null +++ b/json/helpers.py @@ -0,0 +1,39 @@ +import argparse +import os.path +import sys +import time +import os +import json +import math + +def initialize(): + global IS_DRY + global genfile + + IS_DRY=False + genfile = open("setup-generated.sh", "w") + genfile.write("#!/bin/bash\n") + +def teardown(): + genfile.close() + +def err_exit(msg): + print(msg) + exit(1) + +def ensure_keys_exist(name, mapd, *keys): + for k in keys: + if not k in mapd: + print('Key {} does not exist in {}'.format(k, name)) + exit(2) + +def sh_run(cmd): + genfile.write('echo "Running {}"\n'.format(cmd)) + genfile.write(cmd + "\n") + +def run_with_out(cmd, outfile): + cmd += ['2&>', outfile, '&\n'] + cmd2 = ' '.join(cmd) + genfile.write('echo "Running (async) {}"\n'.format(cmd2)) + genfile.write(cmd2) + diff --git a/json/helpers.sh b/json/helpers.sh new file mode 100644 index 0000000..02ad439 --- /dev/null +++ b/json/helpers.sh @@ -0,0 +1,445 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +# Helper functions. This script executes nothing. +# TODO: Move these into a python script, perhaps launch.py? + +set_irq_smp_affinity(){ + + IFACE=$1 + AFFINITY_FILE=$2 + + # echo "Setting IRQ affinity based on $AFFINITY_FILE" + # echo "Note: affinity file should have empty new line at the end." + + #Rather than mapping all irqs to a CPU, we only map the queues we use + # via the affinity file. Only 2 columns used, last column is comments + while IFS=, read -r CSV_Q CSV_CORE CSV_COMMENTS + do + IRQ_NUM=$(cat /proc/interrupts | grep $IFACE:$CSV_Q | awk '{print $1}' | rev | cut -c 2- | rev) + # echo "Echo-ing 0x$CSV_CORE > /proc/irq/$IRQ_NUM/smp_affinity $IFACE:$CSV_Q " + echo $CSV_CORE > /proc/irq/$IRQ_NUM/smp_affinity + done < $AFFINITY_FILE +} + +setup_sp1a(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Make sure systemd do not manage the interface + mv /lib/systemd/network/80-wired.network . 2> /dev/null + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + systemctl restart systemd-networkd.service + ip link set dev $IFACE address aa:00:aa:00:aa:00 + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.1.11/24 brd 169.254.1.255 dev $IFACE + ip addr add 169.254.11.11/24 brd 169.254.11.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx.map + else + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_4tx_4rx.map + fi + + # OPCUA-PKT3? multistream +} + +setup_sp2a(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + ip link set dev $IFACE address aa:11:aa:11:aa:11 + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.2.11/24 brd 169.254.2.255 dev $IFACE + ip addr add 169.254.22.11/24 brd 169.254.22.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx-multi.map + else + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_4tx_4rx.map + fi +} + + +setup_sp1b(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Make sure systemd do not manage the interface + mv /lib/systemd/network/80-wired.network . 2> /dev/null + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + systemctl restart systemd-networkd.service + ip link set dev $IFACE address 22:bb:22:bb:22:bb + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.1.22/24 brd 169.254.1.255 dev $IFACE + ip addr add 169.254.11.22/24 brd 169.254.11.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx.map + else + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_4tx_4rx.map + fi +} + +setup_sp2b(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + ip link set dev $IFACE address 22:cc:22:cc:22:cc + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.2.22/24 brd 169.254.22.255 dev $IFACE + ip addr add 169.254.22.22/24 brd 169.254.22.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx-multi.map + else + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_4tx_4rx.map + fi +} + + +calc_rx_u2u(){ + # Output format (1 way): + # latency, rx_sequence, sdata->id, txTime, RXhwTS, rxTime + + local RX_FILENAME=$1 #*-rxtstamps.txt + SHORTNAME=$(echo $RX_FILENAME | awk -F"-" '{print $1}') + + #U2U stats + cat $RX_FILENAME | \ + awk '{ print $1 "\t" $2}' | \ + awk '{ if($2 > 0){ + if(min==""){min=max=$1}; + if($1>max) {max=$1}; + if($1 temp1.txt + + echo -e "Results\nAvg\nMax\nMin" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt + + #Plot u2u + cat $RX_FILENAME | \ + awk '{ print $1 }' > $SHORTNAME-traffic.txt +} + +calc_return_u2u(){ + # Output format (return): + # a2bLatency, seqA, sdata->id, txPubA, RXhwTS, rxSubB, processingLatency, + # b2aLatency, seqB, sdata->id, txPubB, RXhwTS, rxSubA, returnLatency + + local RX_FILENAME=$1 #*-rxtstamps.txt + SHORTNAME=$(echo $RX_FILENAME | awk -F"-" '{print $1}') + + #U2U stats + cat $RX_FILENAME | \ + awk '{ print $14 "\t" $9}' | \ + awk '{ if($2 > 0){ + if(min==""){min=max=$1}; + if($1>max) {max=$1}; + if($1 temp1.txt + + echo -e "Results\nAvg\nMax\nMin" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt + + #Plot u2u + cat $RX_FILENAME | \ + awk '{ print $14 }' > $SHORTNAME-traffic.txt +} + + +calc_rx_duploss(){ + # Output format (1 way): + # latency, rx_sequence, sdata->id, txTime, RXhwTS, rxTime + + local XDP_TX_FILENAME=$1 #*-txtstamps.txt + local JSON_FILE=$2 #Json configuration file + + # Packet count from json file + PACKET_COUNT=$(grep -s packet_count $JSON_FILE | awk '{print $2}' | sed 's/,//') + echo $PACKET_COUNT >> temp1.txt + + # Total packets received + PACKET_RX=$(cat $XDP_TX_FILENAME \ + | awk '{print $2}' \ + | grep -x -E '[0-9]+' \ + | wc -l) + echo $PACKET_RX >> temp1.txt + + # Total duplicate + PACKET_DUPL=$(cat $XDP_TX_FILENAME \ + | awk '{print $2}' \ + | uniq -D \ + | wc -l) + echo $PACKET_DUPL >> temp1.txt + + # Total missing: Same as: packet count - total_packet - total_duplicate + PACKET_LOSS=$((PACKET_COUNT-PACKET_RX-PACKET_DUPL)) + echo $PACKET_LOSS >> temp1.txt + + echo -e "TotalExpected\nTotalReceived\nDuplicates\nLosses" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt +} + +calc_return_duploss(){ + # Output format (return): + # a2bLatency, seqA, sdata->id, txPubA, RXhwTS, rxSubB, processingLatency, + # b2aLatency, seqB, sdata->id, txPubB, RXhwTS, rxSubA, returnLatency + + local XDP_TX_FILENAME=$1 #*-txtstamps.txt + local JSON_FILE=$2 #Json configuration file + + # Packet count from json file + PACKET_COUNT=$(grep -s packet_count $JSON_FILE | awk '{print $2}' | sed 's/,//') + echo $PACKET_COUNT >> temp1.txt + + # Total packets received + PACKET_RX=$(cat $XDP_TX_FILENAME \ + | awk '{print $9}' \ + | grep -x -E '[0-9]+' \ + | wc -l) + echo $PACKET_RX >> temp1.txt + + # Total duplicate + PACKET_DUPL=$(cat $XDP_TX_FILENAME \ + | awk '{print $9}' \ + | uniq -D \ + | wc -l) + echo $PACKET_DUPL >> temp1.txt + + # Total missing: Same as: packet count - total_packet - total_duplicate + PACKET_LOSS=$((PACKET_COUNT-PACKET_RX-PACKET_DUPL)) + echo $PACKET_LOSS >> temp1.txt + + echo -e "TotalExpected\nTotalReceived\nDuplicates\nLosses" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt +} + + +stop_if_empty(){ + wc -l $1 > /dev/null + if [ $? -gt 0 ]; then + exit + fi + + COUNT=$(wc -l $1 | awk '{print $1}') + + if [ $COUNT -lt 3000 ]; then + echo "Too little data $1 has only $COUNT lines" + exit + fi + echo "$1 has $COUNT lines" +} + +save_result_files(){ + + ID=$(date +%Y%m%d) + IDD=$(date +%Y%m%d-%H%M) + mkdir -p results-$ID + + rm plot_pic.png -f + + CONFIG=$1 + NUMPKTS=$2 + SIZE=$3 + INTERVAL=$4 + + case "$CONFIG" in + + opcua-pkt0b | opcua-pkt1b | opcua-pkt2a | opcua-pkt2b | opcua-pkt3a | opcua-pkt3b) + cp afpkt-rxtstamps.txt results-$ID/afpkt-$CONFIG-rxtstamps-$IDD.txt + ;; + opcua-xdp0b | opcua-xdp1b | opcua-xdp2a | opcua-xdp2b | opcua-xdp3a | opcua-xdp3b) + cp afxdp-rxtstamps.txt results-$ID/afxdp-$CONFIG-rxtstamps-$IDD.txt + ;; + *) + echo "Error: save_results_files() invalid config: $CONFIG" + exit + ;; + esac +} diff --git a/json/opcua-run.sh b/json/opcua-run.sh new file mode 100755 index 0000000..aef62c1 --- /dev/null +++ b/json/opcua-run.sh @@ -0,0 +1,236 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [[ "$1" = "-h" ]] || [[ "$1" = "" ]]; then + echo "\ +Usage: opcua-run.sh [interface2] [mode] +Example: ./opcua-run.sh ehl eth0 opcua-pkt1a init + ./opcua-run.sh ehl2 eth0 eth1 opcua-pkt2a setup + " + exit 0 +fi + +if [[ "$2" = "" ]]; then + echo "Please specify an interface name" + exit 1 +fi + +PLAT="$1" +IFACE="$2" + +if [[ "$PLAT" = "ehl2" ]];then + IFACE2="$3" + CONFIG="$4" + MODE="$5" +else + CONFIG="$3" + MODE="$4" +fi + +# Get directory of current script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh +JSONDIR="$DIR/$PLAT/" + +# Parse and check if its valid: e.g. opcua-pkt-1a +LIST=$(ls $JSONDIR | grep 'opcua' | grep 'json.i' | rev | cut -c 8- | rev | sort) +if [[ $LIST =~ (^|[[:space:]])"$CONFIG"($|[[:space:]]) ]] ; then + echo "Selected json-config: $CONFIG" +else + echo -e "Invalid json-config selected: $CONFIG \nAvailable opcua json-configs:" + printf '%s\n' "${LIST[@]}" + exit +fi + +#Iperf3 generated command file from gen_setup.py using json parameters for iperf3 client +IPERF3_GEN_CMD="iperf3-gen-cmd.sh" + +#Selected json file's absolute path +INTERIM_JSON="$JSONDIR/$CONFIG.json.i" +INTERIM_TSN_JSON="$JSONDIR/$CONFIG-tsn.json.i" + +#New filename without .i +NEW_JSON=${INTERIM_JSON%.i} +NEW_TSN_JSON=${INTERIM_TSN_JSON%.i} + +cp -f $INTERIM_JSON $NEW_JSON +cp -f $INTERIM_TSN_JSON $NEW_TSN_JSON + +#Replace interface on .json +sed -i -e "s/_PREPROCESS_STR_interface/$IFACE/gi" $NEW_JSON $NEW_TSN_JSON + +if [[ "$PLAT" = "ehl2" || "$PLAT" = "tgl2" ]];then + sed -i -e "s/_PREPROCESS_STR_2nd_interface/$IFACE2/gi" $NEW_JSON $NEW_TSN_JSON +fi + +case "$MODE" in + init) # Set the static ip and mac address only + case "$CONFIG" in + opcua-*a) + setup_sp1a $IFACE + if [[ ! -z "$IFACE2" ]];then + setup_sp2a $IFACE2 + #Overwrite IFACE1 previous map to use multi-stream map + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx-multi.map + fi + ;; + opcua-*b) + setup_sp1b $IFACE + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx-multi.map + if [[ ! -z "$IFACE2" ]];then + setup_sp2b $IFACE2 + #Overwrite IFACE1 previous map to use multi-stream map + set_irq_smp_affinity $IFACE $DIR/../scripts/irq_affinity_4c_8tx_8rx-multi.map + fi + ;; + "") + ;& + *) + echo "Invalid config $CONFIG" + exit 1 + ;; + esac + exit 0 + ;; + setup) # Run setup using *-tsn.json, and then exit + #Remove existing iperf3 generated command file + rm -f $DIR/../$IPERF3_GEN_CMD + + python3 $DIR/gen_setup.py "$NEW_TSN_JSON" + if [ $? -ne 0 ]; then + echo "gen_setup.py returned non-zero. Abort" && exit + fi + bash ./setup-generated.sh + exit 0 + ;; + run) # Proceed without running init or setup + ;; + + "") # Default if no MODE specified, run init, setup and start opcua-server + case "$CONFIG" in + opcua-*a) + setup_sp1a $IFACE + if [[ ! -z "$IFACE2" ]];then + setup_sp2a $IFACE2 + fi + ;; + opcua-*b) + setup_sp1b $IFACE + if [[ ! -z "$IFACE2" ]];then + setup_sp2a $IFACE2 + fi + ;; + "") + ;& + *) + echo "Invalid config $CONFIG" + exit 1 + ;; + esac + ;& #fallthru + setup-run) + #Remove existing iperf3 generated command file + [[ -f "$DIR/../$IPERF3_GEN_CMD" ]] && rm -f $DIR/../$IPERF3_GEN_CMD + + python3 $DIR/gen_setup.py "$NEW_TSN_JSON" + if [ $? -ne 0 ]; then + echo "gen_setup.py returned non-zero. Abort" && exit + fi + bash ./setup-generated.sh + ;; + *) + echo "Invalid mode $MODE" && exit 1 + ;; +esac + +# If the client iperf3 generated cmd file exists +if [[ -f "$IPERF3_GEN_CMD" ]]; then + CMD="$(cat $DIR/../$IPERF3_GEN_CMD)" + echo "Running: $CMD" + $CMD & +else + echo "Not running iperf3." +fi + +./opcua-server "$NEW_JSON" + +RETVAL_OPCUA=$? + +# Kill iperf3 client. Leave iperf3 server running." +IPERF3_CLI_PID=$(pgrep iperf3 -a | grep "iperf3 -c" | awk '{print $1}') +if [[ ! -z "$IPERF3_CLI_PID" ]];then + kill -9 $IPERF3_CLI_PID +fi + +if [[ "$RETVAL_OPCUA" -ne 0 ]]; then + echo "opcua-server returned non-zero. Abort" && exit +elif [[ "$CONFIG" == "opcua-pkt0b" || "$CONFIG" == "opcua-pkt1b" || + "$CONFIG" == "opcua-pkt2a" || "$CONFIG" == "opcua-pkt3a" || + "$CONFIG" == "opcua-pkt4a" || "$CONFIG" == "opcua-pkt5a" ]]; then + TYPE="afpkt" +elif [[ "$CONFIG" == "opcua-xdp0b" || "$CONFIG" == "opcua-xdp1b" || + "$CONFIG" == "opcua-xdp2a" || "$CONFIG" == "opcua-xdp3a" || + "$CONFIG" == "opcua-xdp4a" || "$CONFIG" == "opcua-xdp5a" ]]; then + TYPE="afxdp" +else + # echo "Nothing to plot" + exit 0 +fi + +stop_if_empty "$TYPE-rxtstamps.txt" + +CONFIGNUM="$(echo $CONFIG | cut -c 10)" +if [ $CONFIGNUM -lt 2 ]; then + calc_rx_u2u "$TYPE-rxtstamps.txt" + calc_rx_duploss "$TYPE-rxtstamps.txt" "$NEW_JSON" +elif [ $CONFIGNUM -lt 4 ]; then + calc_return_u2u "$TYPE-rxtstamps.txt" + calc_return_duploss "$TYPE-rxtstamps.txt" "$NEW_JSON" +else + # echo "Nothing to calculate" + exit 0 +fi + +save_result_files $CONFIG $NUMPKTS $SIZE $INTERVAL #this file's name aka config. + +if [[ "$TYPE" == "afxdp" ]]; then + gnuplot -e "FILENAME='afxdp-traffic.txt'; YMAX=2000000; PLOT_TITLE='Transmission latency from TX User-space to RX User-space (AFXDP)'" $DIR/../scripts/latency_single.gnu -p 2> /dev/null & +elif [[ "$CONFIG" == "opcua-pkt1b" ]]; then + # Plotting for single trip + gnuplot -e "FILENAME='afpkt-traffic.txt'; YMAX=10000000; PLOT_TITLE='Transmission latency from TX User-space to RX User-space (AF-PACKET - single trip)'" $DIR/../scripts/latency_single.gnu -p 2> /dev/null & +else + # Plotting for return trip + gnuplot -e "FILENAME='afpkt-traffic.txt'; YMAX=10000000; PLOT_TITLE='Transmission latency from TX User-space to RX User-space (AF-PACKET - return trip)'" $DIR/../scripts/latency_single.gnu -p 2> /dev/null & +fi + +while [[ ! -s plot_pic.png ]]; do sleep 5; done +cp plot_pic.png results-$ID/plot-$CONFIG-$NUMPKTS-$SIZE-$INTERVAL-$IDD.png # Backup diff --git a/json/run.sh b/json/run.sh new file mode 100755 index 0000000..d610202 --- /dev/null +++ b/json/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +#if [ $USER != "root" ]; then +# echo "Please run as root" +# exit +#fi + +if [ -z $1 ]; then + echo "Please enter interface e.g. ./run.sh eth0 tsq1a" + exit +fi +IFACE=$1 + +if [ -z $2 ]; then + echo "Please enter config e.g. ./run.sh eth0 tsq1a" + exit +else + CONFIG=$2 + # Parse and check if its a valid base/config + LIST=$(ls scripts/ | grep 'tsq\|vs\|opcua' | rev | cut -c 4- | rev | sort) + LIST+=$(echo -ne " \n") + if [[ $LIST =~ (^|[[:space:]])"$CONFIG"($|[[:space:]]) ]] ; then + echo "Selected config: $CONFIG" + else + echo -e "Invalid config selected: $CONFIG \nAvailable configs:" + printf '%s\n' "${LIST[@]}" + exit + fi +fi + +# Run something else first. +if [ -n $3 ]; then + case "$3" in + clock) + echo "[RUN.SH] Running clock-setup.sh" + ./scripts/clock-setup.sh $IFACE + ;; + setup*) + echo "[RUN.SH] Running setup-$CONFIG, then $CONFIG" + ./scripts/setup-$CONFIG.sh $IFACE + ;; + "") #Do nothing + ;; + *) + echo "Invalid third option: $3" + exit + ;; + esac +fi + +#Execute the specific config script, using their own defaults. +./scripts/$CONFIG.sh $IFACE + diff --git a/json/tgl/opcua-pkt1a-tsn.json.i b/json/tgl/opcua-pkt1a-tsn.json.i new file mode 100644 index 0000000..a359ef2 --- /dev/null +++ b/json/tgl/opcua-pkt1a-tsn.json.i @@ -0,0 +1,60 @@ +{ + "ptp": { + "interface": "_PREPROCESS_STR_interface.vlan", + "ignore_existing": true + }, + "phc2sys": { + "interface": "_PREPROCESS_STR_interface", + "clock": "CLOCK_REALTIME", + "ignore_existing": true + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "taprio": { + "handle": 100, + "num_tc": 4, + "queues": "1@0 1@1 1@2 1@3", + "time_elapsed": 5, + "mapping": { + "default": 0, + "p1": 1, + "p2": 2, + "p3": 3 + }, + "schedule": [ + { + "gate_mask": "0e", + "duration": 500000 + }, + { + "gate_mask": "0f", + "duration": 500000 + } + ], + "offload": false + }, + "etf": [ + { + "delta": 400000, + "queue": 3, + "offload": true + } + ], + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 3, + "rx_hw_q": 3 + } + ] + } + ] +} diff --git a/json/tgl/opcua-pkt1a.json.i b/json/tgl/opcua-pkt1a.json.i new file mode 100644 index 0000000..385fca3 --- /dev/null +++ b/json/tgl/opcua-pkt1a.json.i @@ -0,0 +1,25 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": false, + "packet_count": 1000000, + "cycle_time_ns": 1000000, + "polling_duration_ns": 0, + "publishers": { + "pub1": { + "url": "opc.eth://22-bb-22-bb-22-bb:3.3", + "pub_id": 2234, + "dataset_writer_id": 62541, + "writer_group_id": 101, + "early_offset_ns": 700000, + "publish_offset_ns": 900000, + "socket_prio": 3, + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + }, + "subscribers": {} + }, +} diff --git a/json/tgl/opcua-pkt1b-tsn.json.i b/json/tgl/opcua-pkt1b-tsn.json.i new file mode 100644 index 0000000..eec86e4 --- /dev/null +++ b/json/tgl/opcua-pkt1b-tsn.json.i @@ -0,0 +1,30 @@ +{ + "ptp": { + "interface": "_PREPROCESS_STR_interface.vlan", + "ignore_existing": true + }, + "phc2sys": { + "interface": "_PREPROCESS_STR_interface", + "clock": "CLOCK_REALTIME", + "ignore_existing": true + }, + "tc_group": [ + { + "interface": "_PREPROCESS_STR_interface", + "vlanrx": [ + { + "vlan_priority": 1, + "rx_hw_q": 1 + }, + { + "vlan_priority": 2, + "rx_hw_q": 2 + }, + { + "vlan_priority": 3, + "rx_hw_q": 3 + } + ] + } + ] +} diff --git a/json/tgl/opcua-pkt1b.json.i b/json/tgl/opcua-pkt1b.json.i new file mode 100644 index 0000000..6547932 --- /dev/null +++ b/json/tgl/opcua-pkt1b.json.i @@ -0,0 +1,25 @@ +{ + "opcua_server": { + "publisher_interface": "_PREPROCESS_STR_interface", + "subscriber_interface": "_PREPROCESS_STR_interface", + "use_xdp": false, + "packet_count": 1000000, + "cycle_time_ns": 1000000, + "polling_duration_ns": 0, + "publishers": {}, + "subscribers": { + "sub1": { + "url": "opc.eth://22-bb-22-bb-22-bb", + "sub_id": 0, + "subscribed_pub_id": 2234, + "subscribed_dataset_writer_id": 62541, + "subscribed_writer_group_id": 101, + "offset_ns": 1000, + "subscriber_output_file": "afpkt-rxtstamps.txt", + "two_way_data": false, + "cpu_affinity": 2, + "xdp_queue": -1 + } + } + }, +} diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..5991930 --- /dev/null +++ b/run.sh @@ -0,0 +1,117 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +#if [ $USER != "root" ]; then +# echo "Please run as root" +# exit +#fi + +if [ -z $1 ]; then + echo "For tsq*/xdp*/pkt* use: + ./run.sh eth0 tsq1a +For single-ethernet opcua-pkt/xdp* use: + ./run.sh ehl eth0 opcua-pkt1a init +For double-ethernet opcua-pkt/xdp* use: + ./run.sh ehl2 eth0 eth1 opcua-pkt2a init" + exit +fi + +if [[ "$1" == "tgl" || "$1" == "ehl" ]]; then + PLAT=$1 + IFACE=$2 + IFACE2="" + CONFIG=$3 + MODE=$4 +elif [[ "$1" == "tgl2" || "$1" == "ehl2" ]]; then + PLAT=$1 + IFACE=$2 + IFACE2=$3 + CONFIG=$4 + MODE=$5 +else + IFACE=$1 + CONFIG=$2 + ETC=$3 +fi + +if [ -z $CONFIG ]; then + echo "For tsq*/xdp*/pkt* use: + ./run.sh eth0 tsq1a +For single-ethernet opcua-pkt/xdp* use: + ./run.sh ehl eth0 opcua-pkt1a init +For double-ethernet opcua-pkt/xdp* use: + ./run.sh ehl2 eth0 eth1 opcua-pkt2a init" + exit +else + # Parse and check if its a valid base/config + LIST=$(ls scripts/ | grep 'tsq\|vs' | rev | cut -c 4- | rev | sort) + LIST+=$(echo -ne " \n") + if [[ $LIST =~ (^|[[:space:]])"$CONFIG"($|[[:space:]]) ]] ; then + echo "Selected config: $CONFIG" + elif [[ "(^|[[:space:]])"$CONFIG"($|[[:space:]])" =~ "opcua" ]] ; then + echo "opcua config detected" + else + echo -e "Invalid config selected: $CONFIG \nAvailable configs:" + printf '%s\n' "${LIST[@]}" + exit + fi +fi + +# Run something else first. +if [ -n $ETC ]; then + case "$ETC" in + clock) + echo "[RUN.SH] Running clock-setup.sh" + ./scripts/clock-setup.sh $IFACE + ;; + setup*) + echo "[RUN.SH] Running setup-$CONFIG, then $CONFIG" + ./scripts/setup-$CONFIG.sh $IFACE + ;; + "") #Do nothing + ;; + *) + echo "Invalid third option: $3" + exit + ;; + esac +fi + +#Execute the specific config script, using their own defaults. +CHECK=$(echo $CONFIG | cut -c -5 ) +if [ $CHECK == "opcua" ]; then + # For json types like opcua-pkt*, opcua-xdp* + ./json/opcua-run.sh $PLAT $IFACE $IFACE2 $CONFIG $MODE +else + # For shell types like tsq*, xdp*, pkt*, vs* - use script defaults + ./scripts/$CONFIG.sh $IFACE +fi diff --git a/scripts/clock-setup.sh b/scripts/clock-setup.sh new file mode 100755 index 0000000..2d4ee3f --- /dev/null +++ b/scripts/clock-setup.sh @@ -0,0 +1,71 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ -z $1 ]; then + echo "Please enter interface: ./clock-setup.sh eth0" + exit +fi + +echo "Running PTP4L & PHC2SYS" + +pkill ptp4l +pkill phc2sys + +IFACE=$1 + +# Get directory of current script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# Use 1 queue exclusively for PTP traffic +TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') +if [ $TXQ_COUNT -eq 8 ]; then + PTPTXQ_NUM=1 #EHL-HWTXq1 +elif [ $TXQ_COUNT -eq 4 ]; then + PTPTXQ_NUM=2 #TGL-HWTXq2 +else + PTPTXQ_NUM=1 #default +fi + +taskset -c 1 ptp4l -P2Hi $IFACE.vlan -f $DIR/gPTP.cfg \ + --step_threshold=1 --socket_priority=$PTPTXQ_NUM -m$2 2&> /var/log/ptp4l.log & + +sleep 2 # Required + +pmc -u -b 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248 + clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 37 + leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable + 1 frequencyTraceable 0 timeSource 0xa0" 2&> /var/log/pmc.log + +sleep 3 + +taskset -c 1 phc2sys -s $IFACE -c CLOCK_REALTIME --step_threshold=1 \ + --transportSpecific=1 -O 0 -w -ml 7 2&> /var/log/phc2sys.log & diff --git a/scripts/enable_extts.sh b/scripts/enable_extts.sh new file mode 100755 index 0000000..a55b6d6 --- /dev/null +++ b/scripts/enable_extts.sh @@ -0,0 +1,45 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ -z $1 ]; then + echo "Please enter interface: ./enable_extts.sh iface" + exit +fi + +IFACE=$1 + +CLK=`ethtool -T $IFACE | grep -Po "(?<=PTP Hardware Clock: )[\d+]"` +PCLK=ptp$CLK +echo "Enabling extts on $IFACE ($PCLK)" + +# enable ext timestamping +echo 0 1 > /sys/class/ptp/$PCLK/extts_enable diff --git a/scripts/enable_pps.sh b/scripts/enable_pps.sh new file mode 100755 index 0000000..8037b3a --- /dev/null +++ b/scripts/enable_pps.sh @@ -0,0 +1,52 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ -z $1 ]; then + echo "Please enter interface: ./setup_pps.sh iface" + exit +fi + +IFACE=$1 + +CLK=`ethtool -T $IFACE | grep -Po "(?<=PTP Hardware Clock: )[\d+]"` +PCLK=ptp$CLK +echo "Enabling pps on $IFACE ($PCLK)" + +# configure pps +# echo > /sys/class/ptp/ptpX/period +# idx -> PPS number +# ts -> start time (second), based on ptp time +# tns -> start time (nano-second), based on ptp time +# ps -> period (s) +# pns -> period (ns) +# ptpX -> is ptp of ethernet interface +echo 0 0 0 1 0 > /sys/class/ptp/$PCLK/period diff --git a/scripts/gPTP.cfg b/scripts/gPTP.cfg new file mode 100644 index 0000000..97db0d2 --- /dev/null +++ b/scripts/gPTP.cfg @@ -0,0 +1,22 @@ +# +# 802.1AS example configuration containing those attributes which +# differ from the defaults. See the file, default.cfg, for the +# complete list of available options. +# +[global] +gmCapable 1 +priority1 248 +priority2 248 +logAnnounceInterval 0 +logSyncInterval -3 +syncReceiptTimeout 3 +neighborPropDelayThresh 800 +min_neighbor_prop_delay -20000000 +assume_two_step 1 +path_trace_enabled 1 +follow_up_info 1 +transportSpecific 0x1 +ptp_dst_mac 01:80:C2:00:00:0E +network_transport L2 +delay_mechanism P2P +tx_timestamp_timeout 100 \ No newline at end of file diff --git a/scripts/helpers.sh b/scripts/helpers.sh new file mode 100644 index 0000000..fed21c8 --- /dev/null +++ b/scripts/helpers.sh @@ -0,0 +1,301 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +# Helper functions. This script executes nothing. +SEC_IN_NSEC=1000000000 + +set_irq_smp_affinity(){ + + IFACE=$1 + AFFINITY_FILE=$2 + + # echo "Setting IRQ affinity based on $AFFINITY_FILE" + # echo "Note: affinity file should have empty new line at the end." + + #Rather than mapping all irqs to a CPU, we only map the queues we use + # via the affinity file. Only 2 columns used, last column is comments + while IFS=, read -r CSV_Q CSV_CORE CSV_COMMENTS + do + IRQ_NUM=$(cat /proc/interrupts | grep $IFACE:$CSV_Q | awk '{print $1}' | rev | cut -c 2- | rev) + # echo "Echo-ing 0x$CSV_CORE > /proc/irq/$IRQ_NUM/smp_affinity $IFACE:$CSV_Q " + echo $CSV_CORE > /proc/irq/$IRQ_NUM/smp_affinity + done < $AFFINITY_FILE +} + +setup_sp1a(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Make sure systemd do not manage the interface + mv /lib/systemd/network/80-wired.network . 2> /dev/null + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + systemctl restart systemd-networkd.service + ip link set dev $IFACE address aa:00:aa:00:aa:00 + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.1.11/24 brd 169.254.1.255 dev $IFACE + ip addr add 169.254.11.11/24 brd 169.254.11.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + ip link set $IFACE.vlan type vlan egress-qos-map 6:6 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + # Turn off VLAN Stripping + ethtool -K $IFACE rxvlan off + + # Disable EEE + ethtool --set-eee $IFACE eee off 2&> /dev/null + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/irq_affinity_4c_8tx_8rx.map + else + set_irq_smp_affinity $IFACE $DIR/irq_affinity_4c_4tx_4rx.map + fi +} + +setup_sp1b(){ + # Static IP and MAC addresses are hardcoded here + + IFACE=$1 + + # Always remove previous qdiscs + tc qdisc del dev $IFACE parent root 2> /dev/null + tc qdisc del dev $IFACE parent ffff: 2> /dev/null + tc qdisc add dev $IFACE ingress + + RXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==3{ print $2}') + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $RXQ_COUNT != $TXQ_COUNT ]; then + # Set it to even queue count. Minimum is 4 rx 4 tx. + ethtool -L $IFACE rx 4 tx 4 + fi + + # Make sure systemd do not manage the interface + mv /lib/systemd/network/80-wired.network . 2> /dev/null + + # Restart interface and systemd, also set HW MAC address for multicast + ip link set $IFACE down + systemctl restart systemd-networkd.service + ip link set dev $IFACE address 22:bb:22:bb:22:bb + ip link set dev $IFACE up + sleep 3 + + # Set VLAN ID to 3, all traffic fixed to one VLAN ID, but vary the VLAN Priority + ip link delete dev $IFACE.vlan 2> /dev/null + ip link add link $IFACE name $IFACE.vlan type vlan id 3 + + # Provide static ip address for interfaces + ip addr flush dev $IFACE + ip addr flush dev $IFACE.vlan + ip addr add 169.254.1.22/24 brd 169.254.1.255 dev $IFACE + ip addr add 169.254.11.22/24 brd 169.254.11.255 dev $IFACE.vlan + + # Map socket priority N to VLAN priority N + ip link set $IFACE.vlan type vlan egress-qos-map 1:1 + ip link set $IFACE.vlan type vlan egress-qos-map 2:2 + ip link set $IFACE.vlan type vlan egress-qos-map 3:3 + ip link set $IFACE.vlan type vlan egress-qos-map 4:4 + ip link set $IFACE.vlan type vlan egress-qos-map 5:5 + ip link set $IFACE.vlan type vlan egress-qos-map 6:6 + + # Flush neighbours, just in case + ip neigh flush all dev $IFACE + ip neigh flush all dev $IFACE.vlan + + # Turn off VLAN Stripping + ethtool -K $IFACE rxvlan off + + # Disable EEE + ethtool --set-eee $IFACE eee off 2&> /dev/null + + if [ $TXQ_COUNT -eq 8 ]; then + set_irq_smp_affinity $IFACE $DIR/irq_affinity_4c_8tx_8rx.map + else + set_irq_smp_affinity $IFACE $DIR/irq_affinity_4c_4tx_4rx.map + fi +} + +get_XDPTXQ_NUM(){ + IFACE=$1 + + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $TXQ_COUNT -eq 8 ]; then + XDPTXQ_NUM=2 #EHL-HWTXq6 + elif [ $TXQ_COUNT -eq 4 ]; then + XDPTXQ_NUM=1 #TGL-HWTXq3 + else + echo "get_XDPTXQ_NUM()- invalid TX queue count" + fi +} + +get_TXQ_NUM(){ + IFACE=$1 + + TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') + + if [ $TXQ_COUNT -eq 8 ]; then + TXQ_NUM=6 #EHL-HWTXq6, XDPTXq2 + elif [ $TXQ_COUNT -eq 4 ]; then + TXQ_NUM=3 #TGL-HWTXq3, XDPTXq1 + else + echo "get_TXQ_NUM()- invalid TX queue count" + fi +} + +calc_rx_u2u(){ + local RX_FILENAME=$1 #*-rxtstamps.txt + SHORTNAME=$(echo $RX_FILENAME | awk -F"-" '{print $1}') + + #U2U stats + cat $RX_FILENAME | \ + awk '{ print $1 "\t" $2}' | \ + awk '{ if($2 > 0){ + if(min==""){min=max=$1}; + if($1>max) {max=$1}; + if($1 temp1.txt + + echo -e "Results\nAvg\nMax\nMin" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt + + #Plot u2u + cat $RX_FILENAME | \ + awk '{ print $1 }' > $SHORTNAME-traffic.txt +} + +calc_rx_duploss(){ + local XDP_RX_FILENAME=$1 #*-txtstamps.txt + + # Total packets + cat $XDP_RX_FILENAME \ + | awk '{print $2}' \ + | grep -x -E '[0-9]+' \ + | wc -l >> temp1.txt + + # Total duplicate + cat $XDP_RX_FILENAME \ + | awk '{print $2}' \ + | uniq -D \ + | wc -l >> temp1.txt + + # Total missing: Same as: total_packet - total_unique (uniq -c) + cat $XDP_RX_FILENAME \ + | awk '{print $2}' \ + | grep -x -E '[0-9]+' \ + | awk '{for(i=p+1; i<$1; i++) print i} {p=$1}' \ + | wc -l >> temp1.txt + + echo -e "Total\nDuplicates\nLosses" > temp0.txt + paste temp0.txt temp1.txt > temp-total.txt + column -t temp-total.txt + rm temp*.txt +} + +stop_if_empty(){ + wc -l $1 > /dev/null + if [ $? -gt 0 ]; then + exit + fi + + COUNT=$(wc -l $1 | awk '{print $1}') + + if [ $COUNT -lt 3000 ]; then + echo "Too little data $1 has only $COUNT lines" + exit + fi + echo "$1 has $COUNT lines" +} + +save_result_files(){ + + ID=$(date +%Y%m%d) + IDD=$(date +%Y%m%d-%H%M) + mkdir -p results-$ID + + rm -f plot_pic.png #remove existing plot files. + + CONFIG=$1 + NUMPKTS=$2 + SIZE=$3 + INTERVAL=$4 + + case "$CONFIG" in + + vs1b) + cp afxdp-rxtstamps.txt results-$ID/afxdp-$CONFIG-$NUMPKTS-$SIZE-$XDP_INTERVAL-rxtstamps-$IDD.txt + cp afpkt-rxtstamps.txt results-$ID/afpkt-$CONFIG-$NUMPKTS-$SIZE-$INTERVAL-rxtstamps-$IDD.txt + ;; + + *) + echo "Error: save_results_files() invalid config: $CONFIG" + exit + ;; + esac +} diff --git a/scripts/iperf3-bg-client.sh b/scripts/iperf3-bg-client.sh new file mode 100755 index 0000000..3ae0db2 --- /dev/null +++ b/scripts/iperf3-bg-client.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +TERM_EMU=$(ps -o comm= -p "$(($(ps -o ppid= -p "$(($(ps -o sid= -p "$$")))")))") + +#Run Iperf3 +# Send to ip +# UDP +# bandwidth 990mbps +# size of packets 1300 bytes +# output format in Mbits/sec +# output interval 5s +# run for 30s +# use CPU1 or CPUAffinity1 + +if [ "$TERM_EMU" == "sshd" ]; then + #This might still have issues. + iperf3 -c 169.254.1.22 -u -b 200M -l 1440 -f m -i 10 -t 30000 -A 0 & +else + #Run in bg xterm, UI requires xterm -e, nohup is not helpful + xterm -e 'iperf3 -c 169.254.1.22 -u -b 200M -l 1440 -f m -i 10 -t 30000 -A 0' & +fi diff --git a/scripts/iperf3-bg-server.sh b/scripts/iperf3-bg-server.sh new file mode 100755 index 0000000..a9b7989 --- /dev/null +++ b/scripts/iperf3-bg-server.sh @@ -0,0 +1,47 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +TERM_EMU=$(ps -o comm= -p "$(($(ps -o ppid= -p "$(($(ps -o sid= -p "$$")))")))") + +#Run Iperf3 +# run server +# bind to ip +# output interval 5s +# use CPU1 or CPUAffinity1 + +if [ "$TERM_EMU" == "sshd" ]; then + #This might still have issues. + iperf3 -s -B 169.254.1.22 -i 10 -1 -A 0 & +else + #Run in bg xterm, UI requires xterm -e, nohup is not helpful + xterm -e 'iperf3 -s -B 169.254.1.22 -i 10 -A 0 -1' & +fi diff --git a/scripts/irq_affinity_4c_4tx_4rx.map b/scripts/irq_affinity_4c_4tx_4rx.map new file mode 100644 index 0000000..e61abcc --- /dev/null +++ b/scripts/irq_affinity_4c_4tx_4rx.map @@ -0,0 +1,6 @@ +tx-0,01,general +rx-0,01,general +tx-2,02,ptp-tx +rx-2,02,ptp-rx +tx-3,04,afpkt-p3/afxdp-q1 +rx-3,04,afpkt-p3/afxdp-q1 diff --git a/scripts/irq_affinity_4c_8tx_8rx-multi.map b/scripts/irq_affinity_4c_8tx_8rx-multi.map new file mode 100644 index 0000000..f4a25ce --- /dev/null +++ b/scripts/irq_affinity_4c_8tx_8rx-multi.map @@ -0,0 +1,6 @@ +tx-0,01,general +rx-0,01,general +tx-1,02,ptp-tx +rx-1,02,ptp-rx +tx-6,04,afpkt-p6/afxdp-q2 +rx-2,08,afpkt-p6/afxdp-q2 diff --git a/scripts/irq_affinity_4c_8tx_8rx.map b/scripts/irq_affinity_4c_8tx_8rx.map new file mode 100644 index 0000000..bdab721 --- /dev/null +++ b/scripts/irq_affinity_4c_8tx_8rx.map @@ -0,0 +1,6 @@ +tx-0,01,general +rx-0,01,general +tx-1,02,ptp-tx +rx-1,02,ptp-rx +tx-6,04,afpkt-p6/afxdp-q2 +rx-2,04,afpkt-p6/afxdp-q2 diff --git a/scripts/latency_dual.gnu b/scripts/latency_dual.gnu new file mode 100755 index 0000000..30af1c9 --- /dev/null +++ b/scripts/latency_dual.gnu @@ -0,0 +1,109 @@ +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +# Input files should be single-column integers with no headers present + +# Example cmdline: +# gnuplot -e "FILENAME='input.txt';FILENAME2='input2.txt'" latency_dual.gnu -p + +reset + + set terminal pngcairo size 1920,1080 + set output 'plot_pic.png' + + set size 1,1 + set datafile missing'?' # Skip invalid/missing data + set locale "en_US.utf8" # Set to en_US locale for thousands seperator + + set tmargin 5 + set bmargin 5 + set lmargin 15 + set rmargin 5 + + set yrange [0:10000000] #Legacy socket plot YMAX + + set grid + set key center tmargin + set border + + set style fill solid 0.25 + set boxwidth 0.5 + # set logscale y 10 + + set title "Transmission latency from TX User-space to RX User-space" + set xlabel "Packet count" + set ylabel "Latency in nanoseconds" + + set multiplot layout 1,2 + + stats FILENAME u 1 name "a" nooutput + stats FILENAME2 u 1 name "b" nooutput + + min_us = a_min/1000 + max_us = a_max/1000 + avg_us = a_mean/1000 + + set label front sprintf("Minimum: %d us", min_us) at graph 0.01, graph 0.95 + set label front sprintf("Maximum: %d us", max_us) at graph 0.01, graph 0.90 + set label front sprintf("Average: %d us", avg_us) at graph 0.01, graph 0.85 + + plot FILENAME \ + using ($1) title "Legacy socket" lc rgb "red" w points + + unset label + + set yrange [0:2000000] #XDP socket plot YMAX + + min_us = b_min/1000 + max_us = b_max/1000 + avg_us = b_mean/1000 + + set label front sprintf("Minimum: %d us", min_us) at graph 0.01, graph 0.95 + set label front sprintf("Maximum: %d us", max_us) at graph 0.01, graph 0.90 + set label front sprintf("Average: %d us", avg_us) at graph 0.01, graph 0.85 + + plot FILENAME2 \ + using ($1) title "XDP socket" lc rgb "red" w points + + unset multiplot + unset output + + #Replot to GUI + set terminal x11 size 1720,980 + set output + set multiplot layout 1,2 + plot FILENAME \ + using ($1) title "Legacy socket" lc rgb "red" w points + + plot FILENAME2 \ + using ($1) title "XDP socket" lc rgb "red" w points + +pause 10 #in case some one forgets to add -p diff --git a/scripts/latency_single.gnu b/scripts/latency_single.gnu new file mode 100755 index 0000000..66225f9 --- /dev/null +++ b/scripts/latency_single.gnu @@ -0,0 +1,91 @@ +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +# Example cmdline: +# gnuplot -e "FILENAME='input.txt'" latency_single.gnu -p +reset + + set terminal pngcairo size 1920,1080 + set output 'plot_pic.png' + + set size 1,1 + set datafile missing'?' # Skip invalid/missing data + set locale "en_US.utf8" # Set to en_US locale for thousands seperator + + set tmargin 5 + set bmargin 5 + set lmargin 15 + set rmargin 5 + + set yrange [0:YMAX] # up to 1ms latency + + set grid + set key center tmargin + set border + + set style fill solid 0.25 + set boxwidth 0.5 + # set logscale y 10 + + set title PLOT_TITLE + set xlabel "Packet count" + set ylabel "Latency in nanoseconds" + + #set multiplot layout 1 + + stats FILENAME u 1 name "a" nooutput + + min_us = a_min/1000 + max_us = a_max/1000 + avg_us = a_mean/1000 + + set label front sprintf("Minimum: %d us", min_us) at graph 0.01, graph 0.95 + set label front sprintf("Maximum: %d us", max_us) at graph 0.01, graph 0.90 + set label front sprintf("Average: %d us", avg_us) at graph 0.01, graph 0.85 + + plot FILENAME \ + using ($1) title "Legacy socket" lc rgb "red" w points + + unset label + + + unset multiplot + unset output + + #Replot to GUI + set terminal x11 size 1720,980 + set output + #set multiplot layout 1 + plot FILENAME \ + using ($1) title "Legacy socket" lc rgb "red" w points + + +pause 10 #in case some one forgets to add -p diff --git a/scripts/liveplot.gnu b/scripts/liveplot.gnu new file mode 100644 index 0000000..e910eea --- /dev/null +++ b/scripts/liveplot.gnu @@ -0,0 +1,36 @@ +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ +set xrange[0:100] +set yrange[-50:50] +plot filename using ($3) with lines title "nano-seconds" +pause 1 +reread +set autoscale diff --git a/scripts/setup-tsq1a.sh b/scripts/setup-tsq1a.sh new file mode 100755 index 0000000..567eaa2 --- /dev/null +++ b/scripts/setup-tsq1a.sh @@ -0,0 +1,47 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +#Reset qdiscs, setup static IP, setup VLAN & program MAC address +setup_sp1a $IFACE + +$DIR/clock-setup.sh $IFACE +sleep 20 +pkill phc2sys + +$DIR/enable_extts.sh $IFACE +$DIR/enable_pps.sh $IFACE + +echo "Done setup" diff --git a/scripts/setup-tsq1b.sh b/scripts/setup-tsq1b.sh new file mode 100755 index 0000000..84d3159 --- /dev/null +++ b/scripts/setup-tsq1b.sh @@ -0,0 +1,46 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +#Reset qdiscs, setup static IP, setup VLAN & program MAC address +setup_sp1b $IFACE + +$DIR/clock-setup.sh $IFACE +sleep 20 +pkill phc2sys + +$DIR/enable_extts.sh $IFACE + +echo "Done setup" diff --git a/scripts/setup-vs1a.sh b/scripts/setup-vs1a.sh new file mode 100755 index 0000000..b0f2b69 --- /dev/null +++ b/scripts/setup-vs1a.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +#Reset qdiscs, setup static IP, setup VLAN & program MAC address +setup_sp1a $IFACE + +$DIR/clock-setup.sh $IFACE +sleep 30 #Give some time for clock daemons to start. + +#Note: TAPRIO relies on fully synchronized ptp-phc clocks for setting basetime +$DIR/setup_taprio.sh $IFACE +$DIR/setup_vlanrx.sh $IFACE 1 1 +$DIR/setup_vlanrx.sh $IFACE 2 2 + +#Get the right TXQ number and add etf qdisc to it. +get_TXQ_NUM $IFACE +$DIR/setup_etf.sh $IFACE $TXQ_NUM 400000 diff --git a/scripts/setup-vs1b.sh b/scripts/setup-vs1b.sh new file mode 100755 index 0000000..18b07da --- /dev/null +++ b/scripts/setup-vs1b.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +#Reset qdiscs, setup static IP, setup VLAN & program MAC address +setup_sp1b $IFACE + +$DIR/clock-setup.sh $IFACE +sleep 30 #Give some time for clock daemons to start. + +#Route TX packets using 4Q MQPRIO qdisc, required for XDP +$DIR/setup_mqprio.sh $IFACE + +# Steer VLAN prio N packets to RX queue N +$DIR/setup_vlanrx.sh $IFACE 1 1 +$DIR/setup_vlanrx.sh $IFACE 2 2 +$DIR/setup_vlanrx.sh $IFACE 3 1 #TGL only +$DIR/setup_vlanrx.sh $IFACE 6 2 #EHL only diff --git a/scripts/setup_etf.sh b/scripts/setup_etf.sh new file mode 100755 index 0000000..1fa418f --- /dev/null +++ b/scripts/setup_etf.sh @@ -0,0 +1,54 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ $# -eq 1 ];then + echo "Please enter interface, hw queue [0 - N]: ./setup_etf.sh eth0 1" + exit 1 +fi + +IFACE=$1 +NORMAL_QUEUE=$(expr $2 + 1) #TC qdisc id start from 1 instead of 0 + +if [ $# -eq 3 ];then + DELTA=$3 +else + DELTA=50000 +fi + +echo "Running TC-ETF command" + +#ETF qdisc +HANDLE_ID="$( tc qdisc show dev $IFACE | tr -d ':' | awk 'NR==1{print $3}' )" + +# The delta dont really apply to AF_XDP. +tc qdisc replace dev $IFACE parent $HANDLE_ID:$NORMAL_QUEUE etf \ + offload clockid CLOCK_TAI delta $DELTA diff --git a/scripts/setup_mqprio.sh b/scripts/setup_mqprio.sh new file mode 100755 index 0000000..6831763 --- /dev/null +++ b/scripts/setup_mqprio.sh @@ -0,0 +1,40 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 + +#tc qdisc del dev $IFACE parent root 2> /dev/null + +#Universal to EHL/TGL as only 4 queues used +tc qdisc add dev $IFACE root mqprio \ + num_tc 4 map 0 1 2 3 0 0 0 0 0 0 0 0 0 0 0 0 \ + queues 1@0 1@1 1@2 1@3 hw 0 diff --git a/scripts/setup_taprio.sh b/scripts/setup_taprio.sh new file mode 100755 index 0000000..3937ab2 --- /dev/null +++ b/scripts/setup_taprio.sh @@ -0,0 +1,71 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +IFACE=$1 + +if [ $# -ne 1 ];then + echo "Please enter interface : ./setup_taprio.sh eth0" + exit 1 +fi + +echo "Running TC-TAPRIO command" + +# tc qdisc del dev $IFACE parent root 2> /dev/null + +TXQ_COUNT=$(ethtool -l $IFACE | awk 'NR==4{ print $2}') +BASE=$(expr $(date +%s) + 5)000000000 + +# To use replace, we need a base for the first time. +# This command is does nothing if used when there's an existing qdisc. +tc qdisc add dev $IFACE root mqprio \ + num_tc 4 map 0 1 2 3 3 3 3 3 3 3 3 3 3 3 3 0 \ + queues 1@0 1@1 1@2 1@3 hw 0 2&> /dev/null + +if [ $TXQ_COUNT -eq 8 ]; then + tc qdisc replace dev $IFACE parent root handle 100 taprio \ + num_tc 8 map 0 1 2 3 4 5 6 7 0 0 0 0 0 0 0 0 \ + queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 \ + base-time $BASE \ + sched-entry S 43 500000 \ + sched-entry S 42 500000 \ + flags 0x2 +elif [ $TXQ_COUNT -eq 4 ]; then + tc qdisc replace dev $IFACE parent root handle 100 taprio \ + num_tc 4 map 0 1 2 3 0 0 0 0 0 0 0 0 0 0 0 0 \ + queues 1@0 1@1 1@2 1@3 \ + base-time $BASE \ + sched-entry S 0F 500000 \ + sched-entry S 0E 500000 \ + flags 0x2 +else + echo "setup_taprio.sh failed - Invalid TX queue count" +fi \ No newline at end of file diff --git a/scripts/setup_vlanrx.sh b/scripts/setup_vlanrx.sh new file mode 100755 index 0000000..cf5c2d8 --- /dev/null +++ b/scripts/setup_vlanrx.sh @@ -0,0 +1,45 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ $# -ne 3 ];then + echo "Enter interface, vlan priority, hw queue: ./setup_vlanrx.sh eth0 1 1" + exit 1 +fi + +IFACE=$1 +VLAN=$2 +QUEUE=$3 + +# tc qdisc del dev $IFACE parent ffff: 2> /dev/null +# tc qdisc add dev $IFACE ingress +tc filter add dev $IFACE parent ffff: protocol 802.1Q flower \ + vlan_prio $VLAN hw_tc $QUEUE diff --git a/scripts/tsq1a.sh b/scripts/tsq1a.sh new file mode 100755 index 0000000..c7485da --- /dev/null +++ b/scripts/tsq1a.sh @@ -0,0 +1,80 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ $# -eq 0 ]; then + echo "How to run this : $0 < secs ( opt : how long to run the test , def : 50 sec ) >" + exit 1 +fi + +IFACE=$1 + +if [ $# -eq 2 ]; then + TEST_PERIOD=$2 +else + TEST_PERIOD=100 +fi + +IPADDR="169.254.1.11" + +if ! pgrep tsq; then pkill tsq; fi +pkill gnuplot + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +./tsq -L -i $IPADDR -p 7777 -v & + +sleep 5 + +CLK=`ethtool -T $IFACE | grep -Po "(?<=PTP Hardware Clock: )[\d+]"` + +./tsq -T -i $IPADDR -p 7777 -d /dev/ptp$CLK -v -u 1111 & + +$DIR/enable_extts.sh $IFACE + +while [ ! -f tsq-listener-data.txt ] +do + sleep 1 + echo "Waiting for data file" +done + +if [[ $DISPLAY ]];then + echo "Starting plotting" + gnuplot -e "filename='tsq-listener-data.txt'" $DIR/liveplot.gnu & +fi + +sleep $TEST_PERIOD +pkill tsq + +echo "Plot data file : tsq-listener-data.txt is now ready." + +exit 0 + diff --git a/scripts/tsq1b.sh b/scripts/tsq1b.sh new file mode 100755 index 0000000..ee3adc8 --- /dev/null +++ b/scripts/tsq1b.sh @@ -0,0 +1,61 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +if [ $# -eq 0 ]; then + echo "How to run this : $0 < secs (opt: how long to run it)> " + exit 1 +fi + +IFACE=$1 + +if [ $# -eq 2 ]; then + TEST_PERIOD=$2 +else + TEST_PERIOD=100 +fi + +echo $TEST_PERIOD + +if ! pgrep tsq; then pkill tsq; fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +#Launch only talker on platform B +IPADDR="169.254.1.11" + +CLK=`ethtool -T $IFACE | grep -Po "(?<=PTP Hardware Clock: )[\d+]"` + +./tsq -T -i $IPADDR -p 7777 -d /dev/ptp$CLK -v -u 2222 & + +sleep $TEST_PERIOD + +exit 0 diff --git a/scripts/vs1a.sh b/scripts/vs1a.sh new file mode 100755 index 0000000..a461850 --- /dev/null +++ b/scripts/vs1a.sh @@ -0,0 +1,101 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +pkill gnuplot +pkill txrx-tsn +pkill iperf3 + +if [ $# -eq 3 ]; then + IFACE=$1 + NUMPKTS=$2 + SIZE=$3 +else + echo "Note: Using default PKT3a params" + IFACE=$1 + NUMPKTS=1000000 + SIZE=64 +fi + +TXTIME_OFFSET=20000 + +INTERVAL=1000000 +EARLY_OFFSET=700000 +XDP_INTERVAL=200000 +XDP_EARLY_OFFSET=100000 + +# Get this script's dir because cfg file is stored together with this script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +SLEEP_SEC=$(((($NUMPKTS * $INTERVAL) / $SEC_IN_NSEC) + 10)) +XDP_SLEEP_SEC=$(((($NUMPKTS * $XDP_INTERVAL) / $SEC_IN_NSEC) + 10)) + +echo "PHASE 1: AF_PACKET transmit ($SLEEP_SEC seconds)" +$DIR/iperf3-bg-client.sh +sleep 5 + +get_TXQ_NUM $IFACE + +./txrx-tsn -i $IFACE -PtTq $TXQ_NUM -n $NUMPKTS -l $SIZE -y $INTERVAL \ + -e $EARLY_OFFSET -o $TXTIME_OFFSET > /dev/shm/afpkt-txtstamps.txt & +TXRX_PID=$! +taskset -p 4 $TXRX_PID +chrt --fifo -p 90 $TXRX_PID +ps -o psr,pri,pid,cmd $TXRX_PID + +sleep $SLEEP_SEC +pkill iperf3 +pkill txrx-tsn + +cp /dev/shm/afpkt-txtstamps.txt . & + +echo "PHASE 2: AF_XDP transmit ($XDP_SLEEP_SEC seconds)" +$DIR/iperf3-bg-client.sh +sleep 5 + +get_XDPTXQ_NUM $IFACE + +./txrx-tsn -XztTi $IFACE -q $XDPTXQ_NUM -n $NUMPKTS -l $SIZE -y $XDP_INTERVAL \ + -e $XDP_EARLY_OFFSET -o $TXTIME_OFFSET > /dev/shm/afxdp-txtstamps.txt & +TXRX_PID=$! +taskset -p 4 $TXRX_PID +chrt --fifo -p 90 $TXRX_PID +ps -o psr,pri,pid,cmd $TXRX_PID + +sleep $XDP_SLEEP_SEC +pkill iperf3 +pkill txrx-tsn + +cp /dev/shm/afxdp-txtstamps.txt . + +echo Done! +exit diff --git a/scripts/vs1b.sh b/scripts/vs1b.sh new file mode 100755 index 0000000..46b055d --- /dev/null +++ b/scripts/vs1b.sh @@ -0,0 +1,108 @@ +#!/bin/bash +#/****************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# *****************************************************************************/ + +pkill gnuplot +pkill txrx-tsn +pkill iperf3 + +if [ $# -eq 3 ]; then + IFACE=$1 + NUMPKTS=$2 + SIZE=$3 +else + echo "Note: Using default PKT3a params" + IFACE=$1 + NUMPKTS=1000000 + SIZE=64 +fi + +TXTIME_OFFSET=20000 + +INTERVAL=1000000 +EARLY_OFFSET=700000 +XDP_INTERVAL=200000 +XDP_EARLY_OFFSET=100000 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $DIR/helpers.sh + +SLEEP_SEC=$(((($NUMPKTS * $INTERVAL) / $SEC_IN_NSEC) + 10)) +XDP_SLEEP_SEC=$(((($NUMPKTS * $XDP_INTERVAL) / $SEC_IN_NSEC) + 10)) + +echo 0 > afpkt-rxtstamps.txt +echo 0 > afxdp-rxtstamps.txt +echo 0 > afpkt-traffic.txt +echo 0 > afxdp-traffic.txt + +echo "PHASE 1: AF_PACKET receive ($SLEEP_SEC seconds)" +$DIR/iperf3-bg-server.sh +sleep 5 + +get_TXQ_NUM $IFACE + +taskset -c 2 ./txrx-tsn -Pri $IFACE -q $TXQ_NUM > /dev/shm/afpkt-rxtstamps.txt & +sleep $SLEEP_SEC +pkill txrx-tsn +pkill iperf3 + +cp /dev/shm/afpkt-rxtstamps.txt . & + +echo "PHASE 2: AF_XDP receive ($XDP_SLEEP_SEC seconds)" +$DIR/iperf3-bg-server.sh +sleep 5 + +get_XDPTXQ_NUM $IFACE + +taskset -c 2 ./txrx-tsn -Xzri $IFACE -q $XDPTXQ_NUM > /dev/shm/afxdp-rxtstamps.txt & +sleep $XDP_SLEEP_SEC +pkill iperf3 +pkill txrx-tsn + +cp /dev/shm/afxdp-rxtstamps.txt . + +echo "PHASE 3: Calculating.." +pkill gnuplot + +stop_if_empty "afpkt-rxtstamps.txt" +calc_rx_u2u "afpkt-rxtstamps.txt" +calc_rx_duploss "afpkt-rxtstamps.txt" + +stop_if_empty "afxdp-rxtstamps.txt" +calc_rx_u2u "afxdp-rxtstamps.txt" +calc_rx_duploss "afxdp-rxtstamps.txt" + +save_result_files $(basename $0 .sh) $NUMPKTS $SIZE $INTERVAL + +gnuplot -e "FILENAME='afpkt-traffic.txt';FILENAME2='afxdp-traffic.txt';" $DIR/latency_dual.gnu -p 2> /dev/null & + +while [[ ! -s plot_pic.png ]]; do sleep 5; done +cp plot_pic.png results-$ID/plot-$(basename $0 .sh)-$NUMPKTS-$SIZE-$INTERVAL-$XDP_INTERVAL-$IDD.png diff --git a/src/Makefile b/src/Makefile new file mode 100644 index 0000000..a8eae04 --- /dev/null +++ b/src/Makefile @@ -0,0 +1,29 @@ +CC?=gcc + +OPT=-O2 -g +EXTRA_CFLAGS += -fstack-protector-strong -fPIE -fPIC -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security +CFLAGS=$(OPT) $(EXTRA_CFLAGS) -Wall -Wextra -Wno-parentheses -Wno-missing-field-initializers + +LDFLAGS += -z noexecstack -z relro -z now -pie +EXTRA_LDFLAGS += -pthread -lbpf -lelf + +.PHONY: all +all: tsq txrx-tsn + +tsq.o: tsq.c + $(CC) $(CFLAGS) -c $< -o $@ + +tsq: tsq.o + $(CC) $(LDFLAGS) $^ -o $@ + $(RM) $@.o + +txrx%.o: txrx%.c txrx%.h + $(CC) $(CFLAGS) $(EXTRA_LDFLAGS) -c $< -o $@ + +txrx-tsn: txrx.o txrx-afxdp.o txrx-afpkt.o + $(CC) $(LDFLAGS) $^ $(EXTRA_LDFLAGS) -o txrx-tsn + $(RM) $@.o $^ + +clean: + $(RM) tsq txrx-tsn *~ .[!.]*.un~ .[!.]*.o .[!.]*.a + $(RM) `find . -name "*.[oa]" -o -name "\#*\#" -o -name TAGS -o -name core -o -name "*.orig"` diff --git a/src/opcua-tsn/CMakeLists.txt b/src/opcua-tsn/CMakeLists.txt new file mode 100644 index 0000000..c3601d8 --- /dev/null +++ b/src/opcua-tsn/CMakeLists.txt @@ -0,0 +1,25 @@ +if (${CROSS_COMPILE}) + set( CMAKE_FIND_ROOT_PATH ${OVERRIDE_CMAKE_ROOT_DIR_HOST} ${OVERRIDE_CMAKE_ROOT_DIR_NATIVE} ${OVERRIDE_CMAKE_ROOT_CROSS_DIR} ) + set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY ) + set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY ) + set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY ) + set( CMAKE_MODULE_PATH ${OVERRIDE_CMAKE_MODULE} ) +endif(${CROSS_COMPILE}) + +cmake_minimum_required(VERSION 3.10) + +# Debug to have debug symbols +# Release for optimisations and no debug symbols +# RelWithDebInfo for optimisation and debug symbols +set(CMAKE_BUILD_TYPE RelWithDebInfo) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -g -D_FORTIFY_SOURCE=2 \ + -fstack-protector-strong -fPIE -fPIC \ + -Wformat -Wformat-security -Wall -Wextra -Wno-parentheses \ + -Wno-missing-field-initializers -z noexecstack -z relro -z now -pie") + +add_executable(opcua-server multicallback_server.c opcua_common.c + opcua_publish.c json_helper.c opcua_subscribe.c + opcua_custom.c opcua_datasource.c) + +target_link_libraries(opcua-server open62541 pthread elf rt json-c) diff --git a/src/opcua-tsn/json_helper.c b/src/opcua-tsn/json_helper.c new file mode 100644 index 0000000..fa9aa48 --- /dev/null +++ b/src/opcua-tsn/json_helper.c @@ -0,0 +1,186 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include +#include +#include +#include +#include + +#include "json_helper.h" +#include "opcua_utils.h" + +typedef struct json_object JsObj; + +void ensureType(JsObj *toCheck, json_type type, JsObj *parent, char *key) +{ + json_type objType = json_object_get_type(toCheck); + + if (objType != type) { + log_error("Key '%s' in object '%s' has invalid type. Exiting.", + key, json_object_to_json_string(parent)); + exit(EXIT_FAILURE); + } + + return; +} + +char *getOptionalStr(struct json_object *parent, + char *key, char *defaultVal) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_string, parent, key); + return strdup(json_object_get_string(val)); + } + + return defaultVal; +} + +int getOptionalInt(struct json_object *parent, char *key, int defaultVal) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_int, parent, key); + return json_object_get_int(val); + } + + return defaultVal; +} + +bool getOptionalBool(struct json_object *parent, char *key, bool defaultVal) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_boolean, parent, key); + return json_object_get_boolean(val); + } + + return defaultVal; +} + +int64_t getOptionalInt64(struct json_object *parent, char *key, int64_t defaultVal) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_int, parent, key); + return json_object_get_int64(val); + } + + return defaultVal; +} + +char *getString(struct json_object *parent, char *key) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_string, parent, key); + return strndup(json_object_get_string(val), json_object_get_string_len(val)); + } + + log_error("Key '%s' not found in object '%s'. Exiting.", key, + json_object_to_json_string(parent)); + exit(EXIT_FAILURE); +} + +int getInt(struct json_object *parent, char *key) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_int, parent, key); + return json_object_get_int(val); + } + + log_error("Key '%s' not found in object '%s'. Exiting.", key, + json_object_to_json_string(parent)); + exit(EXIT_FAILURE); +} + +bool getBool(struct json_object *parent, char *key) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_boolean, parent, key); + return json_object_get_boolean(val); + } + + log_error("Key '%s' not found in object '%s'. Exiting.", key, + json_object_to_json_string(parent)); + exit(EXIT_FAILURE); +} + +int64_t getInt64(struct json_object *parent, char *key) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) { + ensureType(val, json_type_int, parent, key); + return json_object_get_int64(val); + } + + log_error("Key '%s' not found in object '%s'. Exiting.", key, + json_object_to_json_string(parent)); + exit(EXIT_FAILURE); +} + +int countChildrenEntries(struct json_object *json) +{ + return json_object_object_length(json); +} + +struct json_object *getValue(struct json_object *parent, char *key) +{ + JsObj *val; + json_bool found = json_object_object_get_ex(parent, key, &val); + + if (found) + return val; + + log_error("Key '%s' not found in object '%s'. Exiting.", key, + json_object_to_json_string(parent)); + exit(EXIT_FAILURE); +} diff --git a/src/opcua-tsn/json_helper.h b/src/opcua-tsn/json_helper.h new file mode 100644 index 0000000..a8e560e --- /dev/null +++ b/src/opcua-tsn/json_helper.h @@ -0,0 +1,52 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _JSON_HELPER_H +#define _JSON_HELPER_H + +#include +#include + +char *getOptionalStr(struct json_object *parent, char *key, char *defaultVal); +int getOptionalInt(struct json_object *parent, char *key, int defaultVal); +bool getOptionalBool(struct json_object *parent, char *key, bool defaultVal); +int64_t getOptionalInt64(struct json_object *parent, char *key, int64_t defaultVal); + +char *getString(struct json_object *json, char *key); +int getInt(struct json_object *json, char *key); +bool getBool(struct json_object *json, char *key); +int64_t getInt64(struct json_object *parent, char *key); + +int countChildrenEntries(struct json_object *json); +struct json_object *getValue(struct json_object *json, char *key); + +#endif diff --git a/src/opcua-tsn/multicallback_server.c b/src/opcua-tsn/multicallback_server.c new file mode 100644 index 0000000..659a5dc --- /dev/null +++ b/src/opcua-tsn/multicallback_server.c @@ -0,0 +1,230 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opcua_common.h" +#include "opcua_publish.h" +#include "opcua_subscribe.h" +#include "opcua_custom.h" +#include "opcua_datasource.h" + +#define VERBOSE 0 + +/* Globals */ +struct threadParams *g_thread; +UA_UInt16 g_threadRun; +UA_Boolean g_running = true; +UA_NodeId g_writerGroupIdent; +struct ServerData *g_sData; +int verbose = VERBOSE; + +static void stopHandler(int sign) +{ + (void) sign; + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_SERVER, "received ctrl-c"); + g_running = false; +} + +int configureServer(struct ServerData *sdata) +{ + struct timespec ts; + + int ret = clock_gettime(CLOCK_TAI, &ts); + catch_err(ret == -1, "Failed to get current time"); + + /* Initialize server start time to +3 secs from now */ + sdata->startTime = (UA_UInt64) ((ts.tv_sec + 3) * 1E9L); + + sdata->transportProfile = + UA_STRING("http://opcfoundation.org/UA-Profile/Transport/pubsub-eth-uadp"); + + for (int i = 0; i < sdata->pubCount; i++) { + struct PublisherData *pub = &sdata->pubData[i]; + pub->writeFunc = &dummyDSWrite; + if( pub->twoWayData == false) + pub->readFunc = &pubGetDataToTransmit; + else + pub->readFunc = &pubReturnGetDataToTransmit; + } + + for (int i = 0; i < sdata->subCount; i++) { + struct SubscriberData *sub = &sdata->subData[i]; + sub->readFunc = &dummyDSRead; + if ( sub->twoWayData == false) + sub->writeFunc = &subStoreDataReceived; + else + sub->writeFunc = &subReturnStoreDataReceived; + } + + return 0; + +error: + return -1; +} + +static UA_Server *setupOpcuaServer(struct ServerData *sdata) +{ + UA_NodeId connId = {}; + UA_Server *server = UA_Server_new(); + UA_ServerConfig *config = UA_Server_getConfig(server); + struct PublisherData *pub = NULL; + struct SubscriberData *sub = NULL; + int ret = 0; + + UA_ServerConfig_setMinimal(config, getpid(), NULL); + + /* Setup transport layers through UDPMP and Ethernet */ + config->pubsubTransportLayersSize = 0; + config->pubsubTransportLayers = + (UA_PubSubTransportLayer *) UA_calloc(2, sizeof(UA_PubSubTransportLayer)); + catch_err(config->pubsubTransportLayers == NULL, "Out of memory"); + + if (sdata->useXDP) + config->pubsubTransportLayers[0] = UA_PubSubTransportLayerEthernetXDP(); + else + config->pubsubTransportLayers[0] = UA_PubSubTransportLayerEthernetETF(); + + config->pubsubTransportLayersSize++; + + for (int i = 0; i < sdata->pubCount; i++) { + pub = &sdata->pubData[i]; + addPubSubConnection(server, &connId, sdata, pub); + ret = createPublisher(server, sdata, pub, &connId); + catch_err(ret == -1, "createPublisher() returned -1"); + } + + for (int i = 0; i < sdata->subCount; i++) { + sub = &sdata->subData[i]; + + /* Skip adding a new socket (PubSubConnection) + * using XDP and already have a publisher on the same interface's queue. + * This is because XDP sockets are bi-directional, so both + * TX/RX queue is bound together to a single socket. + * Uni direction sockets are available in 5.5+ kernels only and + * requires a different bpf program. + */ + if (sdata->pubCount && sdata->useXDP && sub->xdpQueue == pub->xdpQueue && + strcmp(sdata->pubInterface, sdata->subInterface) == 0) { + log("Round trip AF_XDP same queue, same interface, sharing layer"); + } else if (sdata->pubCount && sdata->useXDP) { + log("Round trip AF_XDP different queue use new layer"); + config->pubsubTransportLayers[1] = UA_PubSubTransportLayerEthernetXDP(); + config->pubsubTransportLayersSize++; + addSubConnection(server, &connId, sdata, sub); + } else if (sdata->pubCount) { + log("Round trip AF_PACKET use new layer"); + config->pubsubTransportLayers[1] = UA_PubSubTransportLayerEthernetETF(); + config->pubsubTransportLayersSize++; + addSubConnection(server, &connId, sdata, sub); + } else { + log("Single trip AF_XDP or AF_PACKET addSubConnection only"); + addSubConnection(server, &connId, sdata, sub); + } + + ret = createSubscriber(server, sdata, sub, &connId); + catch_err(ret == -1, "createSubscriber() returned -1"); + } + + return server; + +error: + UA_Server_delete(server); + return NULL; +} + +int main(int argc, char **argv) +{ + (void) argc; + + int i, ret = 0; + UA_UInt16 threadCount = 0; + signal(SIGINT, stopHandler); + signal(SIGTERM, stopHandler); + + struct ServerData *sdata = parseArgs(argv); + catch_err(sdata == NULL, "parseArgs() returned NULL"); + + ret = configureServer(sdata); + catch_err(ret == -1, "configureServer() returned -1"); + + threadCount = sdata->pubCount + sdata->subCount; + g_threadRun = 0; + g_thread = calloc(threadCount, sizeof(struct threadParams)); + g_sData = sdata; + + /* Pub/sub threads will be started by the WG/RG addRepeatedCallback() */ + UA_Server *server = setupOpcuaServer(sdata); + catch_err(server == NULL, "setupOpcuaServer() returned NULL"); + + UA_StatusCode servrun = UA_Server_run(server, &g_running); + + /* Pub/sub threads are started by the server. Wait till they finish. */ + for (i = 0; i < g_threadRun; i++) { + ret = pthread_join(g_thread[i].id, NULL); + if (ret != 0) + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "\nPthread Join Failed:%ld\n", g_thread[i].id); + } + + /* Teardown and free any malloc-ed memory (incl those by strndup) */ + UA_Server_delete(server); + + free_resources(sdata); + + free(g_thread); + + return servrun == UA_STATUSCODE_GOOD ? EXIT_SUCCESS : EXIT_FAILURE; + +error: + exit(-1); +} diff --git a/src/opcua-tsn/opcua_common.c b/src/opcua-tsn/opcua_common.c new file mode 100644 index 0000000..93e22df --- /dev/null +++ b/src/opcua-tsn/opcua_common.c @@ -0,0 +1,334 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opcua_common.h" +#include "json_helper.h" + +#define PKT_COUNT_MAX 1000000 +#define CYCLETIME_NS_MAX 5000000 + +#define HUNDRED_USEC_NSEC 100000 +#define ONE_MSEC_NSEC 1000000 +#define TEN_MSEC_NSEC 10000000 +#define FIVE_DIGIT_MAX 99999 + +/* TODO: check if we can reuse this + * Set threads to use RT scheduling so that we pre-empt all other + * less important tasks + * Set CPU affinity to ensure we're not migrated to another processor + * to improve performance + */ +int setRtPriority(pthread_t thread, int priority, uint32_t cpu) +{ + catch_err(priority < 0, "Priority must be greater than 0"); + + cpu_set_t cpuset; + int policy; + struct sched_param sp; + int ret = pthread_getschedparam(thread, &policy, &sp); + catch_err(ret != 0, "Error getting thread scheduler"); + + sp.sched_priority = priority; + ret = pthread_setschedparam(thread, SCHED_FIFO, &sp); + catch_err(ret != 0, "Failed to set thread scheduler"); + + if (cpu == 0) + return 0; + + CPU_ZERO(&cpuset); + CPU_SET(cpu, &cpuset); + ret = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); + catch_err(ret != 0, "Failed to set thread CPU affinity"); + return 0; + +error: + perror("Failed to set RT proirity"); + return -1; +} + +void free_resources(struct ServerData *sdata) +{ + int i = 0; + + for (i = 0; i < sdata->pubCount; i++) { + if (sdata->pubData[i].url) + free(sdata->pubData[i].url); + } + + if (sdata->pubData) + free(sdata->pubData); + + for (i = 0; i < sdata->subCount; i++) { + if (sdata->subData[i].url) + free(sdata->subData[i].url); + + if (sdata->subData[i].subscriberOutputFileName) + free(sdata->subData[i].subscriberOutputFileName); + + if (sdata->subData[i].fpSubscriberOutput) + fclose(sdata->subData[i].fpSubscriberOutput); + } + + if (sdata->subData) + free(sdata->subData); + + if(sdata->subInterface) + free(sdata->subInterface); + + if(sdata->subInterface) + free(sdata->pubInterface); + + free(sdata); +} + +struct ServerData *parseJson(struct json_object *json) +{ + struct ServerData *s = malloc(sizeof(struct ServerData)); + catch_err(s == NULL, "ServerData - Out of memory"); + + log("Parsing OPCUA server"); + s->pubInterface = getString(json, "publisher_interface"); + s->subInterface = getString(json, "subscriber_interface"); + + s->useXDP = getBool(json, "use_xdp"); + + s->pollingDurationNs = getInt(json, "polling_duration_ns"); + catch_err(s->pollingDurationNs < 0 || s->pollingDurationNs > TEN_MSEC_NSEC, + "Invalid polling_duration_ns"); + + s->packetCount = getInt64(json, "packet_count"); + catch_err(s->packetCount < 1 || s->packetCount > PKT_COUNT_MAX, + "Invalid packet_count"); + + s->cycleTimeNs = getInt64(json, "cycle_time_ns"); + catch_err(s->cycleTimeNs < HUNDRED_USEC_NSEC || s->cycleTimeNs > CYCLETIME_NS_MAX, + "Invalid cycle_time_ns"); + + debug("Found: pub if %s, sub if %s, cycleTimeNs %ld pollNs %d, packet_count %ld", + s->pubInterface, s->subInterface, s->cycleTimeNs, + s->pollingDurationNs, s->packetCount); + + struct PublisherData *pubData = NULL; + struct SubscriberData *subData = NULL; + + struct json_object *pubs = getValue(json, "publishers"); + int pubCount = countChildrenEntries(pubs); + pubData = calloc(pubCount, sizeof(struct PublisherData)); + if (pubData == NULL) { + perror("PubData - Out of memory"); + goto error; + } + + struct json_object *subs = getValue(json, "subscribers"); + int subCount = countChildrenEntries(subs); + subData = calloc(subCount, sizeof(struct SubscriberData)); + if (subData == NULL) { + free(pubData); + perror("PubData - Out of memory"); + goto error; + } + + s->pubCount = pubCount; + s->subCount = subCount; + s->pubData = pubData; + s->subData = subData; + log("Identified: %d publishers, %d subscribers", s->pubCount, s->subCount); + + struct json_object_iter iter; + int cpuAff = 0; + int i = 0; + + json_object_object_foreachC(pubs, iter) { + struct json_object *pubJson = iter.val; + struct PublisherData *pd = &pubData[i]; + + pd->earlyOffsetNs = getInt(pubJson, "early_offset_ns"); + catch_err(pd->earlyOffsetNs < 0 || pd->earlyOffsetNs > ONE_MSEC_NSEC, + "Invalid early_offset_ns"); + + pd->publishOffsetNs = getInt(pubJson, "publish_offset_ns"); + catch_err(pd->publishOffsetNs < 0 || pd->publishOffsetNs > TEN_MSEC_NSEC, + "Invalid publish_offset_ns"); + + pd->socketPriority = getInt(pubJson, "socket_prio"); + catch_err(pd->socketPriority < 0 || pd->socketPriority > 7, + "Invalid socket_prio"); + + if (s->useXDP) { + pd->xdpQueue = getInt(pubJson, "xdp_queue"); + catch_err(pd->xdpQueue < 0 || pd->xdpQueue > 3, "Invalid xdp_queue"); + } else { + pd->xdpQueue = -1; + } + + pd->url = getString(pubJson, "url"); + + pd->id = getInt(pubJson, "pub_id"); + catch_err(pd->id < 0 || pd->id > FIVE_DIGIT_MAX, "Invalid pub_id"); + + pd->dataSetWriterId = getInt(pubJson, "dataset_writer_id"); + catch_err(pd->dataSetWriterId < 0, + "Negative dataset_writer_id is not valid"); + + pd->writerGroupId = getInt(pubJson, "writer_group_id"); + catch_err(pd->writerGroupId < 0, + "Negative writerGroupId is not valid"); + + pd->twoWayData = getBool(pubJson, "two_way_data"); + + cpuAff = getInt(pubJson, "cpu_affinity"); + catch_err(cpuAff < 0 || cpuAff > 3, "Invalid cpu_affinity"); + pd->cpuAffinity = cpuAff; + + log("Publisher: %s %s CPU%ld", + pd->url, s->useXDP ? "AF_XDP" : "AF_PACKET", pd->cpuAffinity); + + i++; + } + + int j = 0; + + json_object_object_foreachC(subs, iter) { + struct json_object *subJson = iter.val; + struct SubscriberData *sd = &subData[j]; + + sd->offsetNs = getInt(subJson, "offset_ns"); + catch_err(sd->offsetNs < 0 || sd->offsetNs > TEN_MSEC_NSEC, + "Invalid offset_ns"); + + if (s->useXDP) { + sd->xdpQueue = getInt(subJson, "xdp_queue"); + catch_err(sd->xdpQueue < 0 || sd->xdpQueue > 3, "Invalid xdp_queue"); + } else { + sd->xdpQueue = -1; + } + + sd->url = getString(subJson, "url"); + + sd->id = getInt(subJson, "sub_id"); + catch_err(sd->id < 0 || sd->id > FIVE_DIGIT_MAX, "Invalid sub_id"); + + sd->subscribedPubId = getInt(subJson, "subscribed_pub_id"); + catch_err(sd->subscribedPubId < 0 || sd->subscribedPubId > FIVE_DIGIT_MAX, + "Invalid subscribed_pub_id"); + + sd->subscribedDSWriterId = getInt(subJson, "subscribed_dataset_writer_id"); + catch_err(sd->subscribedDSWriterId < 0 || sd->subscribedDSWriterId > FIVE_DIGIT_MAX, + "Invalid subscribed_dataset_writer_id"); + + sd->subscribedWGId = getInt(subJson, "subscribed_writer_group_id"); + catch_err(sd->subscribedWGId < 0 || sd->subscribedWGId > FIVE_DIGIT_MAX, + "Invalid subscribed_writer_group_id"); + + sd->twoWayData = getBool(subJson, "two_way_data"); + + cpuAff = getInt(subJson, "cpu_affinity"); + catch_err(cpuAff < 0 || cpuAff > 3, "Invalid cpu_affinity"); + sd->cpuAffinity = cpuAff; + + sd->subscriberOutputFileName = getString(subJson, "subscriber_output_file"); + sd->fpSubscriberOutput = fopen(sd->subscriberOutputFileName, "w" ); + if (sd->fpSubscriberOutput) + fflush(sd->fpSubscriberOutput); + + log("Subscriber: %s %s CPU%ld", + sd->url, s->useXDP ? "AF_XDP" : "AF_PACKET", sd->cpuAffinity); + + j++; + } + + return s; + +error: + if (s) + free_resources(s); + + return NULL; +} + +struct ServerData *parseArgs(char **argv) +{ + struct json_object *js = NULL, *opcuaJson; + struct ServerData *sdata; + + char *jsonPath = argv[1]; + catch_err(jsonPath == NULL, "Path to JSON not specified (usage: ./opcua-server some_valid.json"); + + debug("Reading: %s", jsonPath); + + js = json_object_from_file(jsonPath); + catch_err(js == NULL, "Failed to extract json objects"); + + opcuaJson = getValue(js, "opcua_server"); + sdata = parseJson(opcuaJson); + catch_err(sdata == NULL, "Failed to parse JSON"); + + /* Free json since we shouldn't be using it outside of parsing */ + json_object_put(js); + + return sdata; + +error: + if (js) + json_object_put(js); + + return NULL; +} + +UA_UInt64 as_nanoseconds(struct timespec *ts) +{ + return (uint64_t)(ts->tv_sec) * (uint64_t)1E9L + (uint64_t)(ts->tv_nsec); +} diff --git a/src/opcua-tsn/opcua_common.h b/src/opcua-tsn/opcua_common.h new file mode 100644 index 0000000..2c07246 --- /dev/null +++ b/src/opcua-tsn/opcua_common.h @@ -0,0 +1,113 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_COMMON_H_ +#define _OPCUA_COMMON_H_ + +#include +#include + +#include +#include +#include + +#include "opcua_utils.h" + +typedef UA_StatusCode (DSCallbackRead)(UA_Server *server, + const UA_NodeId *sessionId, void *sessionContext, + const UA_NodeId *nodeId, void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *value); + +typedef UA_StatusCode (DSCallbackWrite)(UA_Server *server, + const UA_NodeId *sessionId, void *sessionContext, + const UA_NodeId *nodeId, void *nodeContext, + const UA_NumericRange *range, const UA_DataValue *value); + +// TODO: Some of these params are no longer required. +struct PublisherData { + int socketPriority; + int xdpQueue; + int32_t earlyOffsetNs; + int32_t publishOffsetNs; + char *url; + int32_t id; + int32_t dataSetWriterId; + int32_t writerGroupId; + int32_t prev_sequence_num; + bool twoWayData; + size_t cpuAffinity; + DSCallbackRead *readFunc; + DSCallbackWrite *writeFunc; +}; + +struct SubscriberData { + int xdpQueue; + int32_t offsetNs; + char *url; + int32_t id; + int32_t subscribedPubId; + int32_t subscribedDSWriterId; + int32_t subscribedWGId; + FILE *fpSubscriberOutput; + char *subscriberOutputFileName; + char *temp_targetVars; + char *temp_dataSetMetaData; + bool twoWayData; + size_t cpuAffinity; + DSCallbackRead *readFunc; + DSCallbackWrite *writeFunc; +}; + +struct ServerData { + char *pubInterface; + char *subInterface; + int64_t cycleTimeNs; + int32_t pollingDurationNs; + int cpu; + bool useXDP; + + int pubCount; + int subCount; + struct PublisherData *pubData; + struct SubscriberData *subData; + + UA_UInt64 startTime; + UA_String transportProfile; + UA_UInt64 packetCount; +}; + +void free_resources(struct ServerData *sdata); +struct ServerData *parseArgs(char **argv); +int setRtPriority(pthread_t thread, int priority, uint32_t cpu); +UA_UInt64 as_nanoseconds(struct timespec *ts); + +#endif diff --git a/src/opcua-tsn/opcua_custom.c b/src/opcua-tsn/opcua_custom.c new file mode 100644 index 0000000..af34d18 --- /dev/null +++ b/src/opcua-tsn/opcua_custom.c @@ -0,0 +1,319 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "opcua_custom.h" +#include "opcua_common.h" + +#define CLOCKID CLOCK_TAI +#define ONESEC_IN_NSEC (1000 * 1000 * 1000) +#define PUB_THREAD_PRIORITY 91 +#define SUB_THREAD_PRIORITY 90 + +extern struct threadParams *g_thread; +extern UA_UInt16 g_threadRun; +extern UA_Boolean g_running; +extern UA_NodeId g_writerGroupIdent; +extern struct ServerData *g_sData; +UA_UInt16 g_indexPub; +UA_UInt16 g_indexSub; + +static void normalize(struct timespec *timeSpecValue) +{ + /* In nsec is bigger than sec, we increment the sec + * and realign the nsec value + */ + while (timeSpecValue->tv_nsec > (ONESEC_IN_NSEC - 1)) { + timeSpecValue->tv_sec += 1; + timeSpecValue->tv_nsec -= ONESEC_IN_NSEC; + } + + /* If the nsec is in negative, we shd go back to the + * previous second, and realign the nanosec + */ + while (timeSpecValue->tv_nsec < 0) { + timeSpecValue->tv_sec -= 1; + timeSpecValue->tv_nsec += ONESEC_IN_NSEC; + } +} + +static pthread_t threadCreation(UA_Int16 threadPriority, size_t coreAffinity, + void *(*thread) (void *), char *applicationName, + void *serverConfig) +{ + struct sched_param schedParam; + pthread_t threadID; + cpu_set_t cpuset; + UA_Int32 ret = 0; + + /* Set scheduler and thread priority */ + threadID = pthread_self(); + schedParam.sched_priority = threadPriority; + + ret = pthread_setschedparam(threadID, SCHED_FIFO, &schedParam); + if (ret != 0) { + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "pthread_setschedparam: failed\n"); + exit(1); + } + + /* + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND,\ + "\npthread_setschedparam:%s Thread priority is %d \n", \ + applicationName, schedParam.sched_priority); + */ + + /* Set thread CPU affinity */ + CPU_ZERO(&cpuset); + CPU_SET(coreAffinity, &cpuset); + + ret = pthread_setaffinity_np(threadID, sizeof(cpu_set_t), &cpuset); + if (ret) { + UA_LOG_ERROR(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "pthread_setaffinity_np ret: %s\n", strerror(ret)); + exit(1); + } + + /* TODO: check if this should be run earlier. Also check setRTpriority() */ + ret = pthread_create(&threadID, NULL, thread, serverConfig); + if (ret != 0) { + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + ":%s Cannot create thread\n", applicationName); + } + + if (CPU_ISSET(coreAffinity, &cpuset)) { + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "%s CPU CORE: %ld\n", applicationName, coreAffinity); + } + + return threadID; +} + +/* Custom publisher thread */ +void *pub_thread(void *arg) +{ + struct timespec nextnanosleeptime; + struct timespec temp_t; + UA_UInt64 tx_timestamp; + UA_ServerCallback pubCallback; + UA_Server *server; + UA_WriterGroup *currentWriterGroup; + UA_Int32 earlyOffsetNs = 0; + UA_Int32 publishOffsetNs = 0; + UA_UInt32 ind = 0; + UA_UInt64 cycleTimeNs = 0; + struct PublisherData *pData; + + /* Initialise value for nextnanosleeptime timespec */ + tx_timestamp = 0; + nextnanosleeptime.tv_nsec = 0; + nextnanosleeptime.tv_sec = 0; + threadParams *threadArgumentsPublisher = (threadParams *)arg; + server = threadArgumentsPublisher->server; + pubCallback = threadArgumentsPublisher->callback; + currentWriterGroup = (UA_WriterGroup *)threadArgumentsPublisher->data; + pData = (struct PublisherData *)g_sData->pubData; + ind = threadArgumentsPublisher->data_index; + earlyOffsetNs = pData[ind].earlyOffsetNs; + publishOffsetNs = pData[ind].publishOffsetNs; + cycleTimeNs = g_sData->cycleTimeNs; + + /* Define Ethernet ETF transport settings */ + UA_EthernetETFWriterGroupTransportDataType ethernetETFtransportSettings; + memset(ðernetETFtransportSettings, 0, + sizeof(UA_EthernetETFWriterGroupTransportDataType)); + + ethernetETFtransportSettings.txtime_enabled = UA_TRUE; + ethernetETFtransportSettings.transmission_time = 0; + + /* Encapsulate ETF config in transportSettings */ + UA_ExtensionObject transportSettings; + memset(&transportSettings, 0, sizeof(UA_ExtensionObject)); + + /* TODO: transportSettings encoding and type to be defined */ + transportSettings.content.decoded.data = ðernetETFtransportSettings; + currentWriterGroup->config.transportSettings = transportSettings; + + /* Get current time and compute the next nanosleeptime to the nearest 5th second */ + clock_gettime(CLOCKID, &temp_t); + tx_timestamp = ((temp_t.tv_sec + 4) / 5) * 5; + /* Add 3 secs delay for publisher (subscriber starts earlier) */ + tx_timestamp += 3; + tx_timestamp *= ONESEC_IN_NSEC; + /* Add publish offset to tx time*/ + tx_timestamp += publishOffsetNs; + + /* First packet tx_timestamp */ + ethernetETFtransportSettings.transmission_time = tx_timestamp; + + while (g_running) { + /* Calculate publisher wake up time using earlyOffsetNs + * Publisher wakes up earlier to be able to catch + * the ETF transmission time + */ + nextnanosleeptime.tv_nsec = (tx_timestamp - earlyOffsetNs) % ONESEC_IN_NSEC; + nextnanosleeptime.tv_sec = (tx_timestamp - earlyOffsetNs) / ONESEC_IN_NSEC; + clock_nanosleep(CLOCKID, TIMER_ABSTIME, &nextnanosleeptime, NULL); + pubCallback(server, currentWriterGroup); + tx_timestamp += cycleTimeNs; + ethernetETFtransportSettings.transmission_time = tx_timestamp; + } + + UA_free(threadArgumentsPublisher); + return (void *)NULL; +} + +/* Custom subscriber thread */ +void *sub_thread(void *arg) +{ + UA_Server *server; + UA_ReaderGroup *currentReaderGroup; + UA_ServerCallback subCallback; + struct timespec nextnanosleeptimeSub; + UA_UInt64 cycleTimeNs = 0; + UA_UInt32 offsetNs = 0; + UA_UInt32 ind = 0; + struct SubscriberData *sData; + + threadParams *threadArgumentsSubscriber = (threadParams *)arg; + server = threadArgumentsSubscriber->server; + subCallback = threadArgumentsSubscriber->callback; + currentReaderGroup = (UA_ReaderGroup *)threadArgumentsSubscriber->data; + cycleTimeNs = g_sData->cycleTimeNs; + sData = (struct SubscriberData *)g_sData->subData; + ind = threadArgumentsSubscriber->data_index; + offsetNs = sData[ind].offsetNs; + + /* Get current time and compute the next nanosleeptime to the nearest 5th second */ + clock_gettime(CLOCKID, &nextnanosleeptimeSub); + + nextnanosleeptimeSub.tv_sec = ((nextnanosleeptimeSub.tv_sec + 4) / 5) * 5; + /* Add 3 secs delay for subscriber to start */ + nextnanosleeptimeSub.tv_sec += 3; + nextnanosleeptimeSub.tv_nsec = offsetNs; + normalize(&nextnanosleeptimeSub); + + while (g_running) { + clock_nanosleep(CLOCKID, TIMER_ABSTIME, &nextnanosleeptimeSub, NULL); + subCallback(server, currentReaderGroup); + nextnanosleeptimeSub.tv_nsec += cycleTimeNs; + normalize(&nextnanosleeptimeSub); + } + + UA_free(threadArgumentsSubscriber); + return (void *)NULL; +} + +/* The following 3 functions are originally/normally declared in ua_pubsub.h + * But we want a customized cyclic interrupt as well as our own threads + * so we use custom threads to do it. The library will call these functions + * when it needs to register a callback, where our threads will be used instead + * of ua_timer's + */ + +UA_StatusCode +UA_PubSubManager_addRepeatedCallback(UA_Server *server, UA_ServerCallback callback, + void *data, UA_Double interval_ms, + UA_UInt64 *callbackId) +{ + + /* Initialize arguments required for the thread to run */ + threadParams *params = (threadParams *) UA_malloc(sizeof(threadParams)); + if (!params) + return UA_STATUSCODE_BADINTERNALERROR; + + /* Pass the value required for the threads */ + params->server = server; + params->data = data; + params->callback = callback; + params->interval_ms = interval_ms; + params->callbackId = callbackId; + + /* Check the writer group identifier and create the thread accordingly */ + UA_WriterGroup *tmpWriter = (UA_WriterGroup *) data; + + if (UA_NodeId_equal(&tmpWriter->identifier, &g_writerGroupIdent)) { + char threadNamePub[10] = "Publisher"; + params->data_index = g_indexPub; + g_thread[g_threadRun].id = threadCreation(PUB_THREAD_PRIORITY, g_sData->pubData[g_indexPub].cpuAffinity, pub_thread, + threadNamePub, params); + g_indexPub++; + } + else { + char threadNameSub[11] = "Subscriber"; + params->data_index = g_indexSub; + g_thread[g_threadRun].id = threadCreation(SUB_THREAD_PRIORITY, g_sData->subData[g_indexSub].cpuAffinity, sub_thread, + threadNameSub, params); + g_indexSub++; + } + + g_threadRun++; + + return UA_STATUSCODE_GOOD; +} + +UA_StatusCode +UA_PubSubManager_changeRepeatedCallbackInterval(UA_Server *server, + UA_UInt64 callbackId, + UA_Double interval_ms) +{ + (void) server; + (void) callbackId; + (void) interval_ms; + /* Callback interval need not be modified as it is thread based. + * The thread uses nanosleep for calculating cycle time and modification in + * nanosleep value changes cycle time + */ + return UA_STATUSCODE_GOOD; +} + +void +UA_PubSubManager_removeRepeatedPubSubCallback(UA_Server *server, + UA_UInt64 callbackId) +{ + (void) server; + (void) callbackId; + + /* TODO move pthread_join here? */ +} diff --git a/src/opcua-tsn/opcua_custom.h b/src/opcua-tsn/opcua_custom.h new file mode 100644 index 0000000..488b928 --- /dev/null +++ b/src/opcua-tsn/opcua_custom.h @@ -0,0 +1,171 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_CUSTOM_H_ +#define _OPCUA_CUSTOM_H_ + +#include +#include + +#include +#include +#include + +#include "opcua_utils.h" + +/* INTERNAL structs START *************************************************** + * The following internal structures from src/pubsub/ua_pubsub.h are required + * for checking the callback type. If the lib changes, then these structs + * should be updated as well. + * TODO: find out if there are better ways to check the Repeatedcallback type + */ + +#define TAILQ_ENTRY(type) \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ +} + +#define LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +typedef struct UA_ReaderGroup { + UA_ReaderGroupConfig config; + UA_NodeId identifier; + UA_NodeId linkedConnection; + LIST_ENTRY(UA_ReaderGroup) listEntry; + LIST_HEAD(UA_ListOfPubSubDataSetReader, UA_DataSetReader) readers; + /* for simplified information access */ + UA_UInt32 readersCount; + UA_UInt64 subscribeCallbackId; + UA_Boolean subscribeCallbackIsRegistered; +} UA_ReaderGroup; + +/* Offsets for buffered messages in the PubSub fast path. */ +typedef enum { + UA_PUBSUB_OFFSETTYPE_DATASETMESSAGE_SEQUENCENUMBER, + UA_PUBSUB_OFFSETTYPE_NETWORKMESSAGE_SEQUENCENUMBER, + UA_PUBSUB_OFFSETTYPE_TIMESTAMP_PICOSECONDS, + UA_PUBSUB_OFFSETTYPE_TIMESTAMP, /* source pointer */ + UA_PUBSUB_OFFSETTYPE_TIMESTAMP_NOW, /* no source */ + UA_PUBSUB_OFFSETTYPE_PAYLOAD_DATAVALUE, + UA_PUBSUB_OFFSETTYPE_PAYLOAD_VARIANT, + UA_PUBSUB_OFFSETTYPE_PAYLOAD_RAW + /* Add more offset types as needed */ +} UA_NetworkMessageOffsetType; + +typedef struct { + UA_NetworkMessageOffsetType contentType; + union { + union { + UA_DataValue *value; + size_t valueBinarySize; + } value; + UA_DateTime *timestamp; + } offsetData; + size_t offset; +} UA_NetworkMessageOffset; + +typedef struct UA_PubSubConnection{ + UA_PubSubConnectionConfig *config; + /* internal fields */ + UA_PubSubChannel *channel; + UA_NodeId identifier; + LIST_HEAD(UA_ListOfWriterGroup, UA_WriterGroup) writerGroups; + LIST_HEAD(UA_ListOfPubSubReaderGroup, UA_ReaderGroup) readerGroups; + size_t readerGroupsSize; + TAILQ_ENTRY(UA_PubSubConnection) listEntry; + UA_UInt16 configurationFreezeCounter; +} UA_PubSubConnection; + +typedef struct { + UA_ByteString buffer; /* The precomputed message buffer */ + UA_NetworkMessageOffset *offsets; /* Offsets for changes in the message buffer */ + size_t offsetsSize; +} UA_NetworkMessageOffsetBuffer; + + +typedef struct UA_WriterGroup{ + UA_WriterGroupConfig config; + /* internal fields */ + LIST_ENTRY(UA_WriterGroup) listEntry; + UA_NodeId identifier; + UA_PubSubConnection *linkedConnection; + LIST_HEAD(UA_ListOfDataSetWriter, UA_DataSetWriter) writers; + UA_UInt32 writersCount; + UA_UInt64 publishCallbackId; + UA_Boolean publishCallbackIsRegistered; + UA_PubSubState state; + UA_NetworkMessageOffsetBuffer bufferedMessage; + UA_UInt16 sequenceNumber; /* Increased after every succressuly sent message */ +} UA_WriterGroup; + +/* INTERNAL structs END *******************************************************/ + +typedef struct threadParams { + UA_Server *server; + void *data; + UA_ServerCallback callback; + UA_Duration interval_ms; + UA_UInt64 *callbackId; + UA_UInt32 data_index; + pthread_t id; +} threadParams; + +void *pub_thread(void *arg); +void *sub_thread(void *arg); + +UA_StatusCode +UA_PubSubManager_addRepeatedCallback(UA_Server *server, + UA_ServerCallback callback, + void *data, + UA_Double interval_ms, + UA_UInt64 *callbackId); + + +UA_StatusCode +UA_PubSubManager_changeRepeatedCallbackInterval(UA_Server *server, + UA_UInt64 callbackId, + UA_Double interval_ms); + +void +UA_PubSubManager_removeRepeatedPubSubCallback(UA_Server *server, + UA_UInt64 callbackId); +#endif diff --git a/src/opcua-tsn/opcua_datasource.c b/src/opcua-tsn/opcua_datasource.c new file mode 100644 index 0000000..1e98162 --- /dev/null +++ b/src/opcua-tsn/opcua_datasource.c @@ -0,0 +1,356 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opcua_common.h" +#include "json_helper.h" + +UA_UInt64 tx_sequence = -2; +extern struct ServerData *g_sData; +extern UA_Boolean g_running; +UA_UInt64 rx_sequence; +UA_UInt64 txTime; +UA_UInt64 rxTime; + +UA_StatusCode +pubReturnGetDataToTransmit(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *data) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) sourceTimeStamp; + (void) range; + + assert(nodeContext != NULL); + UA_UInt64 currentTime = 0; + static UA_UInt64 prev_rx_sequence; + struct timespec current_time_timespec; + UA_UInt64 packetCount = g_sData->packetCount; + + if(rx_sequence == prev_rx_sequence) + goto ret; + + if (tx_sequence == 0 && rx_sequence != 0) { + /* Init the tx_sequence to the first rx_sequence received by sub thread */ + tx_sequence = rx_sequence; + } + else { + tx_sequence++; + } + clock_gettime(CLOCK_TAI, ¤t_time_timespec); + currentTime = as_nanoseconds(¤t_time_timespec); + + UA_UInt64 d[5] = {rx_sequence, txTime, rxTime, tx_sequence, currentTime}; + + debug("[PUBB] rx_seqA:%ld, txtimePubA:%ld, rxTimeSubB:%ld, tx_seqB:%ld, txtimePubB:%ld\n", rx_sequence, txTime, rxTime, tx_sequence, currentTime); + + UA_StatusCode retval = UA_Variant_setArrayCopy(&data->value, &d[0], 5, + &UA_TYPES[UA_TYPES_UINT64]); + + if (retval != UA_STATUSCODE_GOOD) + debug("[PUB]Error in transmitting data source\n"); + + data->hasValue = true; + data->value.storageType = UA_VARIANT_DATA_NODELETE; + + prev_rx_sequence = rx_sequence; + + if (tx_sequence == packetCount) { + g_running = UA_FALSE; + /* debug("\n[PUB] SendCurrentTime: tx_sequence=packet_count=%ld reached. Exiting...\n", tx_sequence); */ + } + +ret: + return UA_STATUSCODE_GOOD; +} + +UA_StatusCode +pubGetDataToTransmit(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *data) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) sourceTimeStamp; + (void) range; + + assert(nodeContext != NULL); + + UA_UInt64 currentTime = 0; + struct timespec current_time_timespec; + UA_UInt64 packetCount = g_sData->packetCount; + + clock_gettime(CLOCK_TAI, ¤t_time_timespec); + tx_sequence++; + currentTime = as_nanoseconds(¤t_time_timespec); + UA_UInt64 d[2] = {tx_sequence, currentTime}; + + /* debug("[PUB] tx_sequence : %ld, time : %ld\n", tx_sequence, currentTime); */ + + UA_StatusCode retval = UA_Variant_setArrayCopy(&data->value, &d[0], 2, + &UA_TYPES[UA_TYPES_UINT64]); + + if (retval != UA_STATUSCODE_GOOD) + debug("[PUB]Error in transmitting data source\n"); + + data->hasValue = true; + data->value.storageType = UA_VARIANT_DATA_NODELETE; + + if (tx_sequence == packetCount) { + g_running = UA_FALSE; + /* debug("\n[PUB] SendCurrentTime: tx_sequence=packet_count=%ld reached. Exiting...\n", tx_sequence); */ + } + + return UA_STATUSCODE_GOOD; +} + +UA_StatusCode +subReturnStoreDataReceived(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *data) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) range; + assert(nodeContext != NULL); + + struct SubscriberData *sdata = (struct SubscriberData *)nodeContext; + FILE *fpSubscriber = sdata->fpSubscriberOutput; + UA_UInt64 packetCount = g_sData->packetCount; + UA_UInt64 seqA = 0; + UA_UInt64 seqB = 0; + + if (data == NULL) { + debug("[SUBR] Rx data is NULL\n"); + goto ret; + } + + /* Validate incoming data is byte array */ + UA_Variant v = data->value; + + if (v.arrayLength == (UA_UInt64) -1 && v.data == NULL) { + debug("[SUBR] Rx empty variant\n"); + goto ret; + } + + /* Check if valid data is there */ + if (v.arrayLength == (UA_UInt64) -1 || v.arrayLength > 0) { + + /* Data received ; {SeqA, tx-PubA, rx-SubB, SeqB, tx-PubB}; */ + UA_UInt64 *ptr_data = (UA_UInt64 *)data->value.data; + UA_UInt64 RXhwTS = 0; /* TODO: hard code as per now */ + UA_UInt64 txPubA = ptr_data[1]; + UA_UInt64 rxSubB = ptr_data[2]; + UA_UInt64 txPubB = ptr_data[4]; + UA_UInt64 rxSubA = 0; + UA_Int64 a2bLatency = 0; + UA_Int64 b2aLatency = 0; + UA_Int64 processingLatency = 0; + UA_Int64 returnLatency = 0; + struct timespec current_time_timespec; + seqA = ptr_data[0]; + seqB = ptr_data[3]; + + clock_gettime(CLOCK_TAI, ¤t_time_timespec); + rxSubA = as_nanoseconds(¤t_time_timespec); + a2bLatency = (UA_Int64)(rxSubB - txPubA); + b2aLatency = (UA_Int64)(rxSubA - txPubB); + processingLatency = (UA_Int64)(txPubB - rxSubB); + returnLatency = (UA_Int64)(rxSubA-txPubA); + + if (fpSubscriber != NULL) { + fprintf(fpSubscriber, "%ld\t%ld\t%d\t%ld\t%ld\t%ld\t%ld\t%ld\t%ld\t%d\t%ld\t%ld\t%ld\t%ld\n", + a2bLatency, seqA, sdata->id, txPubA, RXhwTS, rxSubB, processingLatency, + b2aLatency, seqB, sdata->id, txPubB, RXhwTS, rxSubA, returnLatency); + } + + /* UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "[seqA:%ld] txPubA(ns)=%ld rxSubB(ns)=%ld (ns) a2bLatency=%ld(ns) processingLatency=%ld(ns) \n", + (long)seqA, (long)txPubA, (long)rxSubB, a2bLatency, processingLatency); + UA_LOG_INFO(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "[seqB:%ld] txPubB(ns)=%ld rxSubA(ns)=%ld (ns) b2aLatency=%ld(ns) returnLatency=%ld(ns)\n", + (long)seqB, (long)txPubB, (long)rxSubA, returnLatency); + */ + } + + if (seqB == packetCount) { + if (fpSubscriber != NULL) { + fflush(fpSubscriber); + fclose(fpSubscriber); + } + g_running = UA_FALSE; + /* debug("\n[SUB][readCurrentTime]: rx_sequence=packet_count=%ld reached. Exit.\n ", rx_sequence); */ + } +ret: + return UA_STATUSCODE_GOOD; +} + +UA_StatusCode +subStoreDataReceived(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *data) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) range; + + assert(nodeContext != NULL); + + struct SubscriberData *sdata = (struct SubscriberData *)nodeContext; + FILE *fpSubscriber = sdata->fpSubscriberOutput; + UA_UInt64 packetCount = g_sData->packetCount; + + if (data == NULL) { + debug("[SUB] Rx Data is null\n"); + goto ret; + } + + /* Validate incoming data is byte array */ + UA_Variant v = data->value; + + if (v.arrayLength == (UA_UInt64) -1 && v.data == NULL) { + debug("[SUB] Rx empty variant\n"); + goto ret; + } + + /* Check if valid data is there */ + if (v.arrayLength == (UA_UInt64) -1 || v.arrayLength > 0) { + + UA_UInt64 *ptr_data = (UA_UInt64 *)data->value.data; + UA_UInt64 RXhwTS = 0; /* TODO: hard code as per now */ + + rx_sequence = ptr_data[0]; + txTime = ptr_data[1]; + struct timespec current_time_timespec; + + clock_gettime(CLOCK_TAI, ¤t_time_timespec); + rxTime = as_nanoseconds(¤t_time_timespec); + UA_Int64 latency = (UA_Int64)(rxTime - txTime); + + if (fpSubscriber != NULL) { + fprintf(fpSubscriber, "%ld\t%ld\t%d\t%ld\t%ld\t%ld\n", + latency, rx_sequence, sdata->id, txTime, RXhwTS, rxTime); + } + + debug("[SUB] RX: seq:%ld rx_t(ns):%ld local_t(ns):%ld diff_t(ns): %ld\n", + (long)rx_sequence, (long)txTime, (long)rxTime, latency); + } + + if (rx_sequence == packetCount) { + if (fpSubscriber != NULL) { + fflush(fpSubscriber); + fclose(fpSubscriber); + } + g_running = UA_FALSE; + debug("[SUB] RX: rx_sequence = packet_count = %ld\n", rx_sequence); + } +ret: + return UA_STATUSCODE_GOOD; +} + +UA_StatusCode +dummyDSRead(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *value) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) sourceTimeStamp; + (void) range; + (void) value; + + debug("WARN: dummyDSRead() shouldn't be called"); + return UA_STATUSCODE_GOOD; +} + + +UA_StatusCode +dummyDSWrite(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *value) +{ + (void) server; + (void) sessionId; + (void) sessionContext; + (void) nodeId; + (void) nodeContext; + (void) range; + (void) value; + + /* debug("WARN: dummyDSWrite() shouldn't be called"); */ + return UA_STATUSCODE_GOOD; +} diff --git a/src/opcua-tsn/opcua_datasource.h b/src/opcua-tsn/opcua_datasource.h new file mode 100644 index 0000000..375f6b6 --- /dev/null +++ b/src/opcua-tsn/opcua_datasource.h @@ -0,0 +1,79 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_DATASOURCE_H_ +#define _OPCUA_DATASOURCE_H_ + +#include +#include +#include +#include +#include + +#include "opcua_utils.h" + +UA_StatusCode +pubReturnGetDataToTransmit(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *data); + +UA_StatusCode +pubGetDataToTransmit(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *value); + +UA_StatusCode +subReturnStoreDataReceived(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *data); + +UA_StatusCode +subStoreDataReceived(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *data); + +UA_StatusCode +dummyDSWrite(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, const UA_NumericRange *range, + const UA_DataValue *value); + +UA_StatusCode +dummyDSRead(UA_Server *server, const UA_NodeId *sessionId, + void *sessionContext, const UA_NodeId *nodeId, + void *nodeContext, UA_Boolean sourceTimeStamp, + const UA_NumericRange *range, UA_DataValue *value); +#endif diff --git a/src/opcua-tsn/opcua_publish.c b/src/opcua-tsn/opcua_publish.c new file mode 100644 index 0000000..4c1468e --- /dev/null +++ b/src/opcua-tsn/opcua_publish.c @@ -0,0 +1,256 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "opcua_common.h" +#include "opcua_publish.h" + +#define PUBSUB_CONFIG_FASTPATH_FIXED_OFFSETS //TODO define this elsewhere +#define KEYFRAME_COUNT 10 +#define PUB_MODE_ZERO_COPY + +extern UA_NodeId g_writerGroupIdent; + +void addPubSubConnection(UA_Server *server, UA_NodeId *connId, + struct ServerData *sdata, struct PublisherData *pdata) +{ + + /* Details about the connection configuration and handling are located + * in the pubsub connection tutorial + */ + UA_PubSubConnectionConfig connectionConfig; + + memset(&connectionConfig, 0, sizeof(connectionConfig)); + connectionConfig.name = UA_STRING("UADP Connection"); + connectionConfig.transportProfileUri = sdata->transportProfile; + connectionConfig.enabled = UA_TRUE; + + int socketPrio = pdata->socketPriority; + connectionConfig.etfConfiguration.socketPriority = socketPrio > 0 ? + socketPrio : -1; + connectionConfig.etfConfiguration.sotxtimeEnabled = UA_TRUE; + + if (sdata->useXDP) { +#ifdef PUB_MODE_ZERO_COPY + connectionConfig.xdp_queue = pdata->xdpQueue; + connectionConfig.xdp_flags |= XDP_FLAGS_DRV_MODE; + connectionConfig.xdp_bind_flags |= XDP_ZEROCOPY; +#else + connectionConfig.xdp_queue = pdata->xdpQueue; + connectionConfig.xdp_flags |= XDP_FLAGS_SKB_MODE; + connectionConfig.xdp_bind_flags |= XDP_COPY; +#endif + } + + UA_NetworkAddressUrlDataType networkAddressUrl = { + UA_STRING(sdata->pubInterface), UA_STRING(pdata->url)}; + + UA_Variant_setScalar(&connectionConfig.address, &networkAddressUrl, + &UA_TYPES[UA_TYPES_NETWORKADDRESSURLDATATYPE]); + + connectionConfig.publisherId.numeric = (UA_UInt16)pdata->id; + UA_Server_addPubSubConnection(server, &connectionConfig, connId); +} + +/** + * **PublishedDataSet handling** + * + * The PublishedDataSet (PDS) and PubSubConnection are the toplevel entities and + * can exist alone. The PDS contains the collection of the published fields. All + * other PubSub elements are directly or indirectly linked with the PDS or + * connection. + */ +static void addPublishedDataSet(UA_Server *server, UA_NodeId *publishedDataSetIdent) +{ + /* The PublishedDataSetConfig contains all necessary public + * informations for the creation of a new PublishedDataSet + */ + UA_PublishedDataSetConfig publishedDataSetConfig; + + memset(&publishedDataSetConfig, 0, sizeof(UA_PublishedDataSetConfig)); + publishedDataSetConfig.publishedDataSetType = UA_PUBSUB_DATASET_PUBLISHEDITEMS; + publishedDataSetConfig.name = UA_STRING("Demo PDS"); + /* Create new PublishedDataSet based on the PublishedDataSetConfig. + */ + UA_Server_addPublishedDataSet(server, &publishedDataSetConfig, publishedDataSetIdent); +} + +/** + * **DataSetField handling** + * + * The DataSetField (DSF) is part of the PDS and describes exactly one published + * field. + */ +static void addDataSetField(UA_Server *server, UA_NodeId *publishedDataSetIdent, + struct ServerData *sdata, struct PublisherData *pdata) +{ + (void) sdata; + + /* Add a variable datasource node. */ + /* Set the variable attributes. */ + UA_VariableAttributes attr = UA_VariableAttributes_default; + + attr.displayName = UA_LOCALIZEDTEXT("en_US", "Payload datasource"); + attr.accessLevel = UA_ACCESSLEVELMASK_READ | UA_ACCESSLEVELMASK_WRITE; + + /* Define where the variable shall be added with which browse name */ + UA_NodeId new_node_id = UA_NODEID_STRING(1, "payload-datasource"); + UA_NodeId parent_node_id = UA_NODEID_NUMERIC(0, UA_NS0ID_OBJECTSFOLDER); + UA_NodeId parent_ref_node_id = UA_NODEID_NUMERIC(0, UA_NS0ID_ORGANIZES); + UA_NodeId variable_type = UA_NODEID_NUMERIC(0, UA_NS0ID_BASEDATAVARIABLETYPE); + UA_QualifiedName browse_name = UA_QUALIFIEDNAME(1, "Shared mem DS"); + + UA_DataSource data_source; + + data_source.read = pdata->readFunc; + data_source.write = pdata->writeFunc; + + /* Add the variable with data source. */ + UA_Server_addDataSourceVariableNode(server, new_node_id, + parent_node_id, + parent_ref_node_id, + browse_name, + variable_type, attr, + data_source, + pdata, NULL); + + /* Add a field to the previous created PublishedDataSet */ + UA_NodeId dataSetFieldIdent; + UA_DataSetFieldConfig dataSetFieldConfig; + memset(&dataSetFieldConfig, 0, sizeof(UA_DataSetFieldConfig)); + dataSetFieldConfig.dataSetFieldType = UA_PUBSUB_DATASETFIELD_VARIABLE; + dataSetFieldConfig.field.variable.fieldNameAlias = UA_STRING("payload"); + dataSetFieldConfig.field.variable.promotedField = UA_FALSE; + dataSetFieldConfig.field.variable.publishParameters.publishedVariable = + UA_NODEID_STRING(1, "payload-datasource"); + dataSetFieldConfig.field.variable.publishParameters.attributeId = UA_ATTRIBUTEID_VALUE; + UA_Server_addDataSetField(server, *publishedDataSetIdent, + &dataSetFieldConfig, &dataSetFieldIdent); +} + +/** + * **WriterGroup handling** + * + * The WriterGroup (WG) is part of the connection and contains the primary + * configuration parameters for the message creation. + */ +static void addWriterGroup(UA_Server *server, UA_NodeId *connectionIdent, + UA_NodeId *writerGroupIdent, struct ServerData *sdata, + struct PublisherData *pdata) +{ + double intervalMs = (double)sdata->cycleTimeNs / 1e6f; + + /* Now we create a new WriterGroupConfig and add the group to the existing + * PubSubConnection. + */ + UA_WriterGroupConfig writerGroupConfig; + + memset(&writerGroupConfig, 0, sizeof(UA_WriterGroupConfig)); + writerGroupConfig.name = UA_STRING("Demo WriterGroup"); + // This is in miliseconds, not nanoseconds + writerGroupConfig.publishingInterval = intervalMs; + writerGroupConfig.enabled = UA_FALSE; + writerGroupConfig.writerGroupId = pdata->writerGroupId; + writerGroupConfig.encodingMimeType = UA_PUBSUB_ENCODING_UADP; +#if defined PUBSUB_CONFIG_FASTPATH_FIXED_OFFSETS + //TODO: enable rtlevel to increase performance? + // writerGroupConfig.rtLevel = UA_PUBSUB_RT_FIXED_SIZE; +#endif + writerGroupConfig.messageSettings.encoding = UA_EXTENSIONOBJECT_DECODED; + writerGroupConfig.messageSettings.content.decoded.type = &UA_TYPES[UA_TYPES_UADPWRITERGROUPMESSAGEDATATYPE]; + + /* The configuration flags for the messages are encapsulated inside the + * message- and transport settings extension objects. These extension + * objects are defined by the standard. e.g. + * UadpWriterGroupMessageDataType + */ + UA_UadpWriterGroupMessageDataType *writerGroupMessage = UA_UadpWriterGroupMessageDataType_new(); + /* Change message settings of writerGroup to send PublisherId, + * WriterGroupId in GroupHeader and DataSetWriterId in PayloadHeader + * of NetworkMessage + */ + writerGroupMessage->networkMessageContentMask = (UA_UadpNetworkMessageContentMask)(UA_UADPNETWORKMESSAGECONTENTMASK_PUBLISHERID | + (UA_UadpNetworkMessageContentMask)UA_UADPNETWORKMESSAGECONTENTMASK_GROUPHEADER | + (UA_UadpNetworkMessageContentMask)UA_UADPNETWORKMESSAGECONTENTMASK_WRITERGROUPID | + (UA_UadpNetworkMessageContentMask)UA_UADPNETWORKMESSAGECONTENTMASK_PAYLOADHEADER); + writerGroupConfig.messageSettings.content.decoded.data = writerGroupMessage; + + UA_Server_addWriterGroup(server, *connectionIdent, &writerGroupConfig, writerGroupIdent); + UA_Server_setWriterGroupOperational(server, *writerGroupIdent); + UA_UadpWriterGroupMessageDataType_delete(writerGroupMessage); +} + +/** + * **DataSetWriter handling** + * + * A DataSetWriter (DSW) is the glue between the WG and the PDS. The DSW is + * linked to exactly one PDS and contains additional informations for the + * message generation. + */ +static void addDataSetWriter(UA_Server *server, UA_NodeId *writerGroupIdent, + UA_NodeId *publishedDataSetIdent, struct PublisherData *pdata) +{ + /* We need now a DataSetWriter within the WriterGroup. This means we must + * create a new DataSetWriterConfig and add call the addWriterGroup function. + */ + UA_NodeId dataSetWriterIdent; + UA_DataSetWriterConfig dataSetWriterConfig; + + memset(&dataSetWriterConfig, 0, sizeof(UA_DataSetWriterConfig)); + dataSetWriterConfig.name = UA_STRING("Demo DataSetWriter"); + dataSetWriterConfig.dataSetWriterId = (UA_UInt16)pdata->dataSetWriterId; + dataSetWriterConfig.keyFrameCount = KEYFRAME_COUNT; + UA_Server_addDataSetWriter(server, *writerGroupIdent, *publishedDataSetIdent, + &dataSetWriterConfig, &dataSetWriterIdent); +} + +int createPublisher(UA_Server *serv, struct ServerData *sdata, + struct PublisherData *pdata, UA_NodeId *connectionIdent) +{ + UA_NodeId publishedDataSetIdent; + addPublishedDataSet(serv, &publishedDataSetIdent); + addDataSetField(serv, &publishedDataSetIdent, sdata, pdata); + addWriterGroup(serv, connectionIdent, &g_writerGroupIdent, sdata, pdata); + addDataSetWriter(serv, &g_writerGroupIdent, &publishedDataSetIdent, pdata); + UA_Server_freezeWriterGroupConfiguration(serv, g_writerGroupIdent); + return 0; +} diff --git a/src/opcua-tsn/opcua_publish.h b/src/opcua-tsn/opcua_publish.h new file mode 100644 index 0000000..cbf8614 --- /dev/null +++ b/src/opcua-tsn/opcua_publish.h @@ -0,0 +1,45 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_PUBLISH_H_ +#define _OPCUA_PUBLISH_H_ + +#include "opcua_common.h" +#include +#include + +int createPublisher(UA_Server *serv, struct ServerData *sdata, + struct PublisherData *pub, UA_NodeId *connectionIdent); +void addPubSubConnection(UA_Server *server, UA_NodeId *connId, + struct ServerData *sdata, struct PublisherData *pdata); + +#endif diff --git a/src/opcua-tsn/opcua_subscribe.c b/src/opcua-tsn/opcua_subscribe.c new file mode 100644 index 0000000..96490a5 --- /dev/null +++ b/src/opcua-tsn/opcua_subscribe.c @@ -0,0 +1,206 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "opcua_common.h" + +#define SUB_MODE_ZERO_COPY + +/* Add new connection to the server */ +void addSubConnection(UA_Server *server, UA_NodeId *connId, + struct ServerData *sdata, struct SubscriberData *sub) +{ + /* Configuration creation for the connection */ + UA_PubSubConnectionConfig connectionConfig; + + memset(&connectionConfig, 0, sizeof(UA_PubSubConnectionConfig)); + connectionConfig.name = UA_STRING("UDPMC Connection 1"); + connectionConfig.transportProfileUri = sdata->transportProfile; + connectionConfig.enabled = UA_TRUE; + + UA_NetworkAddressUrlDataType networkAddressUrl = { + UA_STRING(sdata->subInterface), + UA_STRING(sub->url) + }; + + if (sdata->useXDP) { +#ifdef SUB_MODE_ZERO_COPY + connectionConfig.xdp_queue = sub->xdpQueue; + connectionConfig.xdp_flags |= XDP_FLAGS_DRV_MODE; + connectionConfig.xdp_bind_flags |= XDP_ZEROCOPY; +#else + connectionConfig.xdp_queue = sub->xdpQueue; + connectionConfig.xdp_flags |= XDP_FLAGS_SKB_MODE; + connectionConfig.xdp_bind_flags |= XDP_COPY; +#endif + } + + UA_Variant_setScalar(&connectionConfig.address, &networkAddressUrl, + &UA_TYPES[UA_TYPES_NETWORKADDRESSURLDATATYPE]); + if (sub->id == 0) + connectionConfig.publisherId.numeric = UA_UInt32_random(); + else + connectionConfig.publisherId.numeric = sub->id; + UA_Server_addPubSubConnection(server, &connectionConfig, connId); +} + +/* Add ReaderGroup to the created connection */ +static void addReaderGroup(UA_Server *server, UA_NodeId *connId, + UA_NodeId *readerGroupId) +{ + UA_ReaderGroupConfig readerGroupConfig; + + memset(&readerGroupConfig, 0, sizeof(UA_ReaderGroupConfig)); + readerGroupConfig.name = UA_STRING("ReaderGroup1"); + + UA_Server_addReaderGroup(server, *connId, &readerGroupConfig, + readerGroupId); +} + +/* Add DataSetReader to the ReaderGroup */ +static void addDataSetReader(UA_Server *server, UA_NodeId *readerGroupIdentifier, + UA_NodeId *datasetReaderIdentifier, + struct SubscriberData *sub) +{ + UA_DataSetReaderConfig readerConfig; + memset(&readerConfig, 0, sizeof(UA_DataSetReaderConfig)); + readerConfig.name = UA_STRING("DataSet Reader 1"); + readerConfig.dataSetWriterId = (UA_UInt16)sub->subscribedDSWriterId; + UA_UInt16 publisherIdentifier = (UA_UInt16)sub->subscribedPubId; + readerConfig.publisherId.type = &UA_TYPES[UA_TYPES_UINT16]; + readerConfig.publisherId.data = &publisherIdentifier; + readerConfig.writerGroupId = (UA_UInt16)sub->subscribedWGId; + + /* Setting up Meta data configuration in DataSetReader */ + UA_DataSetMetaDataType *pMetaData = &readerConfig.dataSetMetaData; + UA_DataSetMetaDataType_init(pMetaData); + + /* Static definition of number of fields size = 1 */ + pMetaData->name = UA_STRING("DataSet Test"); + pMetaData->fieldsSize = 1; + pMetaData->fields = (UA_FieldMetaData*)UA_Array_new(pMetaData->fieldsSize, + &UA_TYPES[UA_TYPES_FIELDMETADATA]); + sub->temp_dataSetMetaData = (char *) pMetaData->fields; + + /* Sequence and timestamp is UINT64 DataType */ + UA_FieldMetaData_init(&pMetaData->fields[0]); + UA_NodeId_copy(&UA_TYPES[UA_TYPES_UINT64].typeId, + &pMetaData->fields[0].dataType); + pMetaData->fields[0].builtInType = UA_TYPES_UINT64; + pMetaData->fields[0].name = UA_STRING ("Sequence timestamp"); + pMetaData->fields[0].valueRank = -3; /* scalar or 1-D array */ + + UA_Server_addDataSetReader(server, *readerGroupIdentifier, + &readerConfig, datasetReaderIdentifier); +} + +/* Set SubscribedDataSet type to TargetVariables data type + * Add subscribedvariables to the DataSetReader + */ +static void addSubscribedVariables(UA_Server *server, struct SubscriberData *sub, + UA_NodeId *datasetReaderIdentifier) +{ + /* Add another variable, as a datasource node. */ + /* 1) Set the variable attributes. */ + UA_VariableAttributes attr = UA_VariableAttributes_default; + + attr.displayName = UA_LOCALIZEDTEXT("en_US", "Time stamp datasource"); + attr.accessLevel = UA_ACCESSLEVELMASK_READ | UA_ACCESSLEVELMASK_WRITE; + + /* 2) Define where the variable shall be added with which browse name */ + UA_NodeId datasource_node_id = UA_NODEID_STRING(1, "time-stamp-datasource"); + UA_NodeId parent_node_id = UA_NODEID_NUMERIC(0, UA_NS0ID_OBJECTSFOLDER); + UA_NodeId parent_ref_node_id = UA_NODEID_NUMERIC(0, UA_NS0ID_ORGANIZES); + UA_NodeId variable_type = UA_NODEID_NUMERIC(0, UA_NS0ID_BASEDATAVARIABLETYPE); + UA_QualifiedName browse_name = UA_QUALIFIEDNAME(1, "Time stamp datasource"); + + UA_DataSource data_source; + + data_source.read = sub->readFunc; + data_source.write = sub->writeFunc; + + /* 3) Add the variable with data source. */ + UA_Server_addDataSourceVariableNode(server, datasource_node_id, + parent_node_id, + parent_ref_node_id, + browse_name, + variable_type, attr, + data_source, + sub, NULL); + + UA_TargetVariablesDataType targetVars; + + targetVars.targetVariablesSize = 1; + targetVars.targetVariables = (UA_FieldTargetDataType *) + UA_calloc(targetVars.targetVariablesSize, + sizeof(UA_FieldTargetDataType)); + sub->temp_targetVars = (char *) targetVars.targetVariables; + + if (&targetVars.targetVariables[0] == NULL) { + UA_LOG_WARNING(UA_Log_Stdout, UA_LOGCATEGORY_USERLAND, + "failed to calloc targetVariables"); + return; + } + + UA_FieldTargetDataType_init(&targetVars.targetVariables[0]); + targetVars.targetVariables[0].attributeId = UA_ATTRIBUTEID_VALUE; + targetVars.targetVariables[0].targetNodeId = datasource_node_id; + UA_Server_DataSetReader_createTargetVariables(server, *datasetReaderIdentifier, &targetVars); + + // TODO: Not sure why this will segfault. + // Workaround using temp_targetVars & temp_dataSetMetaData to properly free + // UA_TargetVariablesDataType_deleteMembers(&targetVars); + // UA_free(ua->readerConfig.dataSetMetaData.fields); +} + +int createSubscriber(UA_Server *server, struct ServerData *sdata, + struct SubscriberData *sub, UA_NodeId *connId) +{ + (void) sdata; + + UA_NodeId readerGroupIdentifier; + UA_NodeId dataSetReaderIdentifier; + addReaderGroup(server, connId, &readerGroupIdentifier); + addDataSetReader(server, &readerGroupIdentifier, &dataSetReaderIdentifier, sub); + addSubscribedVariables(server, sub, &dataSetReaderIdentifier); + return 0; +} + diff --git a/src/opcua-tsn/opcua_subscribe.h b/src/opcua-tsn/opcua_subscribe.h new file mode 100644 index 0000000..35ada55 --- /dev/null +++ b/src/opcua-tsn/opcua_subscribe.h @@ -0,0 +1,41 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_SUBSCRIBE_H_ +#define _OPCUA_SUBSCRIBE_H_ + +int createSubscriber(UA_Server *server, struct ServerData *sdata, + struct SubscriberData *pdata, UA_NodeId *connId); +void +addSubConnection(UA_Server *server, UA_NodeId *connId, struct ServerData *sdata, + struct SubscriberData *sub); +#endif diff --git a/src/opcua-tsn/opcua_utils.h b/src/opcua-tsn/opcua_utils.h new file mode 100644 index 0000000..b3d267c --- /dev/null +++ b/src/opcua-tsn/opcua_utils.h @@ -0,0 +1,60 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef _OPCUA_UTILS_H +#define _OPCUA_UTILS_H + +#include +#include +#include + +#define log_error(M, ...) \ + fprintf(stderr, "[ERR] %s():%d " M "\n", \ + __func__, __LINE__, ##__VA_ARGS__) + +#define log(M, ...) fprintf(stdout, "[LOG] " M "\n", ##__VA_ARGS__) + +#define catch_err(A, M, ...) \ + if (A) { \ + log_error(M, ##__VA_ARGS__); \ + errno=0; \ + goto error; \ + } + +extern int verbose; + +#define debug(M, ...) \ + if(verbose) {\ + fprintf(stderr, "[DBG] %s():%d " M "\n", \ + __func__, __LINE__, ##__VA_ARGS__); \ + } +#endif diff --git a/src/tsq.c b/src/tsq.c new file mode 100644 index 0000000..0d72a92 --- /dev/null +++ b/src/tsq.c @@ -0,0 +1,671 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUFFER_SIZE 256 +#define CLIENT_COUNT 2 +#define DEFAULT_LISTENER_OUTFILE "tsq-listener-data.txt" +#define NULL_OUTFILE "NULL" + +#define MODE_LISTENER 1 +#define MODE_TALKER 2 + +int halt_sig; +FILE *glob_fp; +int glob_sockfd; +int glob_ptpfd; +pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + +void error(char *msg, ...) +{ + va_list argptr; + + if(glob_sockfd) + close(glob_sockfd); + + if(glob_fp) + fclose(glob_fp); + + if(glob_ptpfd) + close(glob_ptpfd); + + va_start(argptr, msg); + vfprintf(stderr, msg, argptr); + va_end(argptr); + exit(1); +} + +/* Read shared signal variable */ +int get_signal(void) +{ + int temp = 0, ret = 0; + ret = pthread_mutex_lock(&lock); + if (ret) + fprintf(stderr, "[TSQ] Failed to lock halt_sig.\n"); + + temp = halt_sig; + + ret = pthread_mutex_unlock(&lock); + if (ret) + fprintf(stderr, "[TSQ] Failed to unlock halt_sig.\n"); + return temp; +} + +/* Signal handler */ +void sigint_handler(int signum) +{ + int ret = 0; + + fprintf(stdout, "[TSQ] Thread exiting.\n"); + + ret = pthread_mutex_lock(&lock); + if (ret) + fprintf(stderr, "[TSQ] Failed to lock halt_sig.\n"); + + halt_sig = signum; + + ret = pthread_mutex_unlock(&lock); + if (ret) + fprintf(stderr, "[TSQ] Failed to unlock halt_sig.\n"); + +} + +typedef struct payload { + int uid; + int seq; + long long secs; + long nsecs; +} payload; + +/* User input options */ +struct opt { + char *args[1]; + int verbose; + int mode; + char *server_ip; + int port; + /* Listener */ + char *output_file; + /* Talker */ + int uid; + char *device; + int timeout; +}; + +static struct argp_option options[] = { + /* Shared */ + + {"talker", 'T', 0, 0, "Talker mode (read AUXTS)"}, + {"listener",'L', 0, 0, "Listener mode (listen for 2 talker and compare)"}, + + {"verbose", 'v', 0, 0, "Produce verbose output"}, + {"ip", 'i', "ADDR", 0, "Server IP address (eg. 192.1.2.3)"}, + {"port", 'p', "PORT", 0, "Port number\n" + " Def: 5678 | Min: 999 | Max: 9999"}, + + /* Listener-specific */ + {"output", 'o', "FILE", 0, "Save output to FILE (eg. temp.txt)"}, + + /* Talker-specific */ + {"device", 'd', "FILE", 0, "PTP device to read (eg. /dev/ptp1)"}, + {"uid", 'u', "COUNT", 0, "Unique Talker ID" + " Def: 1234 | Min: 999 | Max: 9999"}, + {"timeout", 't', "MSEC", 0, "Polling timeout in ms" + " Def: 1100ms | Min: 1ms | Max: 2000ms"}, + + { 0 } +}; + +static error_t parser(int key, char *arg, struct argp_state *state) +{ + /* Get the input argument from argp_parse, which we */ + /* know is a pointer to our user_opt structure. */ + struct opt *user_opt = state->input; + + switch (key) { + case 'v': + user_opt->verbose = 1; + break; + case 'T': + user_opt->mode = MODE_TALKER; + break; + case 'L': + user_opt->mode = MODE_LISTENER; + break; + case 'i': + user_opt->server_ip = arg; + break; + case 'p': + user_opt->port = atoi(arg); + if (user_opt->port < 999 || user_opt->port > 9999) + error("Invalid port number. Check --help"); + break; + case 'o': + user_opt->output_file = arg; + break; + case 'u': + user_opt->uid = atoi(arg); + if (user_opt->uid < 999 || user_opt->uid > 9999) + error("Invalid UID. Check --help"); + break; + case 'd': + user_opt->device = arg; + break; + case 't': + user_opt->timeout = atoi(arg); + if (user_opt->timeout < 1 || user_opt->timeout > 2000) + error("Invalid timeout. Check --help"); + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static char usage[] = "-i -p -d /dev/ptp -u {-T|-L}"; + +static char summary[] = "Time Sync Quality Measurement application"; + +static struct argp argp = { options, parser, usage, summary }; + +/** + * @brief Validate payload structure received + * + * @param pl pointer to payload received + * @param cli_ids + * @return validation boolean result + * + */ +bool validate_payload (payload *pl, int *cli_ids, int client_num) +{ + bool ret = false; + + if ((pl->uid != 0) && (pl->secs != 0) && (pl->nsecs != 0)) { + for (int i = 0 ; i < client_num ; i++) { + if (cli_ids[i] == pl->uid) { + ret = true; + break; + } + } + } + return ret; +} + +/** + * @brief Check if all the data is received/ready for each client + * + * @param pl pointer to data ready flags table + * @param size size of data ready flags table + * @return true if data is now all available + */ +bool data_ready(bool *flags, int size) +{ + for (int i = 0 ; i < size ; i++) { + if (*flags == false) + return false; + flags++; + } + return true; +} + +/** + * @brief Reset data ready flags table + * + * @param pl pointer to data ready flags table + * @param size size of data ready flags table + */ +void reset_data_ready(bool *flags, int size) +{ + for (int i = 0; i < size; i++) { + *flags = false; + flags++; + } +} + +/* Listener - wait for 2 talkers to connect and receive. Compare both talker + * timestamps to get the delta and transmit out. + * TODO: Expand it to support 4 talkers at the same time? + */ +void listener(struct opt *user_opt) { + + char *server_ip; + struct sockaddr_in serv; + socklen_t len; + int rounds = 0; + int connfd = 0; + int n = 0; + int i = 0; + int verbose; + int max_sd = 0; + int activity = 0; + char recv_buff[BUFFER_SIZE]; + long long secs_error; + long nsecs_error; + int cli_ids[CLIENT_COUNT]; + int cli_conns[CLIENT_COUNT] = {}; + payload cli_data[CLIENT_COUNT]; + payload temp_data, keep_data; + fd_set readfds; + bool pl_valid_flag[CLIENT_COUNT] = {false, false}; + bool skip_slot = false; + int keep_data_slot = 0; + + memset(&keep_data, 0, sizeof(payload)); + + if(user_opt == NULL) + error("[TSQ-L] User option is NULL"); + + verbose = user_opt->verbose; + + server_ip = user_opt->server_ip; + + glob_fp = fopen(user_opt->output_file, "w"); + if (glob_fp == NULL) + error("[TSQ-L] Error opening file: %s\n", user_opt->output_file); + + printf("[TSQ-L] Saving output in %s\n", user_opt->output_file); + + // Create listener socket + glob_sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (glob_sockfd < 0) + error("[TSQ-L] Error opening listening socket\n"); + + // connection settings + memset(&serv, '0', sizeof(serv)); + serv.sin_family = AF_INET; + serv.sin_addr.s_addr = inet_addr(server_ip); + if (user_opt->port != 0) + serv.sin_port = htons(user_opt->port); + else + serv.sin_port = 0; + + // bind the socket and start listening + if (bind(glob_sockfd, (struct sockaddr *)&serv, sizeof(serv)) < 0) + error("[TSQ-L] Error binding listening socket\n"); + + listen(glob_sockfd, 2); + + len = sizeof(serv); + if (getsockname(glob_sockfd, (struct sockaddr *)&serv, &len) == -1) + error("[TSQ-L] getsockname() failed"); + else + if (verbose) + printf("[TSQ-L] Started listening on %s:%d\n", + server_ip, ntohs(serv.sin_port)); + + // Waiting for CLIENT_COUNT connections + i = 0; + while (i < CLIENT_COUNT && get_signal() == 0) { + + connfd = accept(glob_sockfd, (struct sockaddr *)NULL, NULL); + + if (connfd < 0) { + printf("[TSQ-L] Error accepting connection. Waiting for the next one\n"); + continue; + } + + if (verbose) + printf("[TSQ-L] Accept a connection. glob_sockfd[%d]:%d\n", i, connfd); + + bzero(recv_buff, BUFFER_SIZE); + n = read(connfd, recv_buff, sizeof(recv_buff) - 1); + + if (n < 0) { + //close the connection in case failed read + close(connfd); + error("[TSQ-L] ERROR reading UID from socket\n"); + } + + cli_ids[i] = atoi(recv_buff); + if (verbose) + printf("[TSQ-L] Connected with client_id:%d\n", cli_ids[i]); + cli_conns[i] = connfd; + i++; + } + + // clients connected. Send command to start broadcast + for (int i = 0; i < CLIENT_COUNT; i++) { + + connfd = cli_conns[i]; + bzero(recv_buff, BUFFER_SIZE); + snprintf(recv_buff, sizeof(recv_buff), "start\n"); + n = write(connfd, recv_buff, sizeof(recv_buff)); + if (n < 0) + error("[TSQ-L] Error writing to client socket.\n"); + } + memset(&cli_data, 0, sizeof(cli_data)); + + while (get_signal() == 0) { + + // write to file everytime + fflush(glob_fp); + + FD_ZERO(&readfds); + max_sd = 0; + + for (int i = 0; i < CLIENT_COUNT; i++) { + FD_SET(cli_conns[i], &readfds); + if (cli_conns[i] > max_sd) + max_sd = cli_conns[i]; + } + + /* + * wait for an activity on one of the sockets , timeout is NULL, + * so wait indefinitely + */ + activity = select(max_sd + 1, &readfds, NULL, NULL, NULL); + + if ((activity < 0) && (errno != EINTR)) + error("[TSQ-L] Select error\n"); + + for (int i = 0; i < CLIENT_COUNT; i++) { + + bzero(recv_buff, BUFFER_SIZE); + memset(&temp_data, '0', sizeof(temp_data)); + connfd = cli_conns[i]; + + if (FD_ISSET(connfd, &readfds)) { + if (get_signal() != 0) + error("[TSQ-L] client socket closed. TSQ will end now.\n"); + + n = read(connfd, recv_buff, sizeof(recv_buff) - 1); + + if (n < 0) + error("[TSQ-L] ERROR reading socket. Exiting.\n"); + /* + * If the slot is marked for keeping for sample alignment + * we will recopy the previous value and not use the one read from the socket. + */ + if ((skip_slot == true) && (keep_data_slot == i)) { + memcpy(&cli_data[i], &keep_data, sizeof(payload)); + pl_valid_flag[i] = true; + + /* + * if (verbose) + * printf("[TSQ-L] KEEP SLOT data from %d : Secs : %lld, Nsecs : %ld , seq : %d\n", + * cli_data[i].uid, cli_data[i].secs, cli_data[i].nsecs , cli_data[i].seq); + */ + + } else { + if (n > 0) { // copy the new value + memcpy(&temp_data, recv_buff, sizeof(payload)); + if (validate_payload (&temp_data, cli_ids, CLIENT_COUNT)) { + pl_valid_flag[i] = true; + memcpy(&cli_data[i], &temp_data, sizeof(payload)); + /*if (verbose) + printf("[TSQ-L] Msg from %d : Secs : %lld, Nsecs : %ld , seq : %d\n", + cli_data[i].uid, cli_data[i].secs, cli_data[i].nsecs, cli_data[i].seq); */ + } + } + } + } + } + + if (data_ready(pl_valid_flag, CLIENT_COUNT)) { + secs_error = cli_data[0].secs - cli_data[1].secs; + nsecs_error = cli_data[0].nsecs - cli_data[1].nsecs; + + if (verbose) + printf("[TSQ-L] Msg from %d, %d: Secs off: %lld, Nsecs off: %ld\n", + cli_data[0].uid, cli_data[1].uid, secs_error, nsecs_error); + /* + * 1. When secs_error is equals to 1, t0 sample is 'newer' compared to t1. + * This means the t0 sample is not align to t1 sample, we shd keep t0, discard current t1, and reread t1. + * 2. When secs_error is equals to -1, t1 sample is now 'newer' compare to t0. + * The t1 sample is not align to t0 sample, we shd keep t1, discard current t0, and reread t0. + * 3. When secs_error is 0 , the samples are aligned. The nsecs is then valid. + * 4. When secs_error are not ( 0 | 1 |-1) , the samples are misaligned , and we wait for alignment in future samples. + * In this case, we discard nsecs_errors. + */ + + switch (secs_error) { + + case 1: + skip_slot = true; + keep_data_slot = 0; + memcpy(&keep_data, &cli_data[0], sizeof(payload)); + break; + + case -1: + skip_slot = true; + keep_data_slot = 1; + memcpy(&keep_data, &cli_data[1], sizeof(payload)); + break; + + case 0: + skip_slot = false; + memset(&keep_data, '0', sizeof(payload)); + fprintf(glob_fp, "%d %lld %ld\n", rounds, secs_error, nsecs_error); + rounds++; + break; + + default: + skip_slot = false; + memset(&keep_data, '0', sizeof(payload)); + printf("[TSQ-L] Waiting for samples alignment...\n"); + break; + } + + reset_data_ready(pl_valid_flag, CLIENT_COUNT); + memset(&cli_data, 0, sizeof(cli_data)); + + } + } + + close(glob_sockfd); + fclose(glob_fp); +} + +void talker(struct opt *user_opt){ + char *server_ip; + int verbose; + int uid; + int port; + struct sockaddr_in serv; + char send_buff[BUFFER_SIZE]; + int n; + payload data; + int seq = 0; + char *device; + int timeout_ms; + struct pollfd pfd; + struct ptp_extts_event e; + int ready; + int ret; + + if(user_opt == NULL) + error("[TSQ-T] User option is NULL"); + + server_ip = user_opt->server_ip; + port = user_opt->port; + device = user_opt->device; + verbose = user_opt->verbose; + uid = user_opt->uid; + glob_sockfd = 0; + + if (verbose) + printf("[TSQ-T] Assigned uid %d\n", uid); + + /* Set up the socket for transmission */ + memset(&serv, '0', sizeof(serv)); + serv.sin_family = AF_INET; + serv.sin_addr.s_addr = inet_addr(server_ip); + serv.sin_port = htons(port); + + glob_sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (glob_sockfd < 0) + error("[TSQ-T] Could not create socket\n"); + + if (connect(glob_sockfd, (struct sockaddr *)&serv, sizeof(serv)) < 0) + error("[TSQ-T] Connect Failed\n"); + + if (verbose) + printf("[TSQ-T] Connection established with %s\n", server_ip); + + bzero(send_buff, BUFFER_SIZE); + + // send uid + snprintf(send_buff, sizeof(send_buff), "%d\n", uid); + ret = write(glob_sockfd, send_buff, sizeof(send_buff)); + if (ret < 0) + error("[TSQ-T] ERROR writing UID to socket\n"); + + if (verbose) + printf("[TSQ-T] Sent %s to tsq-listener\n", send_buff); + + bzero(send_buff, BUFFER_SIZE); + n = read(glob_sockfd, send_buff, sizeof(send_buff) - 1); + + if (n < 0) + error("[TSQ-T] ERROR reading command from socket\n"); + + if (strcmp(send_buff, "start\n") != 0) + error("[TSQ-T] Connection aborted\n"); + + if (verbose) + printf("[TSQ-T] Reading from %s\n", device); + + glob_ptpfd = open(device, O_RDWR); + if (glob_ptpfd < 0) + error("[TSQ-T] ERROR to open ptp device %s\n", device); + if (verbose) + printf("[TSQ-T] PTP device : %s is now opened\n", device); + + pfd.fd = glob_ptpfd; + pfd.events = PTP_PF_EXTTS; + pfd.revents = 0; + + timeout_ms = user_opt-> timeout; + if (verbose) + printf("[TSQ-T] Setting timeout to %dms\n", timeout_ms); + + while (get_signal() == 0) { + ready = poll(&pfd, 1, timeout_ms); + if (ready < 0) + error("[TSQ-T] Failed to poll\n"); + e.t.sec = 0; + e.t.nsec = 0; + + while (ready-- > 0) { + n = read(glob_ptpfd, &e, sizeof(e)); + if (n != sizeof(e)) { + error("[TSQ-T] read returns %lu bytes, expecting %lu bytes\n", + n, sizeof(e)); + } + } + data.uid = uid; + data.seq = seq; + data.secs = e.t.sec; + data.nsecs = e.t.nsec; + + bzero(send_buff, BUFFER_SIZE); + memcpy(send_buff, &data, sizeof(data)); + + /* + * If secs / nsecs turns out to be zero, it wont be sent over, + * because the values read are not valid. + */ + if ( (data.secs != 0) && (data.nsecs != 0) ) { + if (verbose) + printf("[TSQ-T:%d] Sending %d: %lld#%ld\n", + uid, seq, data.secs, data.nsecs); + + ret = write(glob_sockfd, send_buff, sizeof(send_buff)); + if (ret < 0) + error("[TSQ-T] Erorr writing to socket.\n"); + + seq++; + } else { + if (verbose) + printf ("[TSQ-T] sec:%lld nsec:%ld seq: %d\n", + data.secs, data.nsecs, seq); + } + } + + close(glob_sockfd); +} + +int main(int argc, char *argv[]) +{ + struct opt user_opt; + + /* Defaults */ + user_opt.mode = 0; + user_opt.verbose = 0; + user_opt.device = "\0"; + user_opt.timeout = 1100; // ms + user_opt.uid = 1234; + user_opt.port = 5678; + user_opt.output_file = DEFAULT_LISTENER_OUTFILE; + glob_fp = NULL; + glob_ptpfd = -1; + halt_sig = 0; + + argp_parse(&argp, argc, argv, 0, 0, &user_opt); + + signal(SIGINT, sigint_handler); + signal(SIGTERM, sigint_handler); + signal(SIGABRT, sigint_handler); + + switch(user_opt.mode){ + case MODE_TALKER: + /* IP, PORT, UID, device */ + talker(&user_opt); + break; + case MODE_LISTENER: + /* IP (itself), PORT*/ + listener(&user_opt); + break; + default: + error("Invalid mode selected\n"); + break; + } + + return 0; +} diff --git a/src/txrx-afpkt.c b/src/txrx-afpkt.c new file mode 100644 index 0000000..fc0b468 --- /dev/null +++ b/src/txrx-afpkt.c @@ -0,0 +1,579 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "txrx-afpkt.h" + +#define MAX_PACKETS 10000 +#define MSG_BUFLEN 1500 +#define RCVBUF_SIZE (MSG_BUFLEN * MAX_PACKETS) + +/* Signal handler */ +void afpkt_sigint_handler(int signum) +{ + fprintf(stderr, "Info: SIGINT triggered.\n"); + halt_tx_sig = signum; +} + +/* Retrieve the hardware timestamp stored in CMSG */ +static uint64_t get_timestamp(struct msghdr *msg) +{ + struct timespec *ts = NULL; + struct cmsghdr *cmsg; + + for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET) + continue; + + switch (cmsg->cmsg_type) { + case SO_TIMESTAMPNS: + case SO_TIMESTAMPING: + ts = (struct timespec *) CMSG_DATA(cmsg); + break; + default: /* Ignore other cmsg options */ + break; + } + } + + if (!ts) { + if (verbose) + fprintf(stderr, "Error: timestamp null. Is ptp4l initialized?\n"); + return 0; + } + + return (ts[2].tv_sec * NSEC_PER_SEC + ts[2].tv_nsec); +} + +static uint64_t extract_ts_from_cmsg(int sock, int recvmsg_flags) +{ + char data[256]; + struct msghdr msg; + struct iovec entry; + struct sockaddr_in from_addr; + struct { + struct cmsghdr cm; + char control[512]; + } control; + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = &entry; + msg.msg_iovlen = 1; + entry.iov_base = data; + entry.iov_len = sizeof(data); + msg.msg_name = (caddr_t)&from_addr; + msg.msg_namelen = sizeof(from_addr); + msg.msg_control = &control; + msg.msg_controllen = sizeof(control); + + recvmsg(sock, &msg, recvmsg_flags|MSG_DONTWAIT); + + return get_timestamp(&msg); +} + +int init_tx_socket(struct user_opt *opt, int *sockfd, + struct sockaddr_ll *sk_addr) +{ + struct ifreq hwtstamp = { 0 }; + struct hwtstamp_config hwconfig = { 0 }; + int sock; + + /* Set up socket */ + sock = socket(AF_PACKET, SOCK_DGRAM, htons(ETH_P_8021Q)); + if (sock < 0) + exit_with_error("socket creation failed"); + + sk_addr->sll_ifindex = opt->ifindex; + memcpy(&sk_addr->sll_addr, dst_mac_addr, ETH_ALEN); + + if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &opt->socket_prio, + sizeof(opt->socket_prio)) < 0) + exit_with_error("setsockopt() failed to set priority"); + + /* Similar to: hwstamp_ctl -r 1 -t 1 -i + * This enables tx hw timestamping for all packets. + */ + int timestamping_flags = SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + strncpy(hwtstamp.ifr_name, opt->ifname, sizeof(hwtstamp.ifr_name)-1); + hwtstamp.ifr_data = (void *)&hwconfig; + hwconfig.tx_type = HWTSTAMP_TX_ON; + hwconfig.rx_filter = HWTSTAMP_FILTER_ALL; + + if (ioctl(sock, SIOCSHWTSTAMP, &hwtstamp) < 0) { + fprintf(stderr, "%s: %s\n", "ioctl", strerror(errno)); + exit(1); + } + + if (setsockopt(sock, SOL_SOCKET, SO_TIMESTAMPING, ×tamping_flags, + sizeof(timestamping_flags)) < 0) + exit_with_error("setsockopt SO_TIMESTAMPING"); + + /* Set socket to use SO_TXTIME to pass the transmit time per packet */ + static struct sock_txtime sk_txtime; + int use_deadline_mode = 0; + int receive_errors = 0; + + sk_txtime.clockid = CLOCK_TAI; + sk_txtime.flags = (use_deadline_mode | receive_errors); + if (opt->enable_txtime && setsockopt(sock, SOL_SOCKET, SO_TXTIME, + &sk_txtime, sizeof(sk_txtime))) { + exit_with_error("setsockopt SO_TXTIME"); + } + + *sockfd = sock; + return sock; +} + +/* Thread which creates a socket on a specified priority and continuously + * loops to send packets. Main thread may call multiples of this thread. + */ +void afpkt_send_thread(struct user_opt *opt, int *sockfd, struct sockaddr_ll *sk_addr) +{ + struct custom_payload *payload; + struct timeval timeout; + fd_set readfs, errorfs; + uint64_t tx_timestampA; + uint64_t tx_timestampB; + uint64_t looping_ts; + struct timespec ts; + tsn_packet *tsn_pkt; + void *payload_ptr; + uint8_t *offset; + int res; + int ret; + + int interval_ns = opt->interval_ns; + int count = opt->frames_to_send; + clockid_t clkid = opt->clkid; + int sock = *sockfd; + uint32_t seq = 1; + + /* Create packet template */ + tsn_pkt = alloca(opt->packet_size); + setup_tsn_vlan_packet(opt, tsn_pkt); + + /* TODO SO_TXTIME option but requires sendmsg which breaks sendto()*/ + + looping_ts = get_time_sec(CLOCK_REALTIME) + (2 * NSEC_PER_SEC); + looping_ts += opt->offset_ns; + ts.tv_sec = looping_ts / NSEC_PER_SEC; + ts.tv_nsec = looping_ts % NSEC_PER_SEC; + + payload_ptr = (void *) (&tsn_pkt->payload); + payload = (struct custom_payload *) payload_ptr; + + offset = (uint8_t *) &tsn_pkt->vlan_prio; + + memcpy(&payload->tx_queue, &opt->socket_prio, sizeof(uint32_t)); + + while (count && !halt_tx_sig) { + ret = clock_nanosleep(clkid, TIMER_ABSTIME, &ts, NULL); + if (ret) { + fprintf(stderr, "Error: failed to sleep %d: %s", ret, strerror(ret)); + break; + } + + tx_timestampA = get_time_nanosec(CLOCK_REALTIME); + + memcpy(&payload->seq, &seq, sizeof(uint32_t)); + memcpy(&payload->tx_timestampA, &tx_timestampA, sizeof(uint64_t)); + + ret = sendto(sock, + offset, /* AF_PACKET generates its own ETH HEADER */ + (size_t) (opt->packet_size) - 14, + 0, + (struct sockaddr *) sk_addr, + sizeof(struct sockaddr_ll)); + + if (ret < 0) + exit_with_error("sendto() failed"); + + looping_ts += interval_ns; + ts.tv_sec = looping_ts / NSEC_PER_SEC; + ts.tv_nsec = looping_ts % NSEC_PER_SEC; + + count--; + seq++; + + if (opt->enable_hwts) { + //Note: timeout is duration not timestamp + timeout.tv_usec = 8000; + FD_ZERO(&readfs); + FD_ZERO(&errorfs); + FD_SET(sock, &readfs); + FD_SET(sock, &errorfs); + + res = select(sock + 1, &readfs, 0, &errorfs, &timeout); + } else { + res = 0; + } + + if (res > 0) { + if (FD_ISSET(sock, &errorfs) && verbose) + fprintf(stderr, "CSMG txtimestamp has error\n"); + + tx_timestampB = extract_ts_from_cmsg(sock, MSG_ERRQUEUE); + + /* Result format: seq, user txtime, hw txtime */ + if (verbose) + fprintf(stdout, "%u\t%lu\t%lu\n", + seq - 1, + tx_timestampA, + tx_timestampB); + } else { + /* Print 0 if txtimestamp failed to return in time, + * either indicating hwtstamp is not enabled OR + * packet failed to transmit. + */ + if (verbose) + fprintf(stdout, "%u %lu 0\n", + seq - 1, tx_timestampA); + } + fflush(stdout); + } + + close(sock); + return; +} + +/* Thread which creates a socket on a specified priority and continuously + * loops to send packets. Main thread may call multiples of this thread. + */ +void afpkt_send_thread_etf(struct user_opt *opt, int *sockfd, struct sockaddr_ll *sk_addr) +{ + struct custom_payload *payload; + struct timeval timeout; + fd_set readfs, errorfs; + uint64_t tx_timestamp; + uint64_t tx_timestampA; + uint64_t tx_timestampB; + uint64_t looping_ts; + struct timespec ts; + tsn_packet *tsn_pkt; + void *payload_ptr; + int res; + int ret; + + int interval_ns = opt->interval_ns; + int count = opt->frames_to_send; + clockid_t clkid = opt->clkid; + int sock = *sockfd; + uint32_t seq = 1; + + /* Create packet template */ + tsn_pkt = alloca(opt->packet_size); + setup_tsn_vlan_packet(opt, tsn_pkt); + + struct cmsghdr *cmsg; + struct msghdr msg; + struct iovec iov; + char control[CMSG_SPACE(sizeof(uint64_t))] = {}; + + /* Construct the packet msghdr, CMSG and initialize packet payload */ + iov.iov_base = &tsn_pkt->vlan_prio; + iov.iov_len = (size_t) opt->packet_size - 14; + + memset(&msg, 0, sizeof(msg)); + msg.msg_name = sk_addr; + msg.msg_namelen = sizeof(struct sockaddr_ll); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_TXTIME; + cmsg->cmsg_len = CMSG_LEN(sizeof(uint64_t)); + + /* CMSG end? */ + + looping_ts = get_time_sec(CLOCK_REALTIME) + (2 * NSEC_PER_SEC); + looping_ts += opt->offset_ns; + looping_ts -= opt->early_offset_ns; + ts.tv_sec = looping_ts / NSEC_PER_SEC; + ts.tv_nsec = looping_ts % NSEC_PER_SEC; + + payload_ptr = (void *) (&tsn_pkt->payload); + payload = (struct custom_payload *) payload_ptr; + + memcpy(&payload->tx_queue, &opt->socket_prio, sizeof(uint32_t)); + + while (count && !halt_tx_sig) { + ret = clock_nanosleep(clkid, TIMER_ABSTIME, &ts, NULL); + if (ret) { + fprintf(stderr, "Error: failed to sleep %d: %s", ret, strerror(ret)); + break; + } + + tx_timestampA = get_time_nanosec(CLOCK_REALTIME); + memcpy(&payload->seq, &seq, sizeof(uint32_t)); + memcpy(&payload->tx_timestampA, &tx_timestampA, sizeof(uint64_t)); + + /* Update CMSG tx_timestamp and payload before sending */ + tx_timestamp = looping_ts + opt->early_offset_ns; + *((__u64 *) CMSG_DATA(cmsg)) = tx_timestamp; + + ret = sendmsg(sock, &msg, 0); + if (ret < 1) + printf("sendmsg failed: %m"); + + looping_ts += interval_ns; + ts.tv_sec = looping_ts / NSEC_PER_SEC; + ts.tv_nsec = looping_ts % NSEC_PER_SEC; + + count--; + seq++; + + if (opt->enable_hwts) { + //Note: timeout is duration not timestamp + timeout.tv_usec = 2000; + FD_ZERO(&readfs); + FD_ZERO(&errorfs); + FD_SET(sock, &readfs); + FD_SET(sock, &errorfs); + + res = select(sock + 1, &readfs, 0, &errorfs, &timeout); + } else { + res = 0; + } + + if (res > 0) { + if (FD_ISSET(sock, &errorfs) && verbose) + fprintf(stderr, "CSMG txtimestamp has error\n"); + + tx_timestampB = extract_ts_from_cmsg(sock, MSG_ERRQUEUE); + + /* Result format: seq, user txtime, hw txtime */ + if (verbose) + fprintf(stdout, "%u\t%lu\t%lu\n", + seq - 1, + tx_timestampA, + tx_timestampB); + } else { + /* Print 0 if txtimestamp failed to return in time, + * either indicating hwtstamp is not enabled OR + * packet failed to transmit. + */ + if (verbose) + fprintf(stdout, "%u %lu 0\n", + seq - 1, tx_timestampA); + } + fflush(stdout); + } + + close(sock); + return; +} + +/* Create a RAW socket to receive all incoming packets from an interface. */ +int init_rx_socket(uint16_t etype, int *sock, char *interface) +{ + struct sockaddr_ll addr; + struct ifreq if_request; + int rsock; + int ret; + int timestamping_flags; + int rcvbuf_size; + struct hwtstamp_config hwconfig = {0}; + + rcvbuf_size = RCVBUF_SIZE; + if (sock == NULL) + return -1; + *sock = -1; + + rsock = socket(PF_PACKET, SOCK_RAW, htons(etype)); + if (rsock < 0) + return -1; + + memset(&if_request, 0, sizeof(if_request)); + strncpy(if_request.ifr_name, interface, sizeof(if_request.ifr_name)-1); + + ret = ioctl(rsock, SIOCGIFINDEX, &if_request); + if (ret < 0) { + close(rsock); + fprintf(stderr, "Error: Couldn't get interface index"); + return -1; + } + + /* Increase receive buffer size - requires root priveleges. */ + if (setsockopt(rsock, SOL_SOCKET, SO_RCVBUFFORCE, &rcvbuf_size, + sizeof(int)) < 0) + fprintf(stderr, "Error setting sockopt: %s\n", strerror(errno)); + + memset(&addr, 0, sizeof(addr)); + addr.sll_ifindex = if_request.ifr_ifindex; + addr.sll_family = AF_PACKET; + addr.sll_protocol = htons(etype); + + ret = bind(rsock, (struct sockaddr *)&addr, sizeof(addr)); + if (ret != 0) { + fprintf(stderr, "%s - Error on bind %s\n", + __func__, strerror(errno)); + close(rsock); + return -1; + } + + if (dst_mac_addr[0] != '\0') { + struct packet_mreq mreq; + + mreq.mr_ifindex = addr.sll_ifindex; + mreq.mr_type = PACKET_MR_MULTICAST; + mreq.mr_alen = ETH_ALEN; + memcpy(&mreq.mr_address, dst_mac_addr, ETH_ALEN); + + ret = setsockopt(rsock, SOL_PACKET, PACKET_ADD_MEMBERSHIP, + &mreq, sizeof(struct packet_mreq)); + if (ret < 0) { + perror("Couldn't set PACKET_ADD_MEMBERSHIP"); + close(rsock); + return -1; + } + } + + /* Similar to: hwstamp_ctl -r 1 -t 1 -i + * This enables rx hw timestamping for all packets. + */ + if_request.ifr_data = (void *)&hwconfig; + + hwconfig.tx_type = HWTSTAMP_TX_ON; + hwconfig.rx_filter = HWTSTAMP_FILTER_ALL; + + if (ioctl(rsock, SIOCSHWTSTAMP, &if_request) < 0) { + fprintf(stderr, "%s: %s\n", "ioctl", strerror(errno)); + exit(1); + } + + timestamping_flags = SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + if (setsockopt(rsock, SOL_SOCKET, SO_TIMESTAMPING, ×tamping_flags, + sizeof(timestamping_flags)) < 0) + exit_with_error("setsockopt SO_TIMESTAMPING"); + + *sock = rsock; + return 0; +} + +int afpkt_recv_pkt(int sock, struct user_opt *opt) +{ + uint64_t rx_timestampC, rx_timestampD; + struct sockaddr_in host_address; + struct custom_payload *payload; + struct msghdr msg; + struct iovec iov; + char buffer[MSG_BUFLEN]; + char control[1024]; + tsn_packet *tsn_pkt; + void *payload_ptr; + int ret; + + ret = 0; + + /* Initialize empty msghdrs and buffers */ + bzero(&host_address, sizeof(struct sockaddr_in)); + host_address.sin_family = AF_INET; + host_address.sin_port = htons(0); + host_address.sin_addr.s_addr = INADDR_ANY; + + iov.iov_base = buffer; + iov.iov_len = 128; //TODO: use correct length based on VLAN header + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = &host_address; + msg.msg_namelen = sizeof(struct sockaddr_in); + msg.msg_control = control; + msg.msg_controllen = 128; + + /* Use non-blocking recvmsg to poll for packets, do nothing if none */ + ret = recvmsg(sock, &msg, MSG_DONTWAIT); + if (ret <= 0) { + usleep(1); /*No message in buffer, do nothing*/ + return 0; + } + rx_timestampD = get_time_nanosec(CLOCK_REALTIME); + + /* Point to payload's location in received packet's buffer */ + tsn_pkt = (tsn_packet *) (buffer - 4); + payload_ptr = (void *) (&tsn_pkt->payload); + payload = (struct custom_payload *) payload_ptr; + + if (opt->enable_hwts) + rx_timestampC = get_timestamp(&msg); + else + rx_timestampC = 0; + + /* Do simple checks and filtering */ + if (payload->tx_queue > 8 || payload->seq > (50 * 1000 * 1000)) { + if (verbose) + fprintf(stderr, "Warn: Skipping invalid packet\n"); + return -1; + } else if (rx_timestampC == 0) { + if (verbose) + fprintf(stderr, "Warn: No RX HW timestamp.\n"); + } + + /* Result format: + * u2u latency, seq, queue, user txtime, hw rxtime, user rxtime + */ + fprintf(stdout, "%ld\t%d\t%d\t%ld\t%ld\t%ld\n", + rx_timestampD - payload->tx_timestampA, + payload->seq, + payload->tx_queue, + payload->tx_timestampA, + rx_timestampC, + rx_timestampD); + fflush(stdout); + return 0; +} diff --git a/src/txrx-afpkt.h b/src/txrx-afpkt.h new file mode 100644 index 0000000..decc8dd --- /dev/null +++ b/src/txrx-afpkt.h @@ -0,0 +1,48 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include +#include +#include /* the L2 protocols */ +#include + +#include "txrx.h" + +extern int halt_tx_sig; +extern int verbose; + +void afpkt_sigint_handler(int signum); +int init_tx_socket(struct user_opt *opt, int *sockfd, struct sockaddr_ll *sk_addr); +void afpkt_send_thread(struct user_opt *opt, int *sockfd, struct sockaddr_ll *sk_addr); +void afpkt_send_thread_etf(struct user_opt *opt, int *sockfd, struct sockaddr_ll *sk_addr); +int init_rx_socket(uint16_t etype, int *sock, char *interface); +int afpkt_recv_pkt(int sock, struct user_opt *opt); diff --git a/src/txrx-afxdp.c b/src/txrx-afxdp.c new file mode 100644 index 0000000..30de4d4 --- /dev/null +++ b/src/txrx-afxdp.c @@ -0,0 +1,630 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Ethernet header */ +#include + +/* if_nametoindex() */ +#include + +/* getpagesize() */ +#include + +/* pthread_yield() */ +#include + +/* Hwtstamp_config */ +#include + +/* RLIMIT */ +#include +#include + +/* XSK */ +#include +#include "linux/if_xdp.h" +#include +#include +#include + +#include "txrx-afxdp.h" + +struct xsk_info *glob_xskinfo_ptr; +uint32_t glob_xdp_flags; +int glob_ifindex; +int verbose; + +/* User Defines */ +#define BATCH_SIZE 64 //for l2fwd only + +/* Signal handler to gracefully shutdown */ +void afxdp_sigint_handler(int signum) +{ + fprintf(stderr, "Info: SIGINT triggered.\n"); //TODO SIGNUM? + halt_tx_sig = signum; +} + +void remove_xdp_program(void) +{ + uint32_t curr_prog_id = 0; + + if (bpf_get_link_xdp_id(glob_ifindex, &curr_prog_id, glob_xdp_flags)) { + fprintf(stderr, "bpf_get_link_xdp_id failed\n"); + exit(EXIT_FAILURE); + } + + if (glob_xskinfo_ptr->bpf_prog_id == curr_prog_id) + bpf_set_link_xdp_fd(glob_ifindex, -1, glob_xdp_flags); + else if (!curr_prog_id) + fprintf(stderr, "couldn't find a prog id on a given interface\n"); + else + fprintf(stderr, "program on interface changed, not removing\n"); +} + +void xdpsock_cleanup(void) +{ + struct xsk_umem *umem = glob_xskinfo_ptr->pktbuff->umem; + + xsk_socket__delete(glob_xskinfo_ptr->xskfd); + (void)xsk_umem__delete(umem); + remove_xdp_program(); + + exit(EXIT_SUCCESS); +} + +void __afxdp_exit_with_error(int error, const char *file, const char *func, int line) +{ + fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, + line, error, strerror(error)); + remove_xdp_program(); + exit(EXIT_FAILURE); +} + +/* Create a umem using the buffer provided? TODO what is ubuf for*/ +static struct pkt_buffer *create_umem(void *ubuf, struct xsk_opt *x_opt) +{ + uint64_t single_umem_ring_size; + struct pkt_buffer *temp_buff; + uint32_t idx = 0; + int ret; + int i; + + struct xsk_umem_config uconfig = { + .fill_size = x_opt->frames_per_ring, + .comp_size = x_opt->frames_per_ring, + .frame_size = x_opt->frame_size, + .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, + }; + + single_umem_ring_size = x_opt->frames_per_ring * x_opt->frame_size; + + ret = posix_memalign(&ubuf, getpagesize(), /* PAGE_SIZE aligned */ + single_umem_ring_size); + if (ret) + exit(EXIT_FAILURE); + + temp_buff = calloc(1, sizeof(*temp_buff)); + if (!temp_buff) + afxdp_exit_with_error(errno); + + ret = xsk_umem__create(&temp_buff->umem, ubuf, single_umem_ring_size, + &temp_buff->rx_fill_ring, &temp_buff->tx_comp_ring, + &uconfig); + if (ret) + afxdp_exit_with_error(-ret); + + temp_buff->buffer = ubuf; + + /* Populate rx fill ring with addresses */ + ret = xsk_ring_prod__reserve(&temp_buff->rx_fill_ring, + x_opt->frames_per_ring, + &idx); + if (ret != x_opt->frames_per_ring) + return (struct pkt_buffer *) - 1; + + for (i = 0; i < x_opt->frames_per_ring; i++) + *xsk_ring_prod__fill_addr(&temp_buff->rx_fill_ring, idx++) = + i * x_opt->frame_size; + + xsk_ring_prod__submit(&temp_buff->rx_fill_ring, + x_opt->frames_per_ring); + + return temp_buff; +} + +static struct xsk_info *create_xsk_info(struct user_opt *opt, struct pkt_buffer *pktbuff) +{ + struct xsk_socket_config cfg; + struct xsk_info *temp_xsk; + int ret; + + temp_xsk = calloc(1, sizeof(*temp_xsk)); + if (!temp_xsk) + afxdp_exit_with_error(errno); + + temp_xsk->pktbuff = pktbuff; + + cfg.rx_size = opt->x_opt.frames_per_ring; //Use same size for TX RX + cfg.tx_size = opt->x_opt.frames_per_ring; + + cfg.libbpf_flags = 0; + cfg.xdp_flags = opt->x_opt.xdp_flags; + cfg.bind_flags = opt->x_opt.xdp_bind_flags; + + ret = xsk_socket__create(&temp_xsk->xskfd, opt->ifname, + opt->x_opt.queue, temp_xsk->pktbuff->umem, + &temp_xsk->rx_ring, &temp_xsk->tx_ring, &cfg); + if (ret) + afxdp_exit_with_error(-ret); + + ret = bpf_get_link_xdp_id(opt->ifindex, &temp_xsk->bpf_prog_id, opt->x_opt.xdp_flags); + if (ret) + afxdp_exit_with_error(-ret); + + return temp_xsk; +} + +static void prefill_tx_umem_rings(void *buff_addr, tsn_packet *example_pkt, + int count, int frame_size) +{ + int i; + + for (i = 0; i < count; i++) { + memcpy(xsk_umem__get_data(buff_addr, i * frame_size), + example_pkt, sizeof(*example_pkt) - 1); + // Note: frame size != packet size + } +} + +void init_xdp_socket(struct user_opt *opt) +{ + struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; + void *ubuf = NULL; + + opt->x_opt.xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; + opt->x_opt.xdp_bind_flags = 0; + + switch(opt->xdp_mode) { + case XDP_MODE_SKB_COPY: + opt->x_opt.xdp_flags |= XDP_FLAGS_SKB_MODE; + opt->x_opt.xdp_bind_flags |= XDP_COPY; + break; + case XDP_MODE_NATIVE_COPY: + opt->x_opt.xdp_flags |= XDP_FLAGS_DRV_MODE; + opt->x_opt.xdp_bind_flags |= XDP_COPY; + break; + case XDP_MODE_ZERO_COPY: + // opt->x_opt.xdp_flags |= XDP_FLAGS_DRV_MODE; + opt->x_opt.xdp_bind_flags |= XDP_ZEROCOPY; + break; + default: + exit_with_error("ERROR: XDP Mode s,c or z must be specified\n"); + break; + } + + if (opt->need_wakeup) + opt->x_opt.xdp_bind_flags |= XDP_USE_NEED_WAKEUP; + + glob_xdp_flags = opt->x_opt.xdp_flags; + glob_ifindex = opt->ifindex; + + /* Let this app have all resource. Need root */ + if (setrlimit(RLIMIT_MEMLOCK, &r)) { + fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", + strerror(errno)); + exit(EXIT_FAILURE); + } + + /* Create the umem and store the pointers */ + struct pkt_buffer *pktbuffer; + pktbuffer = create_umem(ubuf, &opt->x_opt); + + /* Assign the umem to a socket */ + opt->xsk = create_xsk_info(opt, pktbuffer); + glob_xskinfo_ptr = opt->xsk; + +} + +static void kick_tx(struct xsk_info *xsk) +{ + int ret; + + ret = sendto(xsk_socket__fd(xsk->xskfd), NULL, 0, MSG_DONTWAIT, NULL, 0); + if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) + return; + afxdp_exit_with_error(errno); +} + +static void update_txstats(struct xsk_info *xsk) +{ + uint32_t rcvd; + uint32_t idx; + + if (!xsk->outstanding_tx) + return; + + //TODO: keep track of stats + rcvd = xsk_ring_cons__peek(&xsk->pktbuff->tx_comp_ring, 1, &idx); + if (rcvd > 0) { + xsk_ring_cons__release(&xsk->pktbuff->tx_comp_ring, rcvd); + xsk->outstanding_tx -= rcvd; + xsk->tx_npkts += rcvd; + } +} + +static void afxdp_send_pkt(struct xsk_info *xsk, struct user_opt *opt, + uint32_t header_size, uint32_t packet_size, + void *payload, uint64_t tx_timestamp) +{ + uint64_t cur_tx = xsk->cur_tx; //packet_count * frame_size + uint32_t pkt_per_send = 1; //Dont do bactching for now. + uint32_t idx = 0; + uint8_t *umem_data; + int ret; + + if (opt->enable_poll) { + int nfds = 1; + struct pollfd fds[nfds + 1]; + int timeout = 1000; /* in ms, so 1 second */ + + memset(fds, 0, sizeof(fds)); + fds[0].fd = xsk_socket__fd(xsk->xskfd); + fds[0].events = POLLOUT; + + ret = poll(fds, nfds, timeout); + if (ret <= 0) + return; //TODO: Return a EBUSY or EAGAIN + + if (!(fds[0].revents & POLLOUT)) + return; //TODO: Return a EBUSY or EAGAIN + } + + /* Actual filling of payload into umem. Start a loop here if batching. */ + umem_data = xsk_umem__get_data(xsk->pktbuff->buffer, cur_tx << XSK_UMEM__DEFAULT_FRAME_SHIFT); + + memcpy(umem_data + header_size, payload, packet_size - header_size); + + if (xsk_ring_prod__reserve(&xsk->tx_ring, pkt_per_send, &idx) != pkt_per_send) + return; //TODO return EGAGIN or ENOBUFF + + if (!opt->enable_txtime) + tx_timestamp = 0; //Just in case + + //We need to update addr every time, for cases where the umem/tx_ring is shared. + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx)->addr = cur_tx << XSK_UMEM__DEFAULT_FRAME_SHIFT; + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx)->len = packet_size; + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx)->txtime = tx_timestamp; + + /* Update counters */ + xsk_ring_prod__submit(&xsk->tx_ring, pkt_per_send); + xsk->outstanding_tx += pkt_per_send; + + xsk->cur_tx += pkt_per_send; + xsk->cur_tx %= opt->x_opt.frames_per_ring; + + // Complete the TX sequence. + ret = sendto(xsk_socket__fd(xsk->xskfd), NULL, 0, MSG_DONTWAIT, NULL, 0); + if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) { + update_txstats(xsk); + return; //TODO: Return ESUCCESS? + } + + afxdp_exit_with_error(errno); +} + +void *afxdp_send_thread(void *arg) +{ + struct user_opt *opt = (struct user_opt *)arg; + + struct custom_payload *payload; + char buff[opt->packet_size]; + uint64_t sleep_timestamp; + uint64_t tx_timestamp; + tsn_packet *tsn_pkt; + struct timespec ts; + + struct xsk_info *xsk = opt->xsk; + uint64_t seq_num = 1; + uint64_t i = 0; + + /* Create packet template */ + tsn_pkt = alloca(opt->packet_size); + setup_tsn_vlan_packet(opt, tsn_pkt); + + prefill_tx_umem_rings(xsk->pktbuff->buffer, tsn_pkt, + opt->x_opt.frames_per_ring, + opt->x_opt.frame_size); + + payload = (struct custom_payload *) buff; + + tx_timestamp = get_time_sec(CLOCK_REALTIME); //0.5s ahead (stmmac limitation) + tx_timestamp += opt->offset_ns; + tx_timestamp += 2 * NSEC_PER_SEC; + + while(!halt_tx_sig && (i < opt->frames_to_send) ) { + + sleep_timestamp = tx_timestamp - opt->early_offset_ns; + ts.tv_sec = sleep_timestamp / NSEC_PER_SEC; + ts.tv_nsec = sleep_timestamp % NSEC_PER_SEC; + clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &ts, NULL); + + payload->tx_queue = opt->x_opt.queue; + payload->seq = seq_num; + payload->tx_timestampA = get_time_nanosec(CLOCK_REALTIME); + + //Send one packet without caring about descriptors, make it look normal. + if (opt->enable_txtime) + afxdp_send_pkt(xsk, opt, 18, opt->packet_size, &buff, tx_timestamp); + else + afxdp_send_pkt(xsk, opt, 18, opt->packet_size, &buff, 0); + + /* Result format: + * seq, user txtime, hw txtime is via trace for now + */ + if (verbose) + fprintf(stdout, "%d\t%ld\n", payload->seq, payload->tx_timestampA); + seq_num++; + tx_timestamp += opt->interval_ns; + fflush(stdout); + + i++; + } + + update_txstats(xsk); + + return NULL; + /* Calling thread is responsible of removing xdp program */ +} + +// Receive 1 packet at a time and print it. +void afxdp_recv_pkt(struct xsk_info *xsk, void *rbuff) +{ + struct custom_payload *payload; + uint64_t rx_timestampD; + tsn_packet *tsn_pkt; + void *payload_ptr; + (void) rbuff; + int rcvd, i; + int ret; + + uint32_t idx_rx = 0, idx_fq = 0; + + rcvd = xsk_ring_cons__peek(&xsk->rx_ring, 1, &idx_rx); + if (!rcvd) + return; + + ret = xsk_ring_prod__reserve(&xsk->pktbuff->rx_fill_ring, rcvd, &idx_fq); + while (ret != rcvd) { + if (ret < 0) + afxdp_exit_with_error(-ret); + ret = xsk_ring_prod__reserve(&xsk->pktbuff->rx_fill_ring, rcvd, &idx_fq); + } + + for (i = 0; i < rcvd; i++) { + uint64_t addr = xsk_ring_cons__rx_desc(&xsk->rx_ring, idx_rx)->addr; + uint32_t len = xsk_ring_cons__rx_desc(&xsk->rx_ring, idx_rx++)->len; + + char *pkt = xsk_umem__get_data(xsk->pktbuff->buffer, addr); + + if (!len) { + fprintf(stderr, "Warning: packet received with zero-length\n"); + continue; + } + + rx_timestampD = get_time_nanosec(CLOCK_REALTIME); + + tsn_pkt = (tsn_packet *) pkt; + payload_ptr = (void *) (&tsn_pkt->payload); + payload = (struct custom_payload *) payload_ptr; + + if ((tsn_pkt->vlan_hdr == 0x81 || tsn_pkt->vlan_hdr == 0x08) && + (tsn_pkt->eth_hdr == htons(0xb62c)) && + (payload->seq > 0 && payload->seq < (50 * 1000 * 1000)) && + (tsn_pkt->vlan_prio / 32) < 8) { + + fprintf(stdout, "%lu\t%u\t%u\t%lu\t%lu\t%lu\n", + rx_timestampD - payload->tx_timestampA, + payload->seq, + payload->tx_queue, + payload->tx_timestampA, + *(uint64_t *)(pkt - sizeof(uint64_t)), + rx_timestampD); + } else if (verbose) { + fprintf(stderr, "Info: packet received type: 0x%x\n", + tsn_pkt->eth_hdr); + } + } + + xsk_ring_prod__submit(&xsk->pktbuff->rx_fill_ring, rcvd); + xsk_ring_cons__release(&xsk->rx_ring, rcvd); + xsk->rx_npkts += rcvd; + fflush(stdout); + + /* FOR SCHED_FIFO/DEADLINE */ + //TODO:implement for all threads incl afpkt? + pthread_yield(); +} + +static void swap_mac_addresses(void *data) { + struct ether_header *eth = (struct ether_header *)data; + struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; + struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; + struct ether_addr tmp; + + tmp = *src_addr; + *src_addr = *dst_addr; + *dst_addr = tmp; +} + +void afxdp_fwd_pkt(struct xsk_info *xsk, struct pollfd *fds, struct user_opt *opt) +{ + struct custom_payload *payload; + uint64_t tx_timestamp; + tsn_packet *tsn_pkt; + void *payload_ptr; + size_t ndescs; + int rcvd, i; + int ret; + + uint32_t idx_rx = 0, idx_tx = 0; + + if (xsk->outstanding_tx) { + /* Since there is out-standing TX and kernel needs Tx wakeup call, + * user app kicks TX proces here: + * sendto() then inside kernel calls xsk_zc_xmit() --> xsk_wakeup() + * --> driver's ndo_xsk_wakeup() + */ + if (!opt->need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx_ring)) + kick_tx(xsk); + + ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : xsk->outstanding_tx; + /* re-add completed Tx buffers */ + rcvd = xsk_ring_cons__peek(&xsk->pktbuff->tx_comp_ring, ndescs, &idx_tx); + if (rcvd > 0) { + ret = xsk_ring_prod__reserve(&xsk->pktbuff->rx_fill_ring, rcvd, &idx_rx); + while (ret != rcvd) { + if (ret < 0) + afxdp_exit_with_error(-ret); + + if (xsk_ring_prod__needs_wakeup(&xsk->pktbuff->rx_fill_ring)){ + ret = poll(fds, 1, opt->poll_timeout); + continue; + } + + ret = xsk_ring_prod__reserve(&xsk->pktbuff->rx_fill_ring, rcvd, &idx_rx); + } + + for (i = 0; i < rcvd; i++) + *xsk_ring_prod__fill_addr(&xsk->pktbuff->rx_fill_ring, idx_rx++) = + *xsk_ring_cons__comp_addr(&xsk->pktbuff->tx_comp_ring, idx_tx++); + + xsk_ring_prod__submit(&xsk->pktbuff->rx_fill_ring, rcvd); + xsk_ring_cons__release(&xsk->pktbuff->tx_comp_ring, rcvd); + xsk->outstanding_tx -= rcvd; + xsk->tx_npkts += rcvd; + } + } + + /* Now, peek RX ring has any entries. + * Note: driver calls xdp_do_flush_map(XSK_REDIR) --> xsk_map_flush() --> + * xsk_flush() --> xskq_produce_flush_desc() which updates the + * UMEM's RX Desc Ring's producer value. The xsk_ring_cons__peek() + * function calculates the available RX desc based on the cached + * RX Ring producer and consumer copy. + */ + rcvd = xsk_ring_cons__peek(&xsk->rx_ring, BATCH_SIZE, &idx_rx); + if (!rcvd){ + /* Note: + * a) xsk_set|clear_rx_need_wakeup() actually mark RX Fill queue + * b) xsk_set|clear_tx_need_wakeup() actually mark TX XMIT queue + */ + if (xsk_ring_prod__needs_wakeup(&xsk->pktbuff->rx_fill_ring)) + ret = poll(fds, 1, opt->poll_timeout); + return; + } + + ret = xsk_ring_prod__reserve(&xsk->tx_ring, rcvd, &idx_tx); + while (ret != rcvd) { + /* If we fail to reserve equal amount of TX entry for + * 'rcvd' RX entries that show up, we keep polling until + * Tx has empty slots for us. + */ + if (ret < 0) + afxdp_exit_with_error(-ret); + if (xsk_ring_prod__needs_wakeup(&xsk->tx_ring)) + kick_tx(xsk); + ret = xsk_ring_prod__reserve(&xsk->tx_ring, rcvd, &idx_tx); + } + + /* Now, we are good to receive all those RX entries and forward them to + * TX Queue as L2 Forwarding + */ + for (i = 0; i < rcvd; i++) { + uint64_t addr = xsk_ring_cons__rx_desc(&xsk->rx_ring, + idx_rx)->addr; + uint32_t len = xsk_ring_cons__rx_desc(&xsk->rx_ring, + idx_rx++)->len; + uint64_t orig = addr; + + addr = xsk_umem__add_offset_to_addr(addr); + char *pkt = xsk_umem__get_data(xsk->pktbuff->buffer, addr); + + swap_mac_addresses(pkt); + + tsn_pkt = (tsn_packet *) pkt; + payload_ptr = (void *) (&tsn_pkt->payload); + payload = (struct custom_payload *) payload_ptr; + + payload->rx_timestampD = get_time_nanosec(CLOCK_REALTIME); + + if (!opt->enable_txtime) + tx_timestamp = 0; + else + tx_timestamp = (payload->rx_timestampD / opt->interval_ns) * opt->interval_ns + + opt->interval_ns + + opt->offset_ns; + + if (verbose) + fprintf(stdout, "1\t%d\t%d\t%ld\t%ld\t%ld\n", + payload->seq, + tsn_pkt->vlan_prio / 32, + payload->tx_timestampA, + *(uint64_t *)(pkt - sizeof(uint64_t)), //rx hw + payload->rx_timestampD); + + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx_tx)->addr = orig; + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx_tx)->len = len; + xsk_ring_prod__tx_desc(&xsk->tx_ring, idx_tx)->txtime = tx_timestamp; + idx_tx++; + } + + xsk_ring_prod__submit(&xsk->tx_ring, rcvd); + xsk_ring_cons__release(&xsk->rx_ring, rcvd); + + xsk->rx_npkts += rcvd; + xsk->outstanding_tx += rcvd; + fflush(stdout); + + pthread_yield(); +} diff --git a/src/txrx-afxdp.h b/src/txrx-afxdp.h new file mode 100644 index 0000000..1381657 --- /dev/null +++ b/src/txrx-afxdp.h @@ -0,0 +1,52 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#include + +#include "txrx.h" + +extern struct xsk_info *glob_xskinfo_ptr; +extern uint32_t glob_xdp_flags; +extern int glob_ifindex; +extern int halt_tx_sig; +extern int verbose; + +void remove_xdp_program(void); +void xdpsock_cleanup(); +void afxdp_sigint_handler(int sig); +void __afxdp_exit_with_error(int error, const char *file, const char *func, int line); +void init_xdp_socket(struct user_opt *opt); +void *afxdp_send_thread(void *arg); +void afxdp_recv_pkt(struct xsk_info *xsk, void *rbuff); +void afxdp_fwd_pkt(struct xsk_info *xsk, struct pollfd *fds, struct user_opt *opt); + +#define afxdp_exit_with_error(error) __afxdp_exit_with_error(error, __FILE__, __func__, __LINE__) diff --git a/src/txrx.c b/src/txrx.c new file mode 100644 index 0000000..791ae42 --- /dev/null +++ b/src/txrx.c @@ -0,0 +1,417 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "txrx-afxdp.h" +#include "txrx-afpkt.h" + +#define VLAN_ID 3 +#define DEFAULT_NUM_FRAMES 1000 +#define DEFAULT_TX_PERIOD 0 +#define DEFAULT_PACKETS 10 +#define DEFAULT_SOCKET_PRIORITY 0 +#define DEFAULT_PACKET_SIZE 64 +#define DEFAULT_TXTIME_OFFSET 0 +#define DEFAULT_EARLY_OFFSET 100000 +#define DEFAULT_XDP_FRAMES_PER_RING 4096 //Minimum is 4096 +#define DEFAULT_XDP_FRAMES_SIZE 4096 +#define BATCH_SIZE 64 //for l2fwd + +/* Globals */ +unsigned char src_mac_addr[] = { 0xaa, 0x00, 0xaa, 0x00, 0xaa, 0x00}; +unsigned char dst_mac_addr[] = { 0x22, 0xbb, 0x22, 0xbb, 0x22, 0xbb}; +unsigned char src_ip_addr[] = { 169, 254, 1, 11}; +unsigned char dst_ip_addr[] = { 169, 254, 1, 22}; +struct xsk_info *glob_xskinfo_ptr; +uint32_t glob_xdp_flags; +int glob_ifindex; +int halt_tx_sig; +int verbose; + +uint64_t get_time_nanosec(clockid_t clkid) +{ + struct timespec now; + + clock_gettime(clkid, &now); + return now.tv_sec * NSEC_PER_SEC + now.tv_nsec; +} + +uint64_t get_time_sec(clockid_t clkid) +{ + struct timespec now; + + clock_gettime(clkid, &now); + return now.tv_sec * NSEC_PER_SEC; +} + +/* Pre-fill TSN packet with default and user-defined parameters */ +void setup_tsn_vlan_packet(struct user_opt *opt, tsn_packet *pkt) +{ + memset(pkt, 0xab, opt->packet_size); + + memcpy(&pkt->src_mac, src_mac_addr, sizeof(pkt->src_mac)); + memcpy(&pkt->dst_mac, dst_mac_addr, sizeof(pkt->dst_mac)); + + pkt->vlan_hdr = htons(ETHERTYPE_VLAN); + pkt->vlan_id = VLAN_ID; + pkt->vlan_prio = opt->vlan_prio; + + pkt->eth_hdr = htons(ETH_P_TSN); + + /* WORKAROUND: transmit only 0xb62c ETH UADP header packets + * due to a bug where IEEE 1722 (ETH_P_TSN) packets are + * always steered into RX Q0 regardless of its VLAN + * priority + */ + pkt->eth_hdr = htons(0xb62c); +} + +/* Argparse */ +static struct argp_option options[] = { + {"interface", 'i', "NAME", 0, "interface name"}, + + {0,0,0,0, "Socket Mode:" }, + {"afxdp", 'X', 0, 0, "run using AF_XDP socket"}, + {"afpkt", 'P', 0, 0, "run using AF_PACKET socket"}, + + {0,0,0,0, "Mode:" }, + {"transmit", 't', 0, 0, "transmit only"}, + {"receive", 'r', 0, 0, "receive only"}, + {"forward", 'f', 0, 0, "L2-forward (receive-send) packets (AF_XDP only)"}, + {"boomerang", 'b', 0, 0, "L2-send-receive packets (AF_XDP only)"}, + + {0,0,0,0, "XDP Mode:" }, + {"zero-copy", 'z', 0, 0, "zero-copy mode"}, + {"native-copy", 'c', 0, 0, "direct/native copy mode"}, + {"skb", 's', 0, 0, "skb mode"}, + + {0,0,0,0, "AF_XDP control:" }, + {"launchtime", 'T', 0, 0, "enable time-based per-packet tx scheduling (TBS)"}, + {"polling", 'p', 0, 0, "enable polling mode"}, + {"wakeup", 'w', 0, 0, "enable need_wakeup"}, + {"vlan-prio", 'q', "NUM", 0, "packet vlan priority, also socket priority\n" + " Def: 0 | Min: 0 | Max: 7"}, + /* Reserved: u / w */ + + {0,0,0,0, "TX control:" }, + {"packet-size", 'l', "NUM", 0, "packet size/length incl. headers in bytes\n" + " Def: 64 | Min: 64 | Max: 1500"}, + {"cycle-time", 'y', "NSEC", 0, "tx period/interval/cycle-time\n" + " Def: 100000ns | Min: 25000ns | Max: 50000000ns"}, + {"frames-to-send", 'n', "NUM", 0, "number of packets to transmit\n" + " Def: 10 | Min: 1 | Max: 10000000"}, + {"dst-mac-addr", 'd', "MAC_ADDR", 0, "destination mac address\n" + " Def: 22:bb:22:bb:22:bb"}, + + {0,0,0,0, "LaunchTime/TBS-specific:\n(where base is the 0th ns of current second)" }, + {"transmit-offset",'o', "NSEC", 0, "packet txtime positive offset\n" + " Def: 0ns | Min: 1ns | Max: 100000000ns"}, + {"early-offset", 'e', "NSEC", 0, "early execution negative offset\n" + " Def: 0ns | Min: 0ns | Max: 10000000ns"}, + + {0,0,0,0, "Misc:" }, + {"hw-timestamps", 'h', 0, 0, "retrieve per-packet hardware timestamps (AF_PACKET)"}, + {"verbose", 'v', 0, 0, "verbose & print warnings"}, + { 0 } +}; + +static error_t parser(int key, char *arg, struct argp_state *state) +{ + /* Get the input argument from argp_parse, which we */ + /* know is a pointer to our user_opt structure. */ + struct user_opt *opt = state->input; + int ret; + + switch (key) { + case 'v': + verbose = 1; + break; + case 'h': + opt->enable_hwts = 1; + break; + case 'i': + opt->ifname = strdup(arg); + break; + case 'q': + opt->x_opt.queue = opt->socket_prio = atoi(arg); + if (atoi(arg) < 0 || opt->socket_prio >= 7) + exit_with_error("Invalid queue number. Check --help"); + opt->vlan_prio = opt->socket_prio * 32; + break; + case 'X': + opt->socket_mode = MODE_AFXDP; + break; + case 'P': + opt->socket_mode = MODE_AFPKT; + break; + case 't': + opt->mode = MODE_TX; + break; + case 'r': + opt->mode = MODE_RX; + break; + case 'f': + opt->mode = MODE_FWD; + break; + case 'b': + opt->mode = MODE_BMR; + break; + case 'z': + opt->xdp_mode = XDP_MODE_ZERO_COPY; + break; + case 'c': + opt->xdp_mode = XDP_MODE_NATIVE_COPY; + break; + case 's': + opt->xdp_mode = XDP_MODE_SKB_COPY; + break; + case 'T': + opt->enable_txtime = 1; + break; + case 'p': + opt->enable_poll = 1; + break; + case 'w': + opt->need_wakeup = 1; + break; + case 'l': + opt->packet_size = atoi(arg); + if (opt->packet_size < 64 || opt->packet_size > 1500) + exit_with_error("Invalid packet size. Check --help"); + break; + case 'y': + opt->interval_ns = atoi(arg); + if (opt->interval_ns < 25000 || opt->interval_ns > 50000000) + exit_with_error("Invalid cycle time. Check --help"); + break; + case 'n': + opt->frames_to_send = atoi(arg); + if (opt->frames_to_send < 1 || opt->frames_to_send > 10000000) + exit_with_error("Invalid number of frames to send. Check --help"); + break; + case 'o': + opt->offset_ns = atoi(arg); + if (atoi(arg) < 0 || opt->offset_ns > 100000000) + exit_with_error("Invalid offset. Check --help"); + break; + case 'e': + opt->early_offset_ns = atoi(arg); + if (atoi(arg) < 0 || opt->early_offset_ns > 10000000) + exit_with_error("Invalid early offset. Check --help"); + break; + case 'd': + ret = sscanf(arg, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", + &dst_mac_addr[0], &dst_mac_addr[1], &dst_mac_addr[2], + &dst_mac_addr[3], &dst_mac_addr[4], &dst_mac_addr[5]); + if (ret != 6) + exit_with_error("Invalid destination MAC addr. Check --help"); + break; + /* Reserved: u / w */ + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static char usage[] = "-i -P [r|t]\n" + "-i -X [r|t|f] [z|c|s] -q "; + +static char summary[] = " AF_XDP & AF_PACKET Transmit-Receive Application"; + +static struct argp argp = { options, parser, usage, summary }; + +int main(int argc, char *argv[]) +{ + struct pollfd fds[1]; + struct user_opt opt; + int ret = 0; + + /* Defaults */ + opt.socket_mode = MODE_INVALID; + opt.mode = MODE_INVALID; + opt.ifname = NULL; + verbose = 0; + + opt.socket_prio = DEFAULT_SOCKET_PRIORITY; + opt.vlan_prio = DEFAULT_SOCKET_PRIORITY; + opt.frames_to_send = DEFAULT_NUM_FRAMES; + opt.packet_size = DEFAULT_PACKET_SIZE; + opt.interval_ns = DEFAULT_TX_PERIOD; + + opt.x_opt.frames_per_ring = DEFAULT_XDP_FRAMES_PER_RING; + opt.x_opt.frame_size = DEFAULT_XDP_FRAMES_SIZE; + opt.early_offset_ns = DEFAULT_EARLY_OFFSET; + opt.offset_ns = DEFAULT_TXTIME_OFFSET; + opt.xdp_mode = XDP_MODE_ZERO_COPY; + opt.clkid = CLOCK_REALTIME; + opt.enable_poll = 0; + opt.enable_hwts = 0; + opt.enable_txtime = 0; + opt.need_wakeup = false; + opt.poll_timeout = 1000; + + argp_parse(&argp, argc, argv, 0, 0, &opt); + + /* Parse user inputs */ + + if (!opt.ifname) + exit_with_error("Please specify interface using -i\n"); + + opt.ifindex = if_nametoindex(opt.ifname); + if (!opt.ifindex) { + fprintf(stderr, "ERROR: interface \"%s\" do not exist\n", + opt.ifname); + exit(EXIT_FAILURE); + } + + char buff[opt.packet_size]; + pthread_t thread1; + + switch (opt.socket_mode) { + case MODE_AFPKT: + signal(SIGINT, afpkt_sigint_handler); + signal(SIGTERM, afpkt_sigint_handler); + signal(SIGABRT, afpkt_sigint_handler); + + struct sockaddr_ll sk_addr = { + .sll_family = AF_PACKET, + .sll_protocol = htons(ETH_P_8021Q), + .sll_halen = ETH_ALEN, + }; + int sockfd; + + switch (opt.mode) { + case MODE_TX: + ret = init_tx_socket(&opt, &sockfd, &sk_addr); + if (!ret) + perror("init_tx_socket failed"); + + if (!opt.enable_txtime) + afpkt_send_thread(&opt, &sockfd, &sk_addr); + else + afpkt_send_thread_etf(&opt, &sockfd, &sk_addr); + + break; + case MODE_RX: + /* WORKAROUND: receive only 0xb62c ETH UADP header packets + * due to a bug where IEEE 1722 (ETH_P_TSN) packets are + * always steered into RX Q0 regardless of its VLAN + * priority + */ + ret = init_rx_socket(0xb62c, &sockfd, opt.ifname); + if (ret != 0) + perror("initrx_socket failed"); + + while (!halt_tx_sig) + afpkt_recv_pkt(sockfd, &opt); + + close(sockfd); + break; + case MODE_FWD: /* FALLTHRU */ + default: + exit_with_error("Invalid AF_XDP mode: Please specify -t, -r, or -f."); + break; + } + close(sockfd); + break; + case MODE_AFXDP: + + init_xdp_socket(&opt); /* Same for all modes */ + + if (!opt.xsk) + afxdp_exit_with_error(EXIT_FAILURE); + + signal(SIGINT, afxdp_sigint_handler); + signal(SIGTERM, afxdp_sigint_handler); + signal(SIGABRT, afxdp_sigint_handler); + + switch (opt.mode) { + case MODE_TX: + afxdp_send_thread(&opt); + + break; + case MODE_RX: + while (!halt_tx_sig) + afxdp_recv_pkt(opt.xsk, buff); + break; + case MODE_FWD: + memset(fds, 0, sizeof(fds)); + fds[0].fd = xsk_socket__fd(opt.xsk->xskfd); + fds[0].events = POLLOUT | POLLIN; + while (!halt_tx_sig) { + if (opt.enable_poll) { + ret = poll(fds, 1, opt.poll_timeout); + if (ret <= 0) + continue; + } + afxdp_fwd_pkt(opt.xsk, fds, &opt); + } + break; + case MODE_BMR: + ret = pthread_create(&thread1, NULL, afxdp_send_thread, &opt); + if (ret) { + fprintf(stderr, "pthread_create failed\n"); + break; + } + while (!halt_tx_sig) { + afxdp_recv_pkt(opt.xsk, buff); + } + + ret = pthread_join(thread1, NULL); + if (ret) + fprintf(stderr, "pthread_join returned %d", ret); + + break; + default: + exit_with_error("Invalid AF_XDP mode: Please specify -t, -r, -f or -b."); + break; + } + /* Close XDP Application */ + xdpsock_cleanup(); + break; + default: + exit_with_error("Invalid socket type: Please specify -X or -P."); + break; + } + + return 0; +} diff --git a/src/txrx.h b/src/txrx.h new file mode 100644 index 0000000..4332e78 --- /dev/null +++ b/src/txrx.h @@ -0,0 +1,162 @@ +/****************************************************************************** + * + * Copyright (c) 2020, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *****************************************************************************/ +#ifndef TXRX_HEADER +#define TXRX_HEADER + +#include +#include +#include +#include + +#define NSEC_PER_SEC 1000000000L +#define ETH_VLAN_HDR_SZ (18) +#define IP_HDR_SZ (20) +#define UDP_HDR_SZ (8) + +#define MODE_INVALID 99 +#define MODE_AFXDP 1 +#define MODE_AFPKT 2 + +#define MODE_TX 0 +#define MODE_RX 1 +#define MODE_FWD 2 +#define MODE_BMR 3 + +#define XDP_MODE_SKB_COPY 0 +#define XDP_MODE_NATIVE_COPY 1 +#define XDP_MODE_ZERO_COPY 2 + +#define exit_with_error(s) {fprintf(stderr, "Error: %s\n", s); exit(EXIT_FAILURE);} + +extern unsigned char src_mac_addr[]; +extern unsigned char dst_mac_addr[]; +extern unsigned char src_ip_addr[]; +extern unsigned char dst_ip_addr[]; + +struct custom_payload { + uint32_t tx_queue; + uint32_t seq; + uint64_t tx_timestampA; + uint64_t rx_timestampD; +}; + +/* Where we keep and track umem descriptor counters */ +struct pkt_buffer { + struct xsk_ring_prod rx_fill_ring; + struct xsk_ring_cons tx_comp_ring; + struct xsk_umem *umem; + void *buffer; +}; + +struct xsk_opt { + /* Note: XSK binds to queue instead of ports, 1 XSK limited to 1 queue */ + uint8_t queue; + uint32_t xdp_flags; + uint32_t xdp_bind_flags; + + uint16_t frame_size; //"Maximum" packet size, + uint16_t frames_per_ring; //May be bounded by hardware? +}; + +struct xsk_info { + struct xsk_socket *xskfd; + struct xsk_ring_cons rx_ring; //User process managed rings + struct xsk_ring_prod tx_ring; // do not access directly + uint32_t bpf_prog_id; + + uint32_t cur_tx; //to track current addr (pkt count * frame_size) + uint32_t cur_rx; + + struct pkt_buffer* pktbuff; //UMEM and rings + + /* TODO: Some per-XDP socket statistics */ + uint64_t rx_npkts; + uint64_t tx_npkts; + uint64_t prev_rx_npkts; + uint64_t prev_tx_npkts; + uint32_t outstanding_tx; +}; + +struct user_opt { + uint8_t mode; //App mode: TX/RX/FWD + uint8_t socket_mode; //af_packet or af_xdp + + char *ifname; + uint32_t ifindex; + clockid_t clkid; + int enable_hwts; + + /* TX control */ + uint32_t socket_prio; + uint8_t vlan_prio; + uint32_t packet_size; + uint32_t frames_to_send; + uint32_t interval_ns; //Cycle time or time between packets + uint32_t offset_ns; //TXTIME transmission target offset from 0th second + uint32_t early_offset_ns; //TXTIME early offset before transmission + + /* XDP-specific */ + struct xsk_info *xsk; //XDP-socket and ring information + struct xsk_opt x_opt; //XDP-specific mandatory user params + + /* XDP-specific options*/ + uint8_t xdp_mode; //XDP mode: skb/nc/zc + uint8_t enable_poll; //XDP poll mode when sending/receiving + //TODO: need wake up & unaligned chunk + + /* Currently for txrx-afxdp only. */ + uint8_t enable_txtime; + bool need_wakeup; + uint32_t poll_timeout; +}; + +/* Struct for VLAN packets with 1722 header */ +typedef struct __attribute__ ((packed)) { + /* Ethernet */ + uint8_t dst_mac[6]; + uint8_t src_mac[6]; + /* VLAN */ + uint16_t vlan_hdr; + uint8_t vlan_prio; + uint8_t vlan_id; + /* Header */ + uint16_t eth_hdr; + /* Payload */ + void *payload; +} tsn_packet; + +uint64_t get_time_nanosec(clockid_t clkid); +uint64_t get_time_sec(clockid_t clkid); +void setup_tsn_vlan_packet(struct user_opt *opt, tsn_packet *pkt); + +#endif