Capnproto messages, async to syncmetadata

baremetal/start.sh

@@ -14,5 +14,9 @@ sudo rm /ips.txt
 # cargo build --release
 # cd ..
 # cd ..
+
+#CHANGE THIS ACCORDING TO THE NAME OF THE NETWORK DEVICE
+networkdevice="eth0"
+
 sudo cp libTCAL.so /usr/local/bin/libTCAL.so
-sudo ./emulationcore $1 communicationmanager eth0 baremetal
+sudo ./emulationcore $1 communicationmanager $networkdevice baremetal

dockerfiles/Dashboard

@@ -34,5 +34,3 @@ RUN pip3 --no-cache-dir install wheel dnspython flask docker kubernetes netiface
 
 WORKDIR /Kollaps/
 ENTRYPOINT ["python3","kollaps/Dashboard.py","baremetal/topology.xml"]
-
-#docker run -d  --network host dashboard:2.0

dockerfiles/DeploymentGenerator

@@ -23,10 +23,9 @@ ADD ./setup.py /Kollaps/setup.py
 
 ADD /kube/ /root/.kube
 
-RUN pip3 --no-cache-dir install wheel dnspython flask docker kubernetes netifaces netaddr pyroute2 && \
+RUN pip3 --no-cache-dir install wheel dnspython flask docker kubernetes netifaces netaddr pyroute2 requests==2.31.0 && \
     pip3 --no-cache-dir wheel --no-deps -w /Kollaps /Kollaps && \
     pip3 --no-cache-dir install /Kollaps/kollaps-2.0-py3-none-any.whl && \
     rm -rf /Kollaps && \
     mkdir /result && \
-    pip3 --no-cache-dir uninstall -y setuptools wheel pip && \
-    apt-get remove -y gcc make flex bison 
+    pip3 --no-cache-dir uninstall -y setuptools wheel pip

dockerfiles/Kollaps

@@ -3,10 +3,16 @@ FROM ubuntu:22.04
 WORKDIR /
 
 # Location of netem distribution files on archlinux
-
 ARG openssl_arch=linux-x86_64
+
 ENV DEBIAN_FRONTEND=noninteractive \
-    TC_LIB_DIR=/usr/lib/tc/
+    TC_LIB_DIR=/usr/lib/tc/ \
+    RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    KERNEL_VERSION=6.2.0-37-generic \
+    RUST_BACKTRACE=1 \
+    PIP_ROOT_USER_ACTION=ignore
     #JEMALLOC_SYS_WITH_MALLOC_CONF=background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true
 
 RUN CONFIG_BPF=y \
@@ -38,6 +44,7 @@ RUN apt-get update \
     pkgconf \
     python3 \
     python3-pip \
+    vim \
     && apt-get clean -y
 
 ADD ./pid1 /Kollaps/pid1
@@ -49,16 +56,15 @@ ADD ./setup.py /Kollaps/setup.py
 RUN make -C /Kollaps/pid1 && \
     cp /Kollaps/pid1/pid1 /usr/bin/pid1 && \
     make -C /Kollaps/kollaps/TCAL -j8 && \
-    pip3 --no-cache-dir install wheel dnspython flask docker kubernetes netifaces openssh_wrapper netaddr && \
+    pip3 --no-cache-dir install wheel dnspython flask docker kubernetes netifaces openssh_wrapper netaddr requests==2.31.0 && \
     pip3 --no-cache-dir wheel --no-deps -w /Kollaps /Kollaps && \
     pip3 --no-cache-dir install /Kollaps/kollaps-2.0-py3-none-any.whl && \
     cp /Kollaps/kollaps/TCAL/libTCAL.so /usr/local/bin/libTCAL.so && \
     cp -R /Kollaps/kollaps/rustbinaries/communicationmanager /usr/bin/communicationmanager  && \
     cp -R /Kollaps/kollaps/rustbinaries/emulationcore /usr/bin/emulationcore  && \
     cp /Kollaps/kollaps/rustbinaries/libcommunicationcore.so /usr/local/bin/libcommunicationcore.so && \   
     rm -rf /Kollaps && \
-    pip3 --no-cache-dir uninstall -y setuptools wheel pip3 && \
-    apt-get remove -y gcc make flex bison
+    pip3 --no-cache-dir uninstall -y setuptools wheel pip3
 
 WORKDIR /
 

dockerfiles/Oldredbpf

@@ -0,0 +1,51 @@
+FROM ubuntu:22.04
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    DEBIAN_FRONTEND=noninteractive \
+    KERNEL_VERSION=6.1.0-1028-oem
+
+RUN apt-get -y update
+
+RUN apt-get -y install \
+       wget \
+       build-essential \
+       software-properties-common \
+       lsb-release \
+       libelf-dev \
+       linux-headers-generic \
+       pkg-config \
+       curl \
+       linux-headers-$(uname -r)
+
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 13 && rm -f ./llvm.sh
+# llvm-config-13 --version | grep 13
+
+
+RUN curl https://sh.rustup.rs -sSf > rustup.sh \
+    && sh rustup.sh -y \
+          --default-toolchain stable \
+          --profile minimal \
+          --no-modify-path \
+    && rustup component add rustfmt \
+    && rustup target add x86_64-unknown-linux-musl \
+    && rustup toolchain add nightly --profile minimal \
+    && rustup --version \
+    && cargo --version \
+    && rustc --version \
+    && cargo install bindgen-cli \
+    && rm -f rustup.sh
+
+RUN cargo install cargo-bpf --no-default-features --features=llvm13,command-line
+
+RUN mkdir /monitor
+ADD ./kollaps/emulationcore/monitor /monitor
+
+RUN rustup install 1.56
+RUN rustup default 1.56
+
+WORKDIR /monitor
+
+RUN cargo bpf build
+
+ENTRYPOINT ["/bin/bash"]

kollaps/Dashboard.py

@@ -70,6 +70,8 @@ class DashboardState:
     topology_file = ""
     kollaps_folder = ""
     interact_button_pressed = 0
+    links = OrderedDict()
+    link_error = OrderedDict()
 
 
 class Host:
@@ -149,9 +151,24 @@ def flows():
             answer = render_template('flows_baremetal.html', flows=DashboardState.flows, graph=DashboardState.graph)
         else:
             answer = render_template('flows.html', flows=DashboardState.flows, graph=DashboardState.graph)
-            #DashboardState.flows.clear()
+            DashboardState.flows.clear()
         return answer
 
+@app.route('/links_state')
+def links_state():
+    with DashboardState.lock:
+        for link in DashboardState.links:
+            bandwidth = sum(DashboardState.links[link].values())
+            maximum_bandwidth = DashboardState.graph.links[link].bandwidth_bps
+            DashboardState.link_error[link] = ((maximum_bandwidth-bandwidth)/maximum_bandwidth)*100
+            print("max_bw: " + str(maximum_bandwidth) + " and " + " bw" + str(bandwidth) + " in link " + str(link))
+        if DashboardState.mode == "container":
+            answer = render_template('links_state.html', link_error=DashboardState.link_error, graph=DashboardState.graph)
+        for link in DashboardState.links:
+            DashboardState.links[link] = {}
+        return answer
+
+
 @app.route('/graph')
 def graph():
     if DashboardState.mode == "baremetal":
@@ -212,6 +229,20 @@ def initialize():
     with DashboardState.lock:
         DashboardState.initialized = True
 
+
+def collect_flow(bandwidth, links):
+    key = str(links[0]) + ":" + str(links[-1])
+    with DashboardState.lock:
+        #print("Received from " + str(DashboardState.graph.links[links[0]].source.name) + " to " + str(DashboardState.graph.links[links[-1]].destination.name),"to",str(bandwidth))
+        DashboardState.flows[key] = (links[0], links[-1], int(bandwidth/1000))
+        for link in links:
+            DashboardState.links[link][key] = bandwidth
+            #print("ADDED BW ",bandwidth," to ", link)
+
+
+    return True
+
+
 def stopExperiment():
     with DashboardState.lock:
         if DashboardState.mode =="baremetal":
@@ -323,6 +354,8 @@ def startExperiment():
 
 
 def resolve_hostnames():
+
+
     experimentUUID = environ.get('KOLLAPS_UUID', '')
 
     orchestrator = getenv('KOLLAPS_ORCHESTRATOR', 'swarm')
@@ -402,7 +435,6 @@ def resolve_hostnames():
                     DashboardState.hosts[host].ip = ips[i]
                     DashboardState.hosts[host].status = 'Pending'
 
-    start_rust()
 
 
 def start_rust():
@@ -413,17 +445,20 @@ def start_rust():
         libcommunicationcore.start(CONTAINER.id,"dashboard",0,link_count)
 
 
-    if link_count <= BYTE_LIMIT:
-        print_message("Started reading with u8")
-        libcommunicationcore.start_polling_u8()
+    # if link_count <= BYTE_LIMIT:
+    #     print_message("Started reading with u8")
+    #     libcommunicationcore.start_polling_u8()
 
-    else:
-        print_message("Started reading with u16")
-        libcommunicationcore.start_polling_u16()
+    # else:
+        # print_message("Started reading with u16")
+        # libcommunicationcore.start_polling_u16()
+
+    libcommunicationcore.start_polling_u16()
 
     libcommunicationcore.register_communicationmanager(RustComms(collect_flow))
 
 def query_until_ready():
+    start_rust()
     resolve_hostnames()
     print_named("Dashboard", "resolved all hostnames.")
     pending_nodes = []
@@ -434,7 +469,6 @@ def query_until_ready():
         pending_nodes.append(host)
     while pending_nodes:
         host = pending_nodes.pop()
-        sleep(0.05)
         try:
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             #s.settimeout(2)
@@ -502,11 +536,6 @@ def controller_ready():
     return
 
 
-def collect_flow(bandwidth, links):
-    key = str(links[0]) + ":" + str(links[-1])
-    with DashboardState.lock:
-        DashboardState.flows[key] = (links[0], links[-1], int(bandwidth/1000))
-    return True
 
 def add_dashboard_id(id):
         file= open("/tmp/topoinfodashboard", "a")
@@ -561,8 +590,11 @@ def container_deployment():
                 if host.supervisor:
                     continue
                 DashboardState.hosts[host] = Host(host.name, host.machinename)
-
+    
     DashboardState.graph = graph
+
+    for link in graph.links:
+        DashboardState.links[link.index] = {}
 
     if getenv('RUNTIME_EMULATION', 'true') != 'false':
         startup_thread = Thread(target=query_until_ready)

kollaps/Kollapslib/ThunderStorm/Parser.py

@@ -240,7 +240,7 @@ def p_image_declaration(p):
         p[0] = p[1] + [p[2]]
 
 def p_ip_declaration(p):
-    '''ip_declaration : SERVICE ID IP EQ INTEGER ID ID ID ID
+    '''ip_declaration : SERVICE ID IP EQ INipTEGER ID ID ID ID
                       | ip_declaration'''
     node = BaremetalNodeAuxDeclaration()
     setattr(node,"name",p[2])

kollaps/Kollapslib/XMLGraphParser.py

@@ -145,66 +145,85 @@ def parse_links(self, root):
             if 'drop' in link.attrib:
                 drop = link.attrib['drop']
 
-            bidirectional = ('download' in link.attrib)
+            bidirectional = True
 
+            has_download = ('download' in link.attrib)
+            has_upload = ('upload' in link.attrib)
+
+            upload = 0
+            download = 0
+
+            if has_upload:
+                upload = link.attrib['upload']
+
+            if has_download:
+                download = link.attrib['download']
+
+            if has_upload and not has_download:
+                download = upload
+
+            if not has_upload and has_download:
+                upload = download
+
+
             both_shared = (source_nodes[0].shared_link and destination_nodes[0].shared_link)
             if both_shared:
                 src_meta_bridge = self.create_meta_bridge()
 
                 dst_meta_bridge = self.create_meta_bridge()
                 # create a link between both meta bridges
                 self.graph.new_link(src_meta_bridge, dst_meta_bridge, link.attrib['latency'],
-                                    jitter, drop, link.attrib['upload'], network)
+                                    jitter, drop, upload, network)
                 if bidirectional:
                     self.graph.new_link(dst_meta_bridge, src_meta_bridge, link.attrib['latency'],
-                                    jitter, drop, link.attrib['download'], network)
+                                    jitter, drop, download, network)
                 # connect source to src meta bridge
                 self.graph.new_link(link.attrib['origin'], src_meta_bridge, 0,
-                                    0, 0.0, link.attrib['upload'], network)
+                                    0, 0.0, upload, network)
                 if bidirectional:
                     self.graph.new_link(src_meta_bridge, link.attrib['origin'], 0,
-                                    0, 0.0, link.attrib['download'], network)
+                                    0, 0.0, download, network)
                 # connect destination to dst meta bridge
                 self.graph.new_link(dst_meta_bridge, link.attrib['dest'], 0,
-                                    0, 0.0, link.attrib['upload'], network)
+                                    0, 0.0, upload, network)
                 if bidirectional:
                     self.graph.new_link(link.attrib['dest'], dst_meta_bridge, 0,
-                                    0, 0.0, link.attrib['download'], network)
+                                    0, 0.0, download, network)
             elif source_nodes[0].shared_link:
                 meta_bridge = self.create_meta_bridge()
                 # create a link between meta bridge and destination
                 self.graph.new_link(meta_bridge, link.attrib['dest'], link.attrib['latency'],
-                                    jitter, drop, link.attrib['upload'], network)
+                                    jitter, drop, upload, network)
                 if bidirectional:
                     self.graph.new_link(link.attrib['dest'], meta_bridge, link.attrib['latency'],
-                                    jitter, drop, link.attrib['download'], network)
+                                    jitter, drop, download, network)
                 # connect origin to meta bridge
                 self.graph.new_link(link.attrib['origin'], meta_bridge, 0,
-                                    0, 0.0, link.attrib['upload'], network)
+                                    0, 0.0, upload, network)
                 if bidirectional:
                     self.graph.new_link(meta_bridge, link.attrib['origin'], 0,
-                                    0, 0.0, link.attrib['download'], network)
+                                    0, 0.0, download, network)
             elif destination_nodes[0].shared_link:
                 meta_bridge = self.create_meta_bridge()
                 # create a link between origin and meta_bridge
                 self.graph.new_link(link.attrib['origin'], meta_bridge, link.attrib['latency'],
-                                    jitter, drop, link.attrib['upload'], network)
+                                    jitter, drop, upload, network)
                 if bidirectional:
                     self.graph.new_link(meta_bridge, link.attrib['origin'], link.attrib['latency'],
-                                    jitter, drop, link.attrib['download'], network)
+                                    jitter, drop, download, network)
                 # connect meta bridge to destination
                 self.graph.new_link(meta_bridge, link.attrib['dest'], 0,
-                                    0, 0.0, link.attrib['upload'], network)
+                                    0, 0.0, upload, network)
                 if bidirectional:
                     self.graph.new_link(link.attrib['dest'], meta_bridge, 0,
-                                    0, 0.0, link.attrib['download'], network)
+                                    0, 0.0, download, network)
             else:
                 # Regular case create a link between origin and destination
                 self.graph.new_link(link.attrib['origin'], link.attrib['dest'], link.attrib['latency'],
-                                jitter, drop, link.attrib['upload'], network)
+                                jitter, drop, upload, network)
                 if bidirectional:
                     self.graph.new_link(link.attrib['dest'], link.attrib['origin'], link.attrib['latency'],
-                                jitter, drop, link.attrib['download'], network)
+                                jitter, drop, download, network)
 
     def calulate_required_replicas(self, service, hardcoded_count, root, reuse):
         dynamic = None
@@ -323,6 +342,9 @@ def fill_graph(self):
         bridges = None
         links = None
         for child in root:
+
+            if child.tag == 'config':
+                continue
             if child.tag == 'services':
                 if services is not None:
                     print_and_fail("Only one <services> block is allowed.")

kollaps/Kollapslib/bootstrapping/SwarmBootstrapper.py

@@ -172,6 +172,10 @@ def bootstrap_app_container(self, container):
                     "-t", str(pid),
                     "-n", "/usr/bin/emulationcore", str(container_id), str(pid),"docker"]
 
+            # cmd = ["cd /Kollaps/kollaps/emulationcore/","&&","nsenter",
+            #         "-t", str(pid),
+            #         "-n", "cargo flamegraph --", str(container_id), str(pid),"docker"]
+
             emucore_instance = Popen(cmd)
 
             self.instance_count += 1