-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathblog.html
62 lines (62 loc) · 68.6 KB
/
blog.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
<!doctype html>
<html lang="en" dir="ltr" class="blog-wrapper blog-list-page plugin-blog plugin-id-default">
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v2.4.1">
<title data-rh="true">Blog | PhotonLibOS</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://photonlibos.github.io/img/social-card.jpg"><meta data-rh="true" name="twitter:image" content="https://photonlibos.github.io/img/social-card.jpg"><meta data-rh="true" property="og:url" content="https://photonlibos.github.io/blog"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" property="og:title" content="Blog | PhotonLibOS"><meta data-rh="true" name="description" content="Blog"><meta data-rh="true" property="og:description" content="Blog"><meta data-rh="true" name="docusaurus_tag" content="blog_posts_list"><meta data-rh="true" name="docsearch:docusaurus_tag" content="blog_posts_list"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://photonlibos.github.io/blog"><link data-rh="true" rel="alternate" href="https://photonlibos.github.io/cn/blog" hreflang="cn"><link data-rh="true" rel="alternate" href="https://photonlibos.github.io/blog" hreflang="en"><link data-rh="true" rel="alternate" href="https://photonlibos.github.io/blog" hreflang="x-default"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="PhotonLibOS RSS Feed">
<link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="PhotonLibOS Atom Feed"><link rel="stylesheet" href="/assets/css/styles.b9c58f1b.css">
<link rel="preload" href="/assets/js/runtime~main.8ce90755.js" as="script">
<link rel="preload" href="/assets/js/main.65a139a1.js" as="script">
</head>
<body class="navigation-with-keyboard">
<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus">
<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.svg" alt="PhotonLibOS Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.svg" alt="PhotonLibOS Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">PhotonLibOS</b></a><a class="navbar__item navbar__link" href="/docs/category/introduction">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/blog">Blog</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/cn/blog" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="cn">中文</a></li><li><a href="/blog" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><a href="https://github.com/alibaba/PhotonLibOS" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently light mode)" aria-label="Switch between dark and light mode (currently light mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/stackful-coroutine-made-fast">Stackful Coroutine Made Fast</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/photon-dpdk">How to run Photon on top of DPDK</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/blog/thread-local">The thread local variable for coroutines</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/blog/stackful-coroutine-made-fast">Stackful Coroutine Made Fast</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2024-10-14T00:00:00.000Z" itemprop="datePublished">October 14, 2024</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><a href="https://github.com/alibaba/PhotonLibOS/blob/main/doc/static/blog-20241014/Stackful_Coroutine_Made_Fast.pdf" target="_blank" rel="noopener noreferrer">Download pdf</a><iframe src="/blog-20241014/stackful-coroutine-made-fast.html" width="100%" height="30000"></iframe></div><footer class="row docusaurus-mt-lg"></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/blog/photon-dpdk">How to run Photon on top of DPDK</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2023-07-29T00:00:00.000Z" itemprop="datePublished">July 29, 2023</time> · <!-- -->7 min read</div><div class="margin-top--md margin-bottom--sm row"><div class="col col--6 authorCol_Hf19"><div class="avatar margin-bottom--sm"><a href="https://github.com/beef9999" target="_blank" rel="noopener noreferrer" class="avatar__photo-link"><img class="avatar__photo" src="https://github.com/beef9999.png" alt="Bob Chen"></a><div class="avatar__intro" itemprop="author" itemscope="" itemtype="https://schema.org/Person"><div class="avatar__name"><a href="https://github.com/beef9999" target="_blank" rel="noopener noreferrer" itemprop="url"><span itemprop="name">Bob Chen</span></a></div><small class="avatar__subtitle" itemprop="description">Maintainer of PhotonLibOS</small></div></div></div></div></header><div class="markdown" itemprop="articleBody"><p> <!-- --> <!-- -->Since version 0.6, Photon can run on an userspace TCP/IP stack if enabled the <code>INIT_IO_FSTACK_DPDK</code> io engine. </p><p> <!-- --> <a href="https://www.f-stack.org/" target="_blank" rel="noopener noreferrer">F-Stack</a> is an open-source project that has ported the entire <strong>FreeBSD</strong>
network stack on top of <strong>DPDK</strong>, and provided userspace sockets and events API.
We have integrated Photon's coroutine scheduler with F-Stack, and made a busy-polling program more friendly to DPDK
developers than ever before. In terms of performance, the network app has seen the improvement of 20% ~ 40%, compared with
the Linux kernel based on interrupt.</p><p> <!-- --> <!-- -->This article will introduce how to configure SR-IOV on a Mellanox NIC, how to set up F-Stack
and DPDK environment, how to enable the <a href="https://doc.dpdk.org/guides/howto/flow_bifurcation.html" target="_blank" rel="noopener noreferrer">Flow Bifurcation</a>
to filter the specific TCP/IP flow that you only concern, and finally how to run Photon on top of them, in order
to build a high performance net server.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="configure-sr-iov-on-mellanox-connectx-4">Configure SR-IOV on Mellanox ConnectX-4<a href="#configure-sr-iov-on-mellanox-connectx-4" class="hash-link" aria-label="Direct link to Configure SR-IOV on Mellanox ConnectX-4" title="Direct link to Configure SR-IOV on Mellanox ConnectX-4"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="1-enable-iommu">1. Enable IOMMU<a href="#1-enable-iommu" class="hash-link" aria-label="Direct link to 1. Enable IOMMU" title="Direct link to 1. Enable IOMMU"></a></h4><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">CentOS 7</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Debian 10</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token comment" style="color:#999988;font-style:italic"># Edit /etc/default/grub</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Expand GRUB_CMDLINE_LINUX with 'intel_iommu=on iommu=pt pci=realloc'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">grub2-mkconfig </span><span class="token parameter variable" style="color:#36acaa">-o</span><span class="token plain"> /boot/grub2/grub.cfg</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">reboot</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token comment" style="color:#999988;font-style:italic"># Edit /etc/default/grub</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Expand GRUB_CMDLINE_LINUX with 'intel_iommu=on iommu=pt'</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">grub-mkconfig</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-o</span><span class="token plain"> /boot/grub/grub.cfg</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">reboot</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><p>Note the <code>pci=realloc</code> is a work-around solution for CentOS and RHEL.
Without this, kernel would report <code>not enough MMIO resources for SR-IOV</code>,
see this <a href="https://access.redhat.com/solutions/37376" target="_blank" rel="noopener noreferrer">issue</a>.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="2-set-vf-number">2. Set VF number<a href="#2-set-vf-number" class="hash-link" aria-label="Direct link to 2. Set VF number" title="Direct link to 2. Set VF number"></a></h4><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">echo</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">4</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">></span><span class="token plain"> /sys/class/net/eth0/device/sriov_numvfs</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->If you are having an Intel NIC, this step is likely to succeed. However, for the Mellanox one,
it might fail because of the lack of proper mlx driver in your kernel.
Please check the result by typing <code>lspci -nn | grep Ethernet</code> and see if the NICs' virtual function number is correct.</p><p> <!-- --> <!-- -->If succeeded, please jump to the part of 'Install DPDK'.</p><p> <!-- --> <!-- -->If failed, you may need to download the official driver from NVidia.
There are many available releases in <a href="https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/" target="_blank" rel="noopener noreferrer">https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/</a>,
you should choose one that matches to your kernel version and OS version the best.
An improper version might lead to compiling error when building kernel modules later. </p><ul><li>For example, for CentOS 7 and kernel 5.x, you should choose MLNX_OFED_LINUX-5.4-3.6.8.1-rhel7.2-x86_64.tgz</li><li>For Debian 10, it is MLNX_OFED_LINUX-5.8-5.1.1.2-debian10.13-x86_64.tgz</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="3-install-mlnx_ofed-driver">3. Install mlnx_ofed driver<a href="#3-install-mlnx_ofed-driver" class="hash-link" aria-label="Direct link to 3. Install mlnx_ofed driver" title="Direct link to 3. Install mlnx_ofed driver"></a></h4><p> <!-- --> <!-- -->First you need to check your gcc version. It has to be the same one that built your kernel.
Otherwise you will need to upgrade your gcc.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">gcc </span><span class="token parameter variable" style="color:#36acaa">--version</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">cat</span><span class="token plain"> /proc/version</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->Note that the NVidia official doc said we should install 'createrepo', but in CentOS 7,
there are some tiny bugs of its Python scripts. The 'createrepo_c' package will solve this.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">yum </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> python-devel tcl tk elfutils-libelf-devel createrepo_c</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->Because the mlnx_ofed driver has already included rdma packages, to avoid collision,
I decided to remove all rdma-related rpms previously installed in my test machine.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">rpm</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-qa</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">|</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">grep</span><span class="token plain"> rdma</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">rpm</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-e</span><span class="token plain"> </span><span class="token punctuation" style="color:#393A34">..</span><span class="token plain">.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->Build and install the driver and the additional packages.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">cd</span><span class="token plain"> MLNX_OFED_LINUX-5.4-3.6.8.1-rhel7.2-x86_64/</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">./mlnxofedinstall --skip-distro-check --add-kernel-support --without-mlnx-nvme </span><span class="token parameter variable" style="color:#36acaa">--dpdk</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Update initramfs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">dracut </span><span class="token parameter variable" style="color:#36acaa">-f</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># There will be rdma-core, rdma-core-devel, librdmacm and librdmacm-utils.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">rpm</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-qa</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">|</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">grep</span><span class="token plain"> rdma</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->Now we need to restart the server. Be careful, there is a possibility that the interface name
of your NIC might change, for example, from <code>eth0</code> to something like <code>enp3s0f0</code>, where 3 for Bus, 0 for Device,
and 0 for Function, represented in the <code>03:00.0</code> BDF notation. It will incur connection failure
of your server and unable to log in.</p><p> <!-- --> <!-- -->To solve this, your first option is to disable the Consistent Interface Device Naming in Linux,
and then persist the new names by <code>udev rules</code>. See the NVidia docs at
<a href="https://docs.nvidia.com/networking/display/MLNXOFEDv541030/Changes+and+New+Features#ChangesandNewFeatures-CustomerAffectingChanges" target="_blank" rel="noopener noreferrer">1</a>,
<a href="https://enterprise-support.nvidia.com/s/article/howto-change-network-interface-name-in-linux-permanently" target="_blank" rel="noopener noreferrer">2</a>.</p><ol><li>Append <code>GRUB_CMDLINE_LINUX</code> in <code>/etc/default/grub</code> with <code>net.ifnames=0</code></li><li>Create the <code>/etc/udev/rules.d/85-net-persistent-names.rules</code> with the following content</li></ol><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain"># PCI device 15b3:1019 (mlx5_core)</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"># NAME:="some name" , := is used to make sure that device name will be persistent.</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:02:c9:fa:c3:50", ATTR{dev_id}=="0x0", ATTR{type}=="1", KERNEL=="eth*", NAME:="eth0"</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="00:02:c9:fa:c3:51", ATTR{dev_id}=="0x0", ATTR{type}=="1", KERNEL=="eth*", NAME:="eth1"</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->The second option, if you are OK with the new names, you can update the NIC scripts
in <code>/etc/sysconfig/network-scripts/</code> and make them correct.</p><p> <!-- --> <!-- -->Finally, everything get ready, just reboot:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token function" style="color:#d73a49">reboot</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p> <!-- --> <!-- -->After reboot:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token comment" style="color:#999988;font-style:italic"># Start Mellanox Software Tools Service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">mst start</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Show device name and port mapping</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">mst status</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">ibdev2netdev</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Check firmware capabilities</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">mlxconfig </span><span class="token parameter variable" style="color:#36acaa">-d</span><span class="token plain"> /dev/mst/mt4117_pciconf0 query </span><span class="token operator" style="color:#393A34">|</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">grep</span><span class="token plain"> NUM_OF_VFS</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token comment" style="color:#999988;font-style:italic"># Set VF number. Should succeed now</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">echo</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">4</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">></span><span class="token plain"> /sys/class/net/enp3s0f0/device/sriov_numvfs</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">lspci </span><span class="token parameter variable" style="color:#36acaa">-nn</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">|</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">grep</span><span class="token plain"> </span><span class="token string" style="color:#e3116c">'Ethernet controller'</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-dpdk">Install DPDK<a href="#install-dpdk" class="hash-link" aria-label="Direct link to Install DPDK" title="Direct link to Install DPDK"></a></h3><p> <!-- --> <!-- -->The F-Stack version we choose is <a href="https://github.com/F-Stack/f-stack/releases/tag/v1.22.1" target="_blank" rel="noopener noreferrer">1.22.1</a>,
and it has a subdirectory called dpdk that contains the full DPDK 20.11 source code.
Let's start with the DPDK install first.</p><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">CentOS 7</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Debian 10</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">cd</span><span class="token plain"> f-stack-1.22.1/dpdk/</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">yum </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> python3-pip</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">yum </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> pkg-config numactl-devel zlib-devel ninja</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">pip3 </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> meson </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">cd</span><span class="token plain"> f-stack-1.22.1/dpdk/</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">pip3 </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> ninja meson</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">apt</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">install</span><span class="token plain"> pkg-config python3-pyelftools libnuma-dev</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><p>Build and install:</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token assign-left variable" style="color:#36acaa">CONFIG_RTE_LIBRTE_MLX5_PMD</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">y meson </span><span class="token parameter variable" style="color:#36acaa">-Denable_kmods</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">true </span><span class="token parameter variable" style="color:#36acaa">-Dtests</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">false build</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">cd</span><span class="token plain"> build</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">ninja</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">ninja </span><span class="token function" style="color:#d73a49">install</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Allocate 10GB huge-pages</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">echo</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">5120</span><span class="token plain"> </span><span class="token operator" style="color:#393A34">></span><span class="token plain"> /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Attach your PF (with main IP) and one of the VFs (idle) to the poll-mode-driver test</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">./build/app/dpdk-testpmd </span><span class="token parameter variable" style="color:#36acaa">-l</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">0</span><span class="token plain">-3 </span><span class="token parameter variable" style="color:#36acaa">-n</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">4</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-a</span><span class="token plain"> 0000:03:00.0 </span><span class="token parameter variable" style="color:#36acaa">-a</span><span class="token plain"> 0000:03:00.2 -- --nb-cores</span><span class="token operator" style="color:#393A34">=</span><span class="token number" style="color:#36acaa">2</span><span class="token plain"> --flow-isolate-all </span><span class="token parameter variable" style="color:#36acaa">-i</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-a</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: The <code>--flow-isolate-all</code> option is a MUST do. It enables Flow Bifurcation and ensures that all the
undetermined flow will be forwarded to the Linux kernel. Because the default behavior is to drop all packets, so
unless you configure the flow table or enable the <code>--flow-isolate-all</code> option,
your network connection will be lost again ...</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-f-stack">Install F-Stack<a href="#install-f-stack" class="hash-link" aria-label="Direct link to Install F-Stack" title="Direct link to Install F-Stack"></a></h3><p>Let's go back to the parent dir and install F-Stack.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="upgrade-pkg-config">Upgrade pkg-config<a href="#upgrade-pkg-config" class="hash-link" aria-label="Direct link to Upgrade pkg-config" title="Direct link to Upgrade pkg-config"></a></h4><p> <!-- --> <!-- -->The <code>pkg-config</code> command in CentOS 7 is of version 0.27.1, and it has a <a href="https://bugs.freedesktop.org/show_bug.cgi?id=56699" target="_blank" rel="noopener noreferrer">bug</a>
that does not correctly handle gcc's <code>--whole-archive</code> option.
As per F-Stack's document, we can upgrade it to <a href="https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" target="_blank" rel="noopener noreferrer">0.29.2</a>.</p><p> <!-- --> <!-- -->Debian 10 is OK.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="modify-make-scripts">Modify make scripts<a href="#modify-make-scripts" class="hash-link" aria-label="Direct link to Modify make scripts" title="Direct link to Modify make scripts"></a></h4><ol><li>Edit <code>lib/Makefile</code>, comment out <code>DEBUG=...</code>. We want a release build.</li><li>Edit <code>lib/Makefile</code>, enable <code>FF_FLOW_ISOLATE=1</code>. It is the trigger of Flow Bifurcation for TCP. The hardcoded TCP port is 80.</li><li>For CentOS 7, edit <code>mk/kern.mk</code>, add <code>-Wno-error=format-overflow</code> to <code>CWARNFLAGS</code>,
in case a compiler warning being regarded as error. Debian 10 is OK.</li></ol><h4 class="anchor anchorWithStickyNavbar_LWe7" id="build-and-install">Build and install<a href="#build-and-install" class="hash-link" aria-label="Direct link to Build and install" title="Direct link to Build and install"></a></h4><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">CentOS 7</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Debian 10</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">FF_PATH</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">/root/f-stack-1.22.1 </span><span class="token comment" style="color:#999988;font-style:italic"># Change to your own dir</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">REGULAR_PKG_CONFIG_DIR</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">/usr/lib64/pkgconfig/</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">DPDK_PKG_CONFIG_DIR</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">/usr/local/lib64/pkgconfig/</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">PKG_CONFIG_PATH</span><span class="token operator" style="color:#393A34">=</span><span class="token variable" style="color:#36acaa">$(</span><span class="token variable" style="color:#36acaa">pkg-config </span><span class="token variable parameter variable" style="color:#36acaa">--variable</span><span class="token variable operator" style="color:#393A34">=</span><span class="token variable" style="color:#36acaa">pc_path pkg-config</span><span class="token variable" style="color:#36acaa">)</span><span class="token builtin class-name">:</span><span class="token variable" style="color:#36acaa">${REGULAR_PKG_CONFIG_DIR}</span><span class="token builtin class-name">:</span><span class="token variable" style="color:#36acaa">${DPDK_PKG_CONFIG_DIR}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">cd</span><span class="token plain"> f-stack-1.22/lib</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">make</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-j</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">make</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">install</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">export</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">FF_PATH</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">/root/f-stack-1.22.1 </span><span class="token comment" style="color:#999988;font-style:italic"># Change to your own dir</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token builtin class-name">cd</span><span class="token plain"> f-stack-1.22/lib</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">make</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-j</span><span class="token plain"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">make</span><span class="token plain"> </span><span class="token function" style="color:#d73a49">install</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="configurations">Configurations<a href="#configurations" class="hash-link" aria-label="Direct link to Configurations" title="Direct link to Configurations"></a></h4><p> <!-- --> <!-- -->F-Stack has a global config file at <code>/etc/f-stack.conf</code>. We need to make a few changes before running it.</p><ol><li>Change <code>pkt_tx_delay=100</code> to <code>pkt_tx_delay=0</code>. So it will send packets immediately, rather than wait for a while.</li><li>Modify the <code>[port0]</code> section, including <code>addr</code>, <code>netmask</code>, <code>broadcast</code> and <code>gateway</code>. Keep the same to your
test machine, because our DPDK app only needs to have a unique TCP port.</li><li>Add <code>pci_whitelist=03:00.0,03:00.2</code>. As explained above, the first one is your PF with main IP, the other is one of
its idle VFs. The Flow Bifurcation will forward specific TCP flow to VF, while leaving the rest traffic to the PF,
for the Linux kernel.</li></ol><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-photon">Run Photon<a href="#run-photon" class="hash-link" aria-label="Direct link to Run Photon" title="Direct link to Run Photon"></a></h3><p> <!-- --> <!-- -->We have provided a new <a href="https://github.com/alibaba/PhotonLibOS/blob/main/examples/fstack-dpdk/fstack-dpdk-demo.cpp" target="_blank" rel="noopener noreferrer">example</a>.
It looks quite alike the old echo server example, only a few lines of changes, but now the backend becomes DPDK.</p><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token builtin class-name">cd</span><span class="token plain"> PhotonLibOS</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"></span><span class="token function" style="color:#d73a49">git</span><span class="token plain"> checkout release/0.8</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">cmake </span><span class="token parameter variable" style="color:#36acaa">-B</span><span class="token plain"> build </span><span class="token parameter variable" style="color:#36acaa">-D</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">PHOTON_BUILD_TESTING</span><span class="token operator" style="color:#393A34">=</span><span class="token number" style="color:#36acaa">1</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-D</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">PHOTON_ENABLE_FSTACK_DPDK</span><span class="token operator" style="color:#393A34">=</span><span class="token number" style="color:#36acaa">1</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-D</span><span class="token plain"> </span><span class="token assign-left variable" style="color:#36acaa">CMAKE_BUILD_TYPE</span><span class="token operator" style="color:#393A34">=</span><span class="token plain">Release</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">cmake </span><span class="token parameter variable" style="color:#36acaa">--build</span><span class="token plain"> build </span><span class="token parameter variable" style="color:#36acaa">-j</span><span class="token plain"> </span><span class="token number" style="color:#36acaa">32</span><span class="token plain"> </span><span class="token parameter variable" style="color:#36acaa">-t</span><span class="token plain"> fstack-dpdk-demo</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">./build/output/fstack-dpdk-demo</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now you can set up an echo client on another host, and bench this server via port 80.</p></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/dpdk">DPDK</a></li><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/f-stack">F-Stack</a></li></ul></div></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/blog/thread-local">The thread local variable for coroutines</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2023-07-28T00:00:00.000Z" itemprop="datePublished">July 28, 2023</time> · <!-- -->2 min read</div><div class="margin-top--md margin-bottom--sm row"><div class="col col--6 authorCol_Hf19"><div class="avatar margin-bottom--sm"><a href="https://github.com/beef9999" target="_blank" rel="noopener noreferrer" class="avatar__photo-link"><img class="avatar__photo" src="https://github.com/beef9999.png" alt="Bob Chen"></a><div class="avatar__intro" itemprop="author" itemscope="" itemtype="https://schema.org/Person"><div class="avatar__name"><a href="https://github.com/beef9999" target="_blank" rel="noopener noreferrer" itemprop="url"><span itemprop="name">Bob Chen</span></a></div><small class="avatar__subtitle" itemprop="description">Maintainer of PhotonLibOS</small></div></div></div></div></header><div class="markdown" itemprop="articleBody"><p>As we all know, C++11 introduced the <code>thread_local</code> keyword to replace the <code>__thread</code> provided by the compiler,
or the <code>specific key</code> related functions provided by the <code>pthread</code> library.</p><p>Here is a typical example of using thread_local.</p><div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-c++ codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">#include <thread></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">static thread_local int i = 0;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">int main() {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> auto th = std::thread([]{</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> i = 1;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> });</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> th.join();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> assert(i == 0);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Photon begins to support TLS for coroutines since version 0.4.0. Due to some limitations, Photon cannot achieve the
same syntax as <code>thread_local</code>, but implements it in a close way.</p><div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-c++ codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">#include <photon/thread/std-compat.h></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">static photon::thread_local_ptr<int, int> pI(0);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">int main() {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> if (photon::init())</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> abort();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> DEFER(photon::fini());</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> auto th = photon_std::thread([]{</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> *pI = 1;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> });</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> th.join();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> </span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> assert(*pI == 0);</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In this code above, <code>thread_local_ptr</code> is a template class that provides pointer-like operators.
You need to pass the appropriate constructor type to its template parameter, which in this example, is also a int.</p><p>When users access it in different coroutines, they will always get a separate value.</p><p>Below is a more complicated example:</p><div class="language-c++ codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#393A34;--prism-background-color:#f6f8fa"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-c++ codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#393A34"><span class="token plain">class Value {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">public:</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> explicit Value(std::string s) : m_s(std::move(s)) {}</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> size_t size() { return m_s.size(); }</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">private:</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> std::string m_s;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">};</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">class A {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">public:</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> void func();</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">private:</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> static photon::thread_local_ptr<Value, std::string> m_value;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">};</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">static photon::thread_local_ptr<Value, std::string> m_value("123");</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">void A::func() {</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain"> std::cout << "Value size " << m_value->size() << std::endl;</span><br></span><span class="token-line" style="color:#393A34"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><footer class="row docusaurus-mt-lg"><div class="col"><b>Tags:</b><ul class="tags_jXut padding--none margin-left--sm"><li class="tag_QGVx"><a class="tag_zVej tagRegular_sFm0" href="/blog/tags/thread-local">thread-local</a></li></ul></div></footer></article><nav class="pagination-nav" aria-label="Blog list page navigation"></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Blog</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/blog">Blog</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item">
<a href="https://join.slack.com/t/photonlibos/shared_invite/zt-25wauq8g1-iK_oHMrXetcvWNNhIt8Nkg" target="_blank" rel="noreferrer noopener">
<img src="/img/slack.svg" alt="Slack" width="30">
</a>
<a href="https://www.dingtalk.com/download?action=joingroup&code=v1,k1,Q3fyZvf3qFx7aB+9j4FkrK2K45E2g9SiufbbSueS8h0=&_dt_no_comment=1&origin=11" target="_blank" rel="noreferrer noopener" style="padding-left: 10px;">
<img src="/img/dingtalk.svg" alt="Dingtalk" width="30">
</a></li></ul></div><div class="col footer__col"><div class="footer__title">Development</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/alibaba/PhotonLibOS" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2025 PhotonLibOS.</div></div></div></footer></div>
<script src="/assets/js/runtime~main.8ce90755.js"></script>
<script src="/assets/js/main.65a139a1.js"></script>
</body>
</html>