Skip to content

Commit

Permalink
pci-passthrough: move dts from patches to overlays
Browse files Browse the repository at this point in the history 
- Create dts overlays to apply device tree changes for PCI
  passthrough instead of patches over the kernel dts source code
- Re-enable ethernet and wifi passthrough for agx and nx

Signed-off-by: Juan Pablo Ruiz <[email protected]>
  • Loading branch information
@jpruiz84 @brianmcgillion
jpruiz84 authored and brianmcgillion committed Feb 1, 2025
1 parent 707a0f5 commit 8c1d3da
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 90 deletions.
60 changes: 21 additions & 39 deletions ...ack-microvm/pci-passthrough-nx-test.patch → ...ci_acs_override-for-pci-passthrough.patch
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,20 @@
From 5899496e787686c0983c3e9a6b96d5a4edcb07cb Mon Sep 17 00:00:00 2001
From: Juan Pablo Ruiz <[email protected]>
Date: Wed, 29 Jan 2025 10:06:38 +0200
Subject: [PATCH 1/1] pci: add pci_acs_override for pci passthrough

Co-authored-by: Emrah Billur <[email protected]>
Signed-off-by: Juan Pablo Ruiz <[email protected]>
---
.../admin-guide/kernel-parameters.txt | 8 ++
drivers/pci/quirks.c | 101 ++++++++++++++++++
2 files changed, 109 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index d00618967854..a7b459f69e33 100644
index d83a3f47e200..06828b0d77cf 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3641,6 +3641,14 @@
@@ -4304,6 +4304,14 @@
nomsi [MSI] If the PCI_MSI kernel config parameter is
enabled, this kernel boot option can be used to
disable the use of MSI interrupts system-wide.
Expand All @@ -18,10 +30,10 @@ index d00618967854..a7b459f69e33 100644
Safety option to keep boot IRQs enabled. This
should never be necessary.
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index a2e4b5f..fe712d6 100644
index 54061b65a2b7..f4e5904d8a68 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3589,6 +3589,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev)
@@ -3740,6 +3740,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev)
dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET;
}

Expand Down Expand Up @@ -128,44 +140,14 @@ index a2e4b5f..fe712d6 100644
/*
* Some NVIDIA GPU devices do not work with bus reset, SBR needs to be
* prevented for those affected devices.
@@ -5000,5 +5100,6 @@ static const struct pci_dev_acs_enabled {
@@ -5157,6 +5257,7 @@ static const struct pci_dev_acs_enabled {
{ PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs },
/* Wangxun nics */
{ PCI_VENDOR_ID_WANGXUN, PCI_ANY_ID, pci_quirk_wangxun_nic_acs },
+ { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides },
{ 0 }
};
diff --git a/nvidia/platform/t23x/p3768/kernel-dts/Makefile b/nvidia/platform/t23x/p3768/kernel-dts/Makefile
index f306119fe8a3..3034a22ca7ed 100644
--- a/nvidia/platform/t23x/p3768/kernel-dts/Makefile
+++ b/nvidia/platform/t23x/p3768/kernel-dts/Makefile
@@ -23,6 +23,9 @@ dtb-$(BUILD_ENABLE) += tegra234-p3767-0000-as-p3767-0004-p3768-0000-a0.dtb
dtb-$(BUILD_ENABLE) += tegra234-p3767-0001-p3768-0000-a0.dtb
dtb-$(BUILD_ENABLE) += tegra234-p3767-0003-p3768-0000-a0.dtb
dtb-$(BUILD_ENABLE) += tegra234-p3767-0004-p3768-0000-a0.dtb
+
+dtb-$(BUILD_ENABLE) += tegra234-p3767-host-passthrough.dtb
+
dtbo-$(BUILD_ENABLE) += tegra234-p3767-0000-p3509-a02-adafruit-sph0645lm4h.dtbo
dtbo-$(BUILD_ENABLE) += tegra234-p3767-0000-p3509-a02-adafruit-uda1334a.dtbo
dtbo-$(BUILD_ENABLE) += tegra234-p3767-0000-p3509-a02-csi.dtbo
diff --git a/nvidia/platform/t23x/p3768/kernel-dts/tegra234-p3767-host-passthrough.dts b/nvidia/platform/t23x/p3768/kernel-dts/tegra234-p3767-host-passthrough.dts
new file mode 100644
index 000000000000..7b1c2f6fda7d
--- /dev/null
+++ b/nvidia/platform/t23x/p3768/kernel-dts/tegra234-p3767-host-passthrough.dts
@@ -0,0 +1,14 @@
+/dts-v1/;
+#include "tegra234-p3767-0000-p3509-a02.dts"
+
+/*
+ * Update the pci-e ethernet to be accessible from vfio/guest
+ * The ethernet device is pcie@140a0000 or with name pci_c8_rp
+ */
+&pcie_c8_rp {
+ interconnect-names = "dma-mem", "write";
+ /delete-property/ iommus;
+ /delete-property/ dma-coherent;
+ /delete-property/ msi-parent;
+ /delete-property/ msi-map;
+};

--
2.34.1

29 changes: 29 additions & 0 deletions modules/reference/hardware/jetpack-microvm/agx-ethernet-pci-passthough-overlay.dts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
* SPDX-License-Identifier: CC-BY-SA-4.0
*/

/dts-v1/;
/plugin/;

#include <dt-bindings/clock/tegra234-clock.h>
#include <dt-bindings/reset/tegra234-reset.h>
#include <dt-bindings/power/tegra234-powergate.h>
#include <dt-bindings/memory/tegra234-mc.h>
#include <dt-bindings/interrupt-controller/irq.h>
#include <dt-bindings/interrupt-controller/arm-gic.h>

/ {
overlay-name = "PCI ethernet passthrough";
compatible = "nvidia,p3701-0000";

fragment@0 {
target-path = "/bus@0";
__overlay__ {
pcie@14100000 {
iommus = <>;
};
};
};

};
17 changes: 6 additions & 11 deletions modules/reference/hardware/jetpack-microvm/agx-netvm-wlan-pci-passthrough.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,16 @@ in
}
];

boot.kernelPatches = [
# {
# name = "agx-pci-passthrough-patch";
# patch = ./pci-passthrough-agx-test.patch;
# }
hardware.deviceTree.overlays = [
{
name = "agx-ethernet-pci-passthough-overlay";
dtsFile = ./agx-ethernet-pci-passthough-overlay.dts;
}
];

boot.kernelParams = [
"vfio-pci.ids=10ec:c82f"
"vfio-pci.ids=10ec:c822,10ec:c82f"
"vfio_iommu_type1.allow_unsafe_interrupts=1"
];

hardware.deviceTree = {
enable = true;
name = "tegra234-p3701-host-passthrough.dtb";
};
};
}
93 changes: 93 additions & 0 deletions modules/reference/hardware/jetpack-microvm/nx-ethernet-pci-passthough-overlay.dts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2022-2024 TII (SSRC) and the Ghaf contributors
* SPDX-License-Identifier: CC-BY-SA-4.0
*/

/dts-v1/;
/plugin/;

#include <dt-bindings/clock/tegra234-clock.h>
#include <dt-bindings/reset/tegra234-reset.h>
#include <dt-bindings/power/tegra234-powergate.h>
#include <dt-bindings/memory/tegra234-mc.h>
#include <dt-bindings/interrupt-controller/irq.h>
#include <dt-bindings/interrupt-controller/arm-gic.h>

/ {
overlay-name = "PCI ethernet passthrough";
compatible = "nvidia,p3768-0000+p3767-0000";

fragment@0 {
target-path = "/bus@0";
__overlay__ {
pcie@140a0000 {
status = "disabled";
};

// Created a new PCIe node and commented out dma-coherent because with
// device tree overlays it is not possible to delete properties
pcie_pt@140a0000 {
compatible = "nvidia,tegra234-pcie";
power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CA>;
reg = <0x00 0x140a0000 0x0 0x00020000>, /* appl registers (128K) */
<0x00 0x2a000000 0x0 0x00040000>, /* configuration space (256K) */
<0x00 0x2a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */
<0x00 0x2a080000 0x0 0x00040000>, /* DBI reg space (256K) */
<0x35 0x30000000 0x0 0x10000000>; /* ECAM (256MB) */
reg-names = "appl", "config", "atu_dma", "dbi", "ecam";

#address-cells = <3>;
#size-cells = <2>;
device_type = "pci";

num-viewport = <8>;
linux,pci-domain = <8>;

clocks = <&bpmp TEGRA234_CLK_PEX2_C8_CORE>;
clock-names = "core";

resets = <&bpmp TEGRA234_RESET_PEX2_CORE_8_APB>,
<&bpmp TEGRA234_RESET_PEX2_CORE_8>;
reset-names = "apb", "core";

interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
<GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
interrupt-names = "intr", "msi";

#interrupt-cells = <1>;
interrupt-map-mask = <0 0 0 0>;
interrupt-map = <0 0 0 0 &gic GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;

nvidia,bpmp = <&bpmp 8>;

nvidia,aspm-cmrt-us = <60>;
nvidia,aspm-pwr-on-t-us = <20>;
nvidia,aspm-l0s-entrance-latency-us = <3>;

bus-range = <0x0 0xff>;

ranges = <0x43000000 0x32 0x40000000 0x32 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
<0x02000000 0x0 0x40000000 0x35 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
<0x01000000 0x0 0x2a100000 0x00 0x2a100000 0x0 0x00100000>; /* downstream I/O (1 MB) */

interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE8AR &emc>,
<&mc TEGRA234_MEMORY_CLIENT_PCIE8AW &emc>;
interconnect-names = "dma-mem", "write";
iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE8 0x1000>;
iommu-map-mask = <0x0>;
//dma-coherent;

num-lanes = <2>;

phys = <&p2u_gbe_2>, <&p2u_gbe_3>;
phy-names = "p2u-0", "p2u-1";

vddio-pex-ctl-supply = <&vdd_1v8_ao>;
vpcie3v3-supply = <&vdd_3v3_pcie>;

status = "okay";
};
};
};

};
14 changes: 8 additions & 6 deletions modules/reference/hardware/jetpack-microvm/nx-netvm-ethernet-pci-passthrough.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,20 @@ in
name = "nx-pci-passthrough-patch";
# This patch uses Alex Williamson's patch for enabling overrides for missing ACS capabilities on pci
# bus which could be accessed from following link: https://lkml.org/lkml/2013/5/30/513
patch = ./pci-passthrough-nx-test.patch;
patch = ./0001-pci-add-pci_acs_override-for-pci-passthrough.patch;
}
];

hardware.deviceTree.overlays = [
{
name = "nx-ethernet-pci-passthough-overlay";
dtsFile = ./nx-ethernet-pci-passthough-overlay.dts;
}
];

boot.kernelParams = [
"vfio-pci.ids=10ec:8168"
"vfio_iommu_type1.allow_unsafe_interrupts=1"
];

hardware.deviceTree = {
enable = true;
name = "tegra234-p3767-host-passthrough.dtb";
};
};
}
32 changes: 0 additions & 32 deletions modules/reference/hardware/jetpack-microvm/pci-passthrough-agx-test.patch
View file

This file was deleted.

4 changes: 2 additions & 2 deletions targets/nvidia-jetson-orin/flake-module.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ let
hardware.nvidia.orin = {
enable = true;
somType = som;
agx.enableNetvmWlanPCIPassthrough = false; # som == "agx";
nx.enableNetvmEthernetPCIPassthrough = false; # som == "nx";
agx.enableNetvmWlanPCIPassthrough = som == "agx";
nx.enableNetvmEthernetPCIPassthrough = som == "nx";
};

hardware.nvidia = {
Expand Down

0 comments on commit 8c1d3da

Please sign in to comment.