netvm: enable on the new fw baseline.

1. fix uarti passthrough for nvidia Jetpack 6 2. disable pci_acs_override patch 0001-pci-add-pci_acs_override-for-pci-passthrough.patch not needed for Jetpack 36.4.3. PCI passthrough on NX working ok without this patch. 3. add 8 sec delay for nvidia pci passthrough - Add 8 seconds delay to wait for PCI devices to get full enumerated - Remove deprecated kernel parameters for PCI passthrough 4. disable uarti passthrough - The uarti passthrough is currently broken due we need to add a fixed device tree to the netvm that affects it normal behavior. Signed-off-by: Juan Pablo Ruiz <[email protected]>
tiiuae · Feb 1, 2025 · 88b58d2 · 88b58d2
1 parent 8c1d3da
commit 88b58d2
Show file tree

Hide file tree

Showing 7 changed files with 452 additions and 196 deletions.
diff --git a/...eference/hardware/jetpack-microvm/0001-pci-add-pci_acs_override-for-pci-passthrough.patch b/...eference/hardware/jetpack-microvm/0001-pci-add-pci_acs_override-for-pci-passthrough.patch
diff --git a/modules/reference/hardware/jetpack-microvm/nx-netvm-ethernet-pci-passthrough.nix b/modules/reference/hardware/jetpack-microvm/nx-netvm-ethernet-pci-passthrough.nix
@@ -19,19 +19,8 @@ in
             path = "0008:01:00.0";
           }
         ];
-        microvm.kernelParams = [
-          "pci=nomsi"
-          "pcie_acs_override=downstream,multifunction"
-        ];
-      }
-    ];
-
-    boot.kernelPatches = [
-      {
-        name = "nx-pci-passthrough-patch";
-        # This patch uses Alex Williamson's patch for enabling overrides for missing ACS capabilities on pci
-        # bus which could be accessed from following link: https://lkml.org/lkml/2013/5/30/513
-        patch = ./0001-pci-add-pci_acs_override-for-pci-passthrough.patch;
+        # Add 8 seconds delay to wait for PCI devices to get full enumerated
+        microvm.preStart = "/bin/sh -c 'sleep 8'";
       }
     ];
 

diff --git a/...ce/hardware/jetpack/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/default.nix b/...ce/hardware/jetpack/nvidia-jetson-orin/virtualization/common/bpmp-virt-common/default.nix
@@ -38,10 +38,10 @@ in
           TEGRA_BPMP_HOST_PROXY = lib.mkDefault no;
         };
       }
-      # {
-      #   name = "Vfio_platform Reset Required False";
-      #   patch = ./patches/0002-vfio_platform-reset-required-false.patch;
-      # }
+      {
+        name = "Vfio_platform Reset Required False";
+        patch = ./patches/0002-vfio_platform-reset-required-false.patch;
+      }
       # {
       #   name = "Bpmp Support Virtualization";
       #   patch = ./patches/0003-bpmp-support-bpmp-virt.patch;

diff --git a/...e/hardware/jetpack/nvidia-jetson-orin/virtualization/passthrough/uarti-net-vm/default.nix b/...e/hardware/jetpack/nvidia-jetson-orin/virtualization/passthrough/uarti-net-vm/default.nix
@@ -8,6 +8,32 @@
 }:
 let
   cfg = config.ghaf.hardware.nvidia.passthroughs.uarti_net_vm;
+
+  # Derivation to build the GPU-VM guest device tree
+  netvm-dtb = pkgs.stdenv.mkDerivation {
+    name = "netvm-dtb";
+    phases = [ "unpackPhase" "buildPhase" "installPhase" ];
+    src = ./tegra234-netvm.dts;
+    nativeBuildInputs = with pkgs; [
+      dtc
+    ];
+    unpackPhase = ''
+      cp $src ./tegra234-netvm.dts
+    '';
+    buildPhase = ''
+      $CC -E -nostdinc \
+        -I${config.boot.kernelPackages.nvidia-modules.src}/hardware/nvidia/t23x/nv-public/include/nvidia-oot \
+        -I${config.boot.kernelPackages.nvidia-modules.src}/hardware/nvidia/t23x/nv-public/include/kernel \
+        -undef -D__DTS__ \
+        -x assembler-with-cpp \
+        tegra234-netvm.dts > preprocessed.dts
+      dtc -I dts -O dtb -o tegra234-netvm.dtb preprocessed.dts
+    '';
+    installPhase = ''
+      mkdir -p $out
+      cp tegra234-netvm.dtb $out/
+    '';
+  };
 in
 {
   options.ghaf.hardware.nvidia.passthroughs.uarti_net_vm.enable = lib.mkOption {
@@ -19,7 +45,7 @@ in
   config = lib.mkIf cfg.enable {
     services.udev.extraRules = ''
       # Make group kvm all devices that bind to vfio in iommu group 59
-      SUBSYSTEM=="vfio",KERNEL=="59",GROUP="kvm"
+      SUBSYSTEM=="vfio",GROUP="kvm"
     '';
     ghaf.hardware.nvidia.virtualization.enable = true;
 
@@ -34,7 +60,7 @@ in
             extraArgs = [
               # Add custom dtb to Net-VM with 31d0000.serial in platform devices
               "-dtb"
-              "${config.hardware.deviceTree.package}/tegra234-p3701-ghaf-net-vm.dtb"
+              "${netvm-dtb.out}/tegra234-netvm.dtb" 
               # Add UARTI (31d0000.serial) as passtrhough device
               "-device"
               "vfio-platform,host=31d0000.serial"
@@ -50,13 +76,6 @@ in
     # Make sure that Net-VM runs after the binding services are enabled
     systemd.services."microvm@net-vm".after = [ "bindSerial31d0000.service" ];
 
-    boot.kernelPatches = [
-      # {
-      #   name = "Add Net-VM device tree with UARTI in platform devices";
-      #   patch = ./patches/net_vm_dtb_with_uarti.patch;
-      # }
-    ];
-
     systemd.services.bindSerial31d0000 = {
       description = "Bind UARTI to the vfio-platform driver";
       wantedBy = [ "multi-user.target" ];
@@ -75,16 +94,11 @@ in
     # Enable hardware.deviceTree for handle host dtb overlays
     hardware.deviceTree.enable = true;
 
-    # Apply the device tree overlay only to tegra234-p3701-host-passthrough.dtb
-    # hardware.deviceTree.overlays = [
-    #   {
-    #     name = "uarti_pt_host_overlay";
-    #     dtsFile = ./uarti_pt_host_overlay.dts;
-
-    #     # Apply overlay only to host passthrough device tree
-    #     # TODO: make this avaliable if PCI passthrough is disabled
-    #     filter = "tegra234-p3701-host-passthrough.dtb";
-    #   }
-    # ];
+    hardware.deviceTree.overlays = [
+      {
+        name = "uarti_pt_host_overlay";
+        dtsFile = ./uarti_pt_host_overlay.dts;
+      }
+    ];
   };
 }