From 9e6d95f2a559d7ed6124bcd29a47d5d8cd7b013c Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Fri, 17 Nov 2023 14:04:26 +0000 Subject: [PATCH 1/2] CP-46140 PVS IPv6 accept both IPv4, IPv6 for PVS Server We want to accept IPv6 addresses for PVS Server in addition to IPv4 which are currently accepted. This patch widens the predicate that enforces the format and updates the documentation in the data model. The address is later written to xenstore where it is picked up by setup-pvs-proxy-rules which will have to respond to addresses that are in IPv6 format. Signed-off-by: Christian Lindig --- ocaml/idl/datamodel.ml | 2 +- ocaml/xapi/helpers.ml | 2 ++ ocaml/xapi/xapi_pvs_server.ml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index f2ea7399946..aa19b1ab347 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -7298,7 +7298,7 @@ module PVS_server = struct ~result:(Ref _pvs_server, "the new PVS server") ~params: [ - (Set String, "addresses", "IPv4 addresses of the server") + (Set String, "addresses", "IPv4/IPv6 addresses of the server") ; (Int, "first_port", "first UDP port accepted by this server") ; (Int, "last_port", "last UDP port accepted by this server") ; (Ref _pvs_site, "site", "PVS site this server is a part of") diff --git a/ocaml/xapi/helpers.ml b/ocaml/xapi/helpers.ml index c9f70e8c8a8..c31ed7895ec 100644 --- a/ocaml/xapi/helpers.ml +++ b/ocaml/xapi/helpers.ml @@ -1097,6 +1097,8 @@ let assert_is_valid_tcp_udp_port_range ~first_port ~first_name ~last_port let is_valid_ip kind address = match (Unixext.domain_of_addr address, kind) with + | Some x, `ipv4or6 when x = Unix.PF_INET || x = Unix.PF_INET6 -> + true | Some x, `ipv4 when x = Unix.PF_INET -> true | Some x, `ipv6 when x = Unix.PF_INET6 -> diff --git a/ocaml/xapi/xapi_pvs_server.ml b/ocaml/xapi/xapi_pvs_server.ml index c124bf52321..dc6c5f59212 100644 --- a/ocaml/xapi/xapi_pvs_server.ml +++ b/ocaml/xapi/xapi_pvs_server.ml @@ -22,7 +22,7 @@ let introduce ~__context ~addresses ~first_port ~last_port ~site = Pool_features.assert_enabled ~__context ~f:Features.PVS_proxy ; Helpers.assert_using_vswitch ~__context ; List.iter - (fun address -> Helpers.assert_is_valid_ip `ipv4 "addresses" address) + (fun address -> Helpers.assert_is_valid_ip `ipv4or6 "addresses" address) addresses ; let current = Db.PVS_server.get_all_records ~__context in let current_addresses = From 2d68b09e40eca318ceeb3728c7c1fd129bbcf71f Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Tue, 21 Nov 2023 15:51:36 +0000 Subject: [PATCH 2/2] CP-46140 add pvs-setup, replacing setup-pvs-proxy-rules setup-pvs-proxy-rules is a bash script that is hard to maintain. Implement the same functionality plus support for IPv6 in an OCaml binary. This needs a corresponding change in xapi.spec for the new file: %{_libexecdir}/xenopsd/qemu-vif-script %{_libexecdir}/xenopsd/setup-vif-rules %{_libexecdir}/xenopsd/setup-pvs-proxy-rules +%{_libexecdir}/xenopsd/pvs-proxy-ovs-setup %{_libexecdir}/xenopsd/common.py %{_libexecdir}/xenopsd/igmp_query_injector.py * remove the bash script but symlink the name to the new tool * the new tool accepts an IPv6 address for the PVS Server * the new tool comes with a (simple) manual page * a small document for the OVS rules is included Signed-off-by: Christian Lindig --- Makefile | 3 +- doc/content/xenopsd/design/pvs-proxy-ovs.md | 50 ++ ocaml/tests/test_helpers.ml | 12 +- ocaml/xenopsd/pvs/dune | 7 + ocaml/xenopsd/pvs/pvs_proxy_setup.ml | 641 ++++++++++++++++++++ ocaml/xenopsd/pvs/pvs_proxy_setup.mli | 1 + ocaml/xenopsd/scripts/setup-pvs-proxy-rules | 204 ------- ocaml/xenopsd/xc/xc_resources.ml | 2 +- ocaml/xenopsd/xenopsd.conf | 4 +- 9 files changed, 712 insertions(+), 212 deletions(-) create mode 100644 doc/content/xenopsd/design/pvs-proxy-ovs.md create mode 100644 ocaml/xenopsd/pvs/dune create mode 100644 ocaml/xenopsd/pvs/pvs_proxy_setup.ml create mode 100644 ocaml/xenopsd/pvs/pvs_proxy_setup.mli delete mode 100755 ocaml/xenopsd/scripts/setup-pvs-proxy-rules diff --git a/Makefile b/Makefile index 690493952ad..819bd9a432b 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,8 @@ install: build doc sdk doc-json install -D ./ocaml/xenopsd/scripts/tap $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/tap install -D ./ocaml/xenopsd/scripts/qemu-vif-script $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/qemu-vif-script install -D ./ocaml/xenopsd/scripts/setup-vif-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-vif-rules - install -D ./ocaml/xenopsd/scripts/setup-pvs-proxy-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-pvs-proxy-rules + install -D ./_build/install/default/bin/pvs-proxy-ovs-setup $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/pvs-proxy-ovs-setup + (cd $(DESTDIR)/$(XENOPSD_LIBEXECDIR) && ln -s pvs-proxy-ovs-setup setup-pvs-proxy-rules) install -D ./ocaml/xenopsd/scripts/common.py $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/common.py install -D ./ocaml/xenopsd/scripts/igmp_query_injector.py $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/igmp_query_injector.py install -D ./ocaml/xenopsd/scripts/qemu-wrapper $(DESTDIR)/$(QEMU_WRAPPER_DIR)/qemu-wrapper diff --git a/doc/content/xenopsd/design/pvs-proxy-ovs.md b/doc/content/xenopsd/design/pvs-proxy-ovs.md new file mode 100644 index 00000000000..372179077c5 --- /dev/null +++ b/doc/content/xenopsd/design/pvs-proxy-ovs.md @@ -0,0 +1,50 @@ ++++ +title = "PVS Proxy OVS Rules" ++++ + +# Rule Design + +The Open vSwitch (OVS) daemon implements a programmable switch. +XenServer uses it to re-direct traffic between three entities: + + * PVS server - identified by its IP address + * a local VM - identified by its MAC address + * a local Proxy - identified by its MAC address + +VM and PVS server are unaware of the Proxy; xapi configures OVS to +redirect traffic between PVS and VM to pass through the proxy. + +OVS uses rules that match packets. Rules are organised in sets called +tables. A rule can be used to match a packet and to inject it into +another rule set/table table such that a packet can be matched again. + +Furthermore, a rule can set registers associated with a packet which that +can be matched in subsequent rules. In that way, a packet can be tagged +such that it will only match specific rules downstream that match the +tag. + +Xapi configures 3 rule sets: + +## Table 0 - Entry Rules + +Rules match UDP traffic between VM/PVS, Proxy/VM, and PVS/VM where the +PVS server is identified by its IP and all other components by their MAC +address. All packets are tagged with the direction they are going and +re-submitted into Table 101 which handles ports. + +## Table 101 - Port Rules + +Rules match UDP traffic going to a specific port of the PVS server and +re-submit it into Table 102. + +## Table 102 - Exit Rules + +These rules implement the redirection: + +* Rules matching packets coming from VM to PVS are directed to the Proxy. +* Rules matching packets coming from PVS to VM are directed to the Proxy. +* Rules matching packets coming from the Proxy are already addressed + properly (to the VM) are handled normally. + + + diff --git a/ocaml/tests/test_helpers.ml b/ocaml/tests/test_helpers.ml index 4b161cfb2cb..c3b86bbff82 100644 --- a/ocaml/tests/test_helpers.ml +++ b/ocaml/tests/test_helpers.ml @@ -258,17 +258,19 @@ end) module IPCheckers = Generic.MakeStateless (struct module Io = struct - type input_t = [`ipv4 | `ipv6] * string * string + type input_t = [`ipv4 | `ipv6 | `ipv4or6] * string * string type output_t = (unit, exn) result let string_of_input_t = let open Test_printers in - let kind : [`ipv4 | `ipv6] printer = function + let kind : [`ipv4 | `ipv6 | `ipv4or6] printer = function | `ipv4 -> "IPv4" | `ipv6 -> "IPv6" + | `ipv4or6 -> + "IP*" in tuple3 kind string string @@ -323,17 +325,19 @@ end) module CIDRCheckers = Generic.MakeStateless (struct module Io = struct - type input_t = [`ipv4 | `ipv6] * string * string + type input_t = [`ipv4 | `ipv6 | `ipv4or6] * string * string type output_t = (unit, exn) result let string_of_input_t = let open Test_printers in - let kind : [`ipv4 | `ipv6] printer = function + let kind : [`ipv4 | `ipv6 | `ipv4or6] printer = function | `ipv4 -> "IPv4" | `ipv6 -> "IPv6" + | `ipv4or6 -> + "IP*" in tuple3 kind string string diff --git a/ocaml/xenopsd/pvs/dune b/ocaml/xenopsd/pvs/dune new file mode 100644 index 00000000000..6b915db2255 --- /dev/null +++ b/ocaml/xenopsd/pvs/dune @@ -0,0 +1,7 @@ +(executable + (name pvs_proxy_setup) + (public_name pvs-proxy-ovs-setup) + (package xapi-xenopsd-xc) + (libraries ezxenstore.core bos xapi-consts.xapi_version xapi-idl cmdliner) +) + diff --git a/ocaml/xenopsd/pvs/pvs_proxy_setup.ml b/ocaml/xenopsd/pvs/pvs_proxy_setup.ml new file mode 100644 index 00000000000..6d3613c8f82 --- /dev/null +++ b/ocaml/xenopsd/pvs/pvs_proxy_setup.ml @@ -0,0 +1,641 @@ +(* + * Copyright (C) Cloud Software Group, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) + +(* This is a CLI tool called by xenopsd to set up OVS rules for PVS. It + replaces a bash script setup-pvs-proxy-rules used previously and has + a compatible interface *) + +let name = "pvs-proxy-ovs-setup" + +module C = Cmdliner + +module D = Debug.Make (struct + let pid = Unix.getpid () + + let name = Printf.sprintf "%s[%d]" name pid +end) +(* using __MODULE__ leads to a convoluted long name, so avoiding it *) + +module XS = Xenstore + +let error fmt = + Printf.kprintf + (fun msg -> + D.error "%s" msg ; + Result.error (`Msg msg) + ) + fmt + +let ok x = Result.ok x + +let ( let* ) = Result.bind + +let ( >>= ) = Result.bind + +let ( // ) = Filename.concat + +let finally f always = Fun.protect ~finally:always f + +let to_int str = + try Ok (int_of_string str) with _ -> error "not an number: %s" str + +(** iterate over a list, appyling [f] to each element *) +let rec iter f = function + | [] -> + Ok () + | x :: xs -> + let* () = f x in + iter f xs + +(* iterate from lo to hi (inclusive), calling [f] *) +let seq lo hi f = + let rec loop = function + | i when i > hi -> + Ok () + | i -> + let* () = f i in + loop (i + 1) + in + loop lo + +let ip_domain str = + try + let addr = Unix.inet_addr_of_string str in + ok (Unix.domain_of_sockaddr (Unix.ADDR_INET (addr, 1))) + with _e -> error "not an IPv4 or IPv6 address: %s" str + +let is_ipv6 str = + let* domain = ip_domain str in + match domain with + | Unix.PF_INET6 -> + ok true + | Unix.PF_INET -> + ok false + | Unix.PF_UNIX -> + error "unexpected address format: %s" str + +(** open [path] to be consumed by [f] for scanning with Scanf *) +let scanning path f = + let io = Scanf.Scanning.open_in path in + finally (fun () -> f io) (fun () -> Scanf.Scanning.close_in io) + +module Device = struct + type t = VIF | TAP + + let to_string t = match t with VIF -> "vif" | TAP -> "tap" +end + +(** read from xenstore *) +let read path = + try Ok (XS.with_xs (fun xs -> xs.XS.read path)) + with e -> + let msg = Printexc.to_string e in + error "%s: can't read %s: %s" __FUNCTION__ path msg + +(** directory listing from xenstore *) +let dir path = + try Ok (XS.with_xs @@ fun xs -> xs.XS.directory path) + with e -> + let msg = Printexc.to_string e in + error "%s: can't read %s: %s" __FUNCTION__ path msg + +(** write to xenstore *) +let write path value = + try Ok (XS.with_xs @@ fun xs -> xs.XS.write path value) + with e -> + let msg = Printexc.to_string e in + error "%s: can't write %s: %s" __FUNCTION__ path msg + +(* remove path from xenstore *) +let rm path = + try Ok (XS.with_xs @@ fun xs -> xs.XS.rm path) + with e -> + let msg = Printexc.to_string e in + error "%s: can't remove %s: %s" __FUNCTION__ path msg + +(** returns Ok () if PVS is running for the given VIF UUID *) +let pvs_is_running vif_uuid = + let prefix = "/xapi/pvs-proxy" in + let* sites = dir prefix in + let started vif_uuid site = + let* state = read (prefix // site // vif_uuid // "state") in + ok (String.equal state "started") + in + sites + |> List.map (started vif_uuid) + |> List.exists (function Ok true -> true | _ -> false) + |> function + | false -> + error "%s: PVS proxy is not configured for %s" __FUNCTION__ vif_uuid + | true -> + ok () + +(** addresses of PVS server [n] *) +let pvs_server_addrs ~n priv = + let key = Printf.sprintf "pvs-server-%d-addresses" n in + let path = priv // key in + let* str = read path in + String.split_on_char ',' str |> List.map String.trim |> ok +(* we could convert this to an internal IP representation *) + +(** ports of PVS server [n] *) +let pvs_server_ports ~n priv = + let key = Printf.sprintf "pvs-server-%d-ports" n in + let path = priv // key in + let* str = read path in + try Scanf.sscanf str "%d-%d" (fun lo hi -> ok (lo, hi)) + with _e -> error "%s: can't parse %s as two ports" __FUNCTION__ str + +module OVS = struct + open Bos + open Rresult.R.Infix + + let vsctl = Cmd.v "/usr/bin/ovs-vsctl" + + let ofctl = Cmd.v "/usr/bin/ovs-ofctl" + + let echo = Cmd.v "/usr/bin/echo" + + (* Patterns for IPv4/IPv6 *) + + let ip addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6" |> ok + | false -> + Printf.sprintf "ip" |> ok + + let udp addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "udp6" |> ok + | false -> + Printf.sprintf "udp" |> ok + + let nw_src addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6_src=%s" addr |> ok + | false -> + Printf.sprintf "ip_src=%s" addr |> ok + + let nw_dst addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6_dst=%s" addr |> ok + | false -> + Printf.sprintf "ip_dst=%s" addr |> ok + + let run cmd = + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_string) + + let pvs_bridge br = + let cmd = Cmd.(vsctl % "br-to-parent" % br) in + let* out = run cmd in + try Scanf.sscanf out "%s" ok + with _e -> + error "%s: failed to scan first word from '%s'" __FUNCTION__ out + + (** the mac address returned by [cmd] is inside double quotes and we + return it without those. *) + let mac_addr intf = + let cmd = Cmd.(vsctl % "get" % "interface" % intf % "mac_in_use") in + let* out = run cmd in + try Scanf.sscanf out {|"%s@"|} ok + with _e -> error "%s: failed to scan address from '%s'" __FUNCTION__ out + + let ofport intf = + let cmd = Cmd.(vsctl % "get" % "interface" % intf % "ofport") in + let* out = run cmd in + try Scanf.sscanf out {|%d|} ok + with _e -> error "%s: failed to scan port from '%s'" __FUNCTION__ out + + let add_flow ?(debug = false) bridge flow = + let flow' = String.concat "," flow in + let cmd = + match debug with + | true -> + Cmd.(echo % bridge % flow') + | false -> + Cmd.(ofctl % "--strict" % "add-flow" % bridge % flow') + in + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_stdout) + + let del_flows bridge extra = + let args = "del-flows" :: bridge :: extra |> List.map Cmd.v in + let cmd = Cmd.(List.fold_left add_args ofctl args) in + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_stdout) +end + +(** 16bit msb and lsb of the index *) +let interface_id vif = + let path = "/sys/class/net" // vif // "ifindex" in + let ( >> ) = Int64.shift_right in + let ( & ) = Int64.logand in + try + scanning path @@ fun io -> + Scanf.bscanf io {|%Ld|} @@ fun id -> + (* this is what the original code does but it looks suspicious: it + takes the lower 16 bit and upper 32 bit of id *) + ok Int64.(id, id >> 32 |> to_int, (id & 0xffffL) |> to_int) + with _ -> error "failed to scan %s" path + +module Const = struct + let rule_prio = 1000 + + let client2server = 1 + + let proxy2client = 2 + + let server2client = 3 + + let port_table = 101 + + let action_table = 102 + + let dir = "reg0" + + let client_msb = "reg1" + + let client_lsb = "reg2" + + let nxm_dir = "NXM_NX_REG0" + + let nxm_client_msb = "NXM_NX_REG1" + + let nxm_client_lsb = "NXM_NX_REG2" +end + +(** implement the add command *) +let add debug dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let p = Printf.sprintf in + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* pvs_proxy_if = read (priv // "pvs-interface") in + let* mac = read (priv // "mac") in + let* pvs_server_count = read (priv // "pvs-server-num") >>= to_int in + let* bridge' = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge' in + D.debug "%s: pvs_bridge = %s" __FUNCTION__ pvs_bridge ; + let* proxy_mac = OVS.mac_addr pvs_proxy_if in + let* proxy_port = OVS.ofport pvs_proxy_if in + let* vm_port = OVS.ofport vif in + let* id, id_msb, id_lsb = interface_id vif in + + let msg = + String.concat " " + [ + "adding rules to" + ; pvs_bridge + ; "for proxy" + ; p "%s (%d/%s)" pvs_proxy_if proxy_port proxy_mac + ; "and VM" + ; p "%s (%d/%s) id=0x%Lx" vif vm_port mac id + ] + in + D.debug "%s: %s" __FUNCTION__ msg ; + + let add_pvs_server bridge n = + D.debug "%s: bridge=%s server=%d" __FUNCTION__ bridge n ; + let* ip_addrs = pvs_server_addrs ~n priv in + let* port_lo, port_hi = pvs_server_ports ~n priv in + let* () = + (* flows per server address *) + ip_addrs + |> iter @@ fun ip -> + let* nw_src = OVS.nw_src ip in + let* nw_dst = OVS.nw_dst ip in + let* udp = OVS.udp ip in + let* ip = OVS.ip ip in + let* () = + (* Packets from proxied clients that have a PVS-server IP + must be dropped. This is done separately for vif and tap + interfaces by matching on the in_port. *) + OVS.add_flow ~debug bridge + [ + p "cookie=%Ld" id + ; p "priority=%d" Const.rule_prio + ; p "in_port=%d" vm_port + ; ip + ; nw_src + ; "action=drop" + ] + in + (* The following rules are independent of the in_port, so we'll + need just one copy per VIF. We'll only apply them if the + script is called for a vif interface, not for a tap + interface, because tap interfaces are not always present, + while vifs are. *) + match dev with + | Device.TAP -> + ok () + | Device.VIF -> + iter + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "priority=%d" (Const.rule_prio - 1) + ; udp + ; p "dl_src=%s" mac + ; nw_dst + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.client2server Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ; [ + p "cookie=%Ld" id + ; p "priority=%d" Const.rule_prio + ; udp + ; p "dl_src=%s" proxy_mac + ; p "dl_dst=%s" mac + ; nw_src + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.proxy2client Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ; [ + p "cookie=%Ld" id + ; p "priority=%d" (Const.rule_prio - 1) + ; udp + ; p "dl_dst=%s" mac + ; nw_src + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.server2client Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ] + in + (* The following rules are independent of the in_port, so we'll + need just one copy per VIF. We'll only apply them if the script is + called for a vif interface, not for a tap interface, because tap + interfaces are not always present, while vifs are. *) + match dev with + | Device.TAP -> + ok () + | Device.VIF -> + let* () = + seq port_lo port_hi @@ fun port -> + (* flows per port *) + iter (* over flows below *) + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "table=%d" Const.port_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; "udp" + ; p "tp_dst=%d" port + ; p "actions=resubmit(,%d)" Const.action_table + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.port_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; "udp6" + ; p "tp_dst=%d" port + ; p "actions=resubmit(,%d)" Const.action_table + ] + ] + in + + iter (* over flows below *) + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" (Const.rule_prio - 1) + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.client2server + ; p "actions=%d" proxy_port + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.proxy2client + ; "actions=NORMAL" + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" (Const.rule_prio - 1) + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.server2client + ; p "actions=%d" proxy_port + ] + ] + in + + (* now add flows *) + let* () = + iter (* over flows below *) + (OVS.add_flow ~debug pvs_bridge) + [ + ["priority=0"; p "table=%d" Const.port_table; "actions=NORMAL"] + ; ["priority=0"; p "table=%d" Const.action_table; "actions=NORMAL"] + ] + in + (* add flows for each server 0..n *) + let* () = seq 0 (pvs_server_count - 1) (add_pvs_server pvs_bridge) in + (* announce that we are ready *) + let* () = + let path = hotplug // "pvs-rules-active" in + match debug with + | true -> + p "setup complete: %s" path |> print_endline ; + ok () + | false -> + write path "" + in + ok () + +let remove dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let p = Printf.sprintf in + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* bridge = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge in + let path = hotplug // p "%s-ifindex" (Device.to_string dev) in + let* str = read path in + let* intf = + try Scanf.sscanf str "%d" ok + with _e -> error "%s: can't parse %s as a number" __FUNCTION__ str + in + let* () = OVS.del_flows pvs_bridge [p "cookie=%d/-1" intf] in + match dev with TAP -> ok () | VIF -> rm (hotplug // "pvs-rules-active") + +let reset _dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* bridge = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge in + let* () = OVS.del_flows pvs_bridge [] in + OVS.add_flow pvs_bridge ["priority=0"; "actions=NORMAL"] + +(* Command line parsing is delegated to Cmdliner and confined to the CLI + module. *) +module CLI = struct + let build = Xapi_version.version + + let man = + [ + `P "These options are common to all commands." + ; `S "MORE HELP" + ; `P "Use `$(mname) $(i,COMMAND) --help' for help on a single command." + ; `S "BUILD DETAILS" + ; `P build + ] + + (** debug flag for development *) + let debug = + let doc = "Emit to stdout; don't change anything" in + C.Arg.(value & flag & info ["debug"; "d"] ~docv:"DEBUG" ~doc) + + (** device type argument *) + let device = + let dev = C.Arg.enum [("vif", Device.VIF); ("tap", Device.TAP)] in + C.Arg.( + required + & pos 0 (some dev) None + & info [] ~docv:"vif|tap" ~doc:"Device type: vif|tap" + ) + + (** device name, a string *) + let device_name = + C.Arg.( + required + & pos 1 (some string) None + & info [] ~docv:"vif4.0" ~doc:"Device name, e.g. vif4.0" + ) + + (** xenstore path *) + let private_path = + C.Arg.( + required + & pos 2 (some string) None + & info [] ~docv:"PATH" ~doc:"private xenstore path" + ) + + (** xenstore path *) + let hotplug_path = + C.Arg.( + required + & pos 3 (some string) None + & info [] ~docv:"HOTPLUG" ~doc:"hotplug xenstore path" + ) + + (** CLI sub command with args *) + let add = + let doc = "add OVS rules for Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P "Add OVS rules for an interface to use the PVS Proxy." + ; `Blocks man + ] + in + let info = C.Cmd.info "add" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const add + $ debug + $ device + $ device_name + $ private_path + $ hotplug_path + ) + ) + ) + + (** CLI sub command with args *) + let remove = + let doc = "remove OVS rules for a Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P + "Remove OVS rules for an interface such that it no longer uses the \ + PVS proxy." + ; `Blocks man + ] + in + let info = C.Cmd.info "remove" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const remove $ device $ device_name $ private_path $ hotplug_path) + ) + ) + + (** CLI sub command with args *) + let reset = + let doc = "remove OVS rules for all Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P "Remove OVS rules for any interface." + ; `Blocks man + ] + in + let info = C.Cmd.info "reset" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const reset $ device $ device_name $ private_path $ hotplug_path) + ) + ) + + (** sub commands; each sub command can take different arguments *) + let cmds = [add; remove; reset] + + let cmd = + let help = `Help (`Pager, None) in + let doc = "set up OVS rules for PVS proxy" in + let info = C.Cmd.info name ~doc ~man in + let default = C.Term.(ret @@ const help) in + C.Cmd.group info ~default cmds +end + +(* Program entry is below and starts with command line parsing, which + invokes the command implementation *) +let main () = + let this = Sys.argv.(0) in + Debug.set_facility Syslog.User ; + D.info "%s %s" this CLI.build ; + C.Cmd.eval CLI.cmd + +let () = if !Sys.interactive then () else main () |> exit diff --git a/ocaml/xenopsd/pvs/pvs_proxy_setup.mli b/ocaml/xenopsd/pvs/pvs_proxy_setup.mli new file mode 100644 index 00000000000..8ba3226d529 --- /dev/null +++ b/ocaml/xenopsd/pvs/pvs_proxy_setup.mli @@ -0,0 +1 @@ +(* empty - this is not a library *) diff --git a/ocaml/xenopsd/scripts/setup-pvs-proxy-rules b/ocaml/xenopsd/scripts/setup-pvs-proxy-rules deleted file mode 100755 index 6351346b8b9..00000000000 --- a/ocaml/xenopsd/scripts/setup-pvs-proxy-rules +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/bash - -if [[ $# -ne 5 ]] -then - echo "Usage: $0 " >&2 - echo "where is [add|remove|reset]" - echo "where is [vif|tap]" - echo "where is the VIF's device name, e.g. vif2.1" - echo "where is the private xenstore path for the VIF" - echo "where is the xenstore path for the VIF used by the udev script" - exit 1; -fi - -ACTION=$1 -TYPE=$2 -PVS_VM_INTERFACE=$3 -PRIVATE_PATH=$4 -HOTPLUG_PATH=$5 - -VSCTL=/usr/bin/ovs-vsctl -OFCTL=/usr/bin/ovs-ofctl -XSREAD=/usr/bin/xenstore-read -XSWRITE=/usr/bin/xenstore-write -XSRM=/usr/bin/xenstore-rm -XSLIST=/usr/bin/xenstore-list - -LOG_TAG="setup-pvs-proxy-rules" - -handle_error() -{ - echo "$1" >&2 - logger -t "$LOG_TAG" "$1" - exit 1 -} - -handle_xs_error() -{ - handle_error "Failed to read $1 from xenstore" -} - -logger -t "$LOG_TAG" "Called as $0 $*" - -path="${PRIVATE_PATH}/vif-uuid" -VIF=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$VIF" ]; then - handle_xs_error "$path" -fi - -# Only continue if the proxy state is "started". -# According to comments in CA-227626: -# The VIF UUID is maintained across the migration. -# Furthermore, a proxied VIF can be in only one PVS site at once. -pvs_prefix="/xapi/pvs-proxy" -started="false" -for path in $($XSLIST -p "$pvs_prefix"); do - PVS_PROXY_STATE=$($XSREAD "$path/$VIF/state") - if [ $? -eq 0 ] && [ "$PVS_PROXY_STATE" = "started" ]; then - started="true" - break - fi -done -if [ "$started" = "false" ]; then - handle_error "PVS proxy daemon not configured for this proxy - not installing OVS rules." -fi - -path="${PRIVATE_PATH}/pvs-interface" -PVS_PROXY_INTERFACE=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_PROXY_INTERFACE" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/mac" -PVS_VM_MAC=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_VM_MAC" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/pvs-server-num" -PVS_SERVER_NUM=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_SERVER_NUM" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/bridge" -bridge=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$bridge" ]; then - handle_xs_error "$path" -fi -PVS_BRIDGE=$($VSCTL br-to-parent "$bridge") - -PVS_RULE_PRIO=1000 -DIR_CLIENT_TO_SERVER=1 -DIR_PROXY_TO_CLIENT=2 -DIR_SERVER_TO_CLIENT=3 - -PVS_PORT_TABLE=101 -PVS_ACTION_TABLE=102 - -PVS_DIR_REG=reg0 -PVS_CLIENT_ID_REG_MSB=reg1 -PVS_CLIENT_ID_REG_LSB=reg2 -PVS_NXM_NX_DIR_REG=NXM_NX_REG0 -PVS_NXM_NX_CLIENT_ID_REG_MSB=NXM_NX_REG1 -PVS_NXM_NX_CLIENT_ID_REG_LSB=NXM_NX_REG2 - -case $ACTION in - add) - PVS_PROXY_MAC=$($VSCTL get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"') - PVS_PROXY_OFPORT=$($VSCTL get interface "$PVS_PROXY_INTERFACE" ofport) - PVS_VM_OFPORT=$($VSCTL get interface "$PVS_VM_INTERFACE" ofport) - PVS_IF_ID=$(cat /sys/class/net/"$PVS_VM_INTERFACE"/ifindex) - PVS_IF_ID_LSB=$((PVS_IF_ID&0xffff)) - PVS_IF_ID_MSB=$((PVS_IF_ID>>32)) - - if [ $? -ne 0 ] || [ -z "$PVS_VM_OFPORT" ]; then - handle_error "The $PVS_VM_INTERFACE interface was not found on a bridge" - fi - - logger -t "$LOG_TAG" "Adding rules to $PVS_BRIDGE for $PVS_PROXY_INTERFACE $PVS_PROXY_OFPORT/$PVS_PROXY_MAC and $PVS_VM_INTERFACE/$PVS_IF_ID $PVS_VM_OFPORT/$PVS_VM_MAC" - - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,table=$((PVS_PORT_TABLE)),actions=NORMAL - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,table=$((PVS_ACTION_TABLE)),actions=NORMAL - - for ((j=0; jserver that need to be proxied. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_CLIENT_TO_SERVER"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - # Packets from proxy->client. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_PROXY_TO_CLIENT"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - # Packets from server->client that need to be proxied. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_SERVER_TO_CLIENT"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - fi - done - # The following rules are independent of the in_port, so we'll - # need just one copy per VIF. We'll only apply them if the - # script is called for a vif interface, not for a tap interface, - # because tap interfaces are not always present, while vifs are. - if [ "${TYPE}" = "vif" ]; then - for ((i=PVS_STARTPORT; i<=PVS_ENDPORT; i++)) do - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_PORT_TABLE)),priority=$((PVS_RULE_PRIO)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB",udp,tp_dst="$i",actions=resubmit\(,"$PVS_ACTION_TABLE"\) - done - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO-1)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_CLIENT_TO_SERVER",actions="$PVS_PROXY_OFPORT" - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_PROXY_TO_CLIENT",actions=NORMAL - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO-1)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_SERVER_TO_CLIENT",actions="$PVS_PROXY_OFPORT" - - fi - unset IFS - done - # Announce that on the OVS we have set up the rules for this VIF's pvs-proxy - $XSWRITE "${HOTPLUG_PATH}/pvs-rules-active" '' - ;; - remove) - path="${HOTPLUG_PATH}/$TYPE-ifindex" - PVS_IF_ID=$($XSREAD "$path") - if [ $? -ne 0 ] || [ -z "$PVS_IF_ID" ]; then - handle_error "The $PVS_VM_INTERFACE's ID[$PVS_IF_ID] not found" - fi - - logger -t "$LOG_TAG" "Removing rules from $PVS_BRIDGE for $PVS_PROXY_INTERFACE $PVS_VM_INTERFACE/$PVS_IF_ID" - - $OFCTL del-flows "$PVS_BRIDGE" cookie="$PVS_IF_ID"/-1 - if [ "${TYPE}" = "vif" ]; then - # Again, don't do the following when a tap goes away, because - # vif may still be there. - - # Announce that on the OVS we have removed the rules for this VIF's pvs-proxy. - $XSRM "${HOTPLUG_PATH}/pvs-rules-active" - fi - ;; - reset) - $OFCTL del-flows "$PVS_BRIDGE" - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,actions=NORMAL - ;; - *) - handle_error "Unknown command '$1'" - ;; -esac diff --git a/ocaml/xenopsd/xc/xc_resources.ml b/ocaml/xenopsd/xc/xc_resources.ml index 5f9dba7f79f..2199fb04ab6 100644 --- a/ocaml/xenopsd/xc/xc_resources.ml +++ b/ocaml/xenopsd/xc/xc_resources.ml @@ -37,7 +37,7 @@ let ionice = ref "ionice" let setup_vif_rules = ref "setup-vif-rules" -let setup_pvs_proxy_rules = ref "setup-pvs-proxy-rules" +let setup_pvs_proxy_rules = ref "pvs-proxy-ovs-setup" let vgpu = ref "vgpu" diff --git a/ocaml/xenopsd/xenopsd.conf b/ocaml/xenopsd/xenopsd.conf index 46f95207c7d..353e758c0d3 100644 --- a/ocaml/xenopsd/xenopsd.conf +++ b/ocaml/xenopsd/xenopsd.conf @@ -79,8 +79,8 @@ disable-logging-for=http tracing # Path to the setup-vif-rules script # setup-vif-rules=/opt/xensource/libexec/setup-vif-rules -# Path to the setup-pvs-proxy-rules script -# setup-pvs-proxy-rules=/usr/libexec/xenopsd/setup-pvs-proxy-rules +# Path to the pvs-setup script +# setup-pvs-proxy-rules=/usr/libexec/xenopsd/pvs-proxy-ovs-setup # Paths to standard system utilities: # tune2fs=/sbin/tune2fs