diff --git a/Makefile b/Makefile index 690493952ad..819bd9a432b 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,8 @@ install: build doc sdk doc-json install -D ./ocaml/xenopsd/scripts/tap $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/tap install -D ./ocaml/xenopsd/scripts/qemu-vif-script $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/qemu-vif-script install -D ./ocaml/xenopsd/scripts/setup-vif-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-vif-rules - install -D ./ocaml/xenopsd/scripts/setup-pvs-proxy-rules $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/setup-pvs-proxy-rules + install -D ./_build/install/default/bin/pvs-proxy-ovs-setup $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/pvs-proxy-ovs-setup + (cd $(DESTDIR)/$(XENOPSD_LIBEXECDIR) && ln -s pvs-proxy-ovs-setup setup-pvs-proxy-rules) install -D ./ocaml/xenopsd/scripts/common.py $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/common.py install -D ./ocaml/xenopsd/scripts/igmp_query_injector.py $(DESTDIR)/$(XENOPSD_LIBEXECDIR)/igmp_query_injector.py install -D ./ocaml/xenopsd/scripts/qemu-wrapper $(DESTDIR)/$(QEMU_WRAPPER_DIR)/qemu-wrapper diff --git a/doc/content/xenopsd/design/pvs-proxy-ovs.md b/doc/content/xenopsd/design/pvs-proxy-ovs.md new file mode 100644 index 00000000000..372179077c5 --- /dev/null +++ b/doc/content/xenopsd/design/pvs-proxy-ovs.md @@ -0,0 +1,50 @@ ++++ +title = "PVS Proxy OVS Rules" ++++ + +# Rule Design + +The Open vSwitch (OVS) daemon implements a programmable switch. +XenServer uses it to re-direct traffic between three entities: + + * PVS server - identified by its IP address + * a local VM - identified by its MAC address + * a local Proxy - identified by its MAC address + +VM and PVS server are unaware of the Proxy; xapi configures OVS to +redirect traffic between PVS and VM to pass through the proxy. + +OVS uses rules that match packets. Rules are organised in sets called +tables. A rule can be used to match a packet and to inject it into +another rule set/table table such that a packet can be matched again. + +Furthermore, a rule can set registers associated with a packet which that +can be matched in subsequent rules. In that way, a packet can be tagged +such that it will only match specific rules downstream that match the +tag. + +Xapi configures 3 rule sets: + +## Table 0 - Entry Rules + +Rules match UDP traffic between VM/PVS, Proxy/VM, and PVS/VM where the +PVS server is identified by its IP and all other components by their MAC +address. All packets are tagged with the direction they are going and +re-submitted into Table 101 which handles ports. + +## Table 101 - Port Rules + +Rules match UDP traffic going to a specific port of the PVS server and +re-submit it into Table 102. + +## Table 102 - Exit Rules + +These rules implement the redirection: + +* Rules matching packets coming from VM to PVS are directed to the Proxy. +* Rules matching packets coming from PVS to VM are directed to the Proxy. +* Rules matching packets coming from the Proxy are already addressed + properly (to the VM) are handled normally. + + + diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index f2ea7399946..aa19b1ab347 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -7298,7 +7298,7 @@ module PVS_server = struct ~result:(Ref _pvs_server, "the new PVS server") ~params: [ - (Set String, "addresses", "IPv4 addresses of the server") + (Set String, "addresses", "IPv4/IPv6 addresses of the server") ; (Int, "first_port", "first UDP port accepted by this server") ; (Int, "last_port", "last UDP port accepted by this server") ; (Ref _pvs_site, "site", "PVS site this server is a part of") diff --git a/ocaml/tests/test_helpers.ml b/ocaml/tests/test_helpers.ml index 4b161cfb2cb..c3b86bbff82 100644 --- a/ocaml/tests/test_helpers.ml +++ b/ocaml/tests/test_helpers.ml @@ -258,17 +258,19 @@ end) module IPCheckers = Generic.MakeStateless (struct module Io = struct - type input_t = [`ipv4 | `ipv6] * string * string + type input_t = [`ipv4 | `ipv6 | `ipv4or6] * string * string type output_t = (unit, exn) result let string_of_input_t = let open Test_printers in - let kind : [`ipv4 | `ipv6] printer = function + let kind : [`ipv4 | `ipv6 | `ipv4or6] printer = function | `ipv4 -> "IPv4" | `ipv6 -> "IPv6" + | `ipv4or6 -> + "IP*" in tuple3 kind string string @@ -323,17 +325,19 @@ end) module CIDRCheckers = Generic.MakeStateless (struct module Io = struct - type input_t = [`ipv4 | `ipv6] * string * string + type input_t = [`ipv4 | `ipv6 | `ipv4or6] * string * string type output_t = (unit, exn) result let string_of_input_t = let open Test_printers in - let kind : [`ipv4 | `ipv6] printer = function + let kind : [`ipv4 | `ipv6 | `ipv4or6] printer = function | `ipv4 -> "IPv4" | `ipv6 -> "IPv6" + | `ipv4or6 -> + "IP*" in tuple3 kind string string diff --git a/ocaml/xapi/helpers.ml b/ocaml/xapi/helpers.ml index c9f70e8c8a8..c31ed7895ec 100644 --- a/ocaml/xapi/helpers.ml +++ b/ocaml/xapi/helpers.ml @@ -1097,6 +1097,8 @@ let assert_is_valid_tcp_udp_port_range ~first_port ~first_name ~last_port let is_valid_ip kind address = match (Unixext.domain_of_addr address, kind) with + | Some x, `ipv4or6 when x = Unix.PF_INET || x = Unix.PF_INET6 -> + true | Some x, `ipv4 when x = Unix.PF_INET -> true | Some x, `ipv6 when x = Unix.PF_INET6 -> diff --git a/ocaml/xapi/xapi_pvs_server.ml b/ocaml/xapi/xapi_pvs_server.ml index c124bf52321..dc6c5f59212 100644 --- a/ocaml/xapi/xapi_pvs_server.ml +++ b/ocaml/xapi/xapi_pvs_server.ml @@ -22,7 +22,7 @@ let introduce ~__context ~addresses ~first_port ~last_port ~site = Pool_features.assert_enabled ~__context ~f:Features.PVS_proxy ; Helpers.assert_using_vswitch ~__context ; List.iter - (fun address -> Helpers.assert_is_valid_ip `ipv4 "addresses" address) + (fun address -> Helpers.assert_is_valid_ip `ipv4or6 "addresses" address) addresses ; let current = Db.PVS_server.get_all_records ~__context in let current_addresses = diff --git a/ocaml/xenopsd/pvs/dune b/ocaml/xenopsd/pvs/dune new file mode 100644 index 00000000000..6b915db2255 --- /dev/null +++ b/ocaml/xenopsd/pvs/dune @@ -0,0 +1,7 @@ +(executable + (name pvs_proxy_setup) + (public_name pvs-proxy-ovs-setup) + (package xapi-xenopsd-xc) + (libraries ezxenstore.core bos xapi-consts.xapi_version xapi-idl cmdliner) +) + diff --git a/ocaml/xenopsd/pvs/pvs_proxy_setup.ml b/ocaml/xenopsd/pvs/pvs_proxy_setup.ml new file mode 100644 index 00000000000..6d3613c8f82 --- /dev/null +++ b/ocaml/xenopsd/pvs/pvs_proxy_setup.ml @@ -0,0 +1,641 @@ +(* + * Copyright (C) Cloud Software Group, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + *) + +(* This is a CLI tool called by xenopsd to set up OVS rules for PVS. It + replaces a bash script setup-pvs-proxy-rules used previously and has + a compatible interface *) + +let name = "pvs-proxy-ovs-setup" + +module C = Cmdliner + +module D = Debug.Make (struct + let pid = Unix.getpid () + + let name = Printf.sprintf "%s[%d]" name pid +end) +(* using __MODULE__ leads to a convoluted long name, so avoiding it *) + +module XS = Xenstore + +let error fmt = + Printf.kprintf + (fun msg -> + D.error "%s" msg ; + Result.error (`Msg msg) + ) + fmt + +let ok x = Result.ok x + +let ( let* ) = Result.bind + +let ( >>= ) = Result.bind + +let ( // ) = Filename.concat + +let finally f always = Fun.protect ~finally:always f + +let to_int str = + try Ok (int_of_string str) with _ -> error "not an number: %s" str + +(** iterate over a list, appyling [f] to each element *) +let rec iter f = function + | [] -> + Ok () + | x :: xs -> + let* () = f x in + iter f xs + +(* iterate from lo to hi (inclusive), calling [f] *) +let seq lo hi f = + let rec loop = function + | i when i > hi -> + Ok () + | i -> + let* () = f i in + loop (i + 1) + in + loop lo + +let ip_domain str = + try + let addr = Unix.inet_addr_of_string str in + ok (Unix.domain_of_sockaddr (Unix.ADDR_INET (addr, 1))) + with _e -> error "not an IPv4 or IPv6 address: %s" str + +let is_ipv6 str = + let* domain = ip_domain str in + match domain with + | Unix.PF_INET6 -> + ok true + | Unix.PF_INET -> + ok false + | Unix.PF_UNIX -> + error "unexpected address format: %s" str + +(** open [path] to be consumed by [f] for scanning with Scanf *) +let scanning path f = + let io = Scanf.Scanning.open_in path in + finally (fun () -> f io) (fun () -> Scanf.Scanning.close_in io) + +module Device = struct + type t = VIF | TAP + + let to_string t = match t with VIF -> "vif" | TAP -> "tap" +end + +(** read from xenstore *) +let read path = + try Ok (XS.with_xs (fun xs -> xs.XS.read path)) + with e -> + let msg = Printexc.to_string e in + error "%s: can't read %s: %s" __FUNCTION__ path msg + +(** directory listing from xenstore *) +let dir path = + try Ok (XS.with_xs @@ fun xs -> xs.XS.directory path) + with e -> + let msg = Printexc.to_string e in + error "%s: can't read %s: %s" __FUNCTION__ path msg + +(** write to xenstore *) +let write path value = + try Ok (XS.with_xs @@ fun xs -> xs.XS.write path value) + with e -> + let msg = Printexc.to_string e in + error "%s: can't write %s: %s" __FUNCTION__ path msg + +(* remove path from xenstore *) +let rm path = + try Ok (XS.with_xs @@ fun xs -> xs.XS.rm path) + with e -> + let msg = Printexc.to_string e in + error "%s: can't remove %s: %s" __FUNCTION__ path msg + +(** returns Ok () if PVS is running for the given VIF UUID *) +let pvs_is_running vif_uuid = + let prefix = "/xapi/pvs-proxy" in + let* sites = dir prefix in + let started vif_uuid site = + let* state = read (prefix // site // vif_uuid // "state") in + ok (String.equal state "started") + in + sites + |> List.map (started vif_uuid) + |> List.exists (function Ok true -> true | _ -> false) + |> function + | false -> + error "%s: PVS proxy is not configured for %s" __FUNCTION__ vif_uuid + | true -> + ok () + +(** addresses of PVS server [n] *) +let pvs_server_addrs ~n priv = + let key = Printf.sprintf "pvs-server-%d-addresses" n in + let path = priv // key in + let* str = read path in + String.split_on_char ',' str |> List.map String.trim |> ok +(* we could convert this to an internal IP representation *) + +(** ports of PVS server [n] *) +let pvs_server_ports ~n priv = + let key = Printf.sprintf "pvs-server-%d-ports" n in + let path = priv // key in + let* str = read path in + try Scanf.sscanf str "%d-%d" (fun lo hi -> ok (lo, hi)) + with _e -> error "%s: can't parse %s as two ports" __FUNCTION__ str + +module OVS = struct + open Bos + open Rresult.R.Infix + + let vsctl = Cmd.v "/usr/bin/ovs-vsctl" + + let ofctl = Cmd.v "/usr/bin/ovs-ofctl" + + let echo = Cmd.v "/usr/bin/echo" + + (* Patterns for IPv4/IPv6 *) + + let ip addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6" |> ok + | false -> + Printf.sprintf "ip" |> ok + + let udp addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "udp6" |> ok + | false -> + Printf.sprintf "udp" |> ok + + let nw_src addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6_src=%s" addr |> ok + | false -> + Printf.sprintf "ip_src=%s" addr |> ok + + let nw_dst addr = + is_ipv6 addr >>= function + | true -> + Printf.sprintf "ipv6_dst=%s" addr |> ok + | false -> + Printf.sprintf "ip_dst=%s" addr |> ok + + let run cmd = + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_string) + + let pvs_bridge br = + let cmd = Cmd.(vsctl % "br-to-parent" % br) in + let* out = run cmd in + try Scanf.sscanf out "%s" ok + with _e -> + error "%s: failed to scan first word from '%s'" __FUNCTION__ out + + (** the mac address returned by [cmd] is inside double quotes and we + return it without those. *) + let mac_addr intf = + let cmd = Cmd.(vsctl % "get" % "interface" % intf % "mac_in_use") in + let* out = run cmd in + try Scanf.sscanf out {|"%s@"|} ok + with _e -> error "%s: failed to scan address from '%s'" __FUNCTION__ out + + let ofport intf = + let cmd = Cmd.(vsctl % "get" % "interface" % intf % "ofport") in + let* out = run cmd in + try Scanf.sscanf out {|%d|} ok + with _e -> error "%s: failed to scan port from '%s'" __FUNCTION__ out + + let add_flow ?(debug = false) bridge flow = + let flow' = String.concat "," flow in + let cmd = + match debug with + | true -> + Cmd.(echo % bridge % flow') + | false -> + Cmd.(ofctl % "--strict" % "add-flow" % bridge % flow') + in + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_stdout) + + let del_flows bridge extra = + let args = "del-flows" :: bridge :: extra |> List.map Cmd.v in + let cmd = Cmd.(List.fold_left add_args ofctl args) in + D.info "%s: %s" __FUNCTION__ (Cmd.to_string cmd) ; + OS.Cmd.(run_out cmd |> to_stdout) +end + +(** 16bit msb and lsb of the index *) +let interface_id vif = + let path = "/sys/class/net" // vif // "ifindex" in + let ( >> ) = Int64.shift_right in + let ( & ) = Int64.logand in + try + scanning path @@ fun io -> + Scanf.bscanf io {|%Ld|} @@ fun id -> + (* this is what the original code does but it looks suspicious: it + takes the lower 16 bit and upper 32 bit of id *) + ok Int64.(id, id >> 32 |> to_int, (id & 0xffffL) |> to_int) + with _ -> error "failed to scan %s" path + +module Const = struct + let rule_prio = 1000 + + let client2server = 1 + + let proxy2client = 2 + + let server2client = 3 + + let port_table = 101 + + let action_table = 102 + + let dir = "reg0" + + let client_msb = "reg1" + + let client_lsb = "reg2" + + let nxm_dir = "NXM_NX_REG0" + + let nxm_client_msb = "NXM_NX_REG1" + + let nxm_client_lsb = "NXM_NX_REG2" +end + +(** implement the add command *) +let add debug dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let p = Printf.sprintf in + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* pvs_proxy_if = read (priv // "pvs-interface") in + let* mac = read (priv // "mac") in + let* pvs_server_count = read (priv // "pvs-server-num") >>= to_int in + let* bridge' = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge' in + D.debug "%s: pvs_bridge = %s" __FUNCTION__ pvs_bridge ; + let* proxy_mac = OVS.mac_addr pvs_proxy_if in + let* proxy_port = OVS.ofport pvs_proxy_if in + let* vm_port = OVS.ofport vif in + let* id, id_msb, id_lsb = interface_id vif in + + let msg = + String.concat " " + [ + "adding rules to" + ; pvs_bridge + ; "for proxy" + ; p "%s (%d/%s)" pvs_proxy_if proxy_port proxy_mac + ; "and VM" + ; p "%s (%d/%s) id=0x%Lx" vif vm_port mac id + ] + in + D.debug "%s: %s" __FUNCTION__ msg ; + + let add_pvs_server bridge n = + D.debug "%s: bridge=%s server=%d" __FUNCTION__ bridge n ; + let* ip_addrs = pvs_server_addrs ~n priv in + let* port_lo, port_hi = pvs_server_ports ~n priv in + let* () = + (* flows per server address *) + ip_addrs + |> iter @@ fun ip -> + let* nw_src = OVS.nw_src ip in + let* nw_dst = OVS.nw_dst ip in + let* udp = OVS.udp ip in + let* ip = OVS.ip ip in + let* () = + (* Packets from proxied clients that have a PVS-server IP + must be dropped. This is done separately for vif and tap + interfaces by matching on the in_port. *) + OVS.add_flow ~debug bridge + [ + p "cookie=%Ld" id + ; p "priority=%d" Const.rule_prio + ; p "in_port=%d" vm_port + ; ip + ; nw_src + ; "action=drop" + ] + in + (* The following rules are independent of the in_port, so we'll + need just one copy per VIF. We'll only apply them if the + script is called for a vif interface, not for a tap + interface, because tap interfaces are not always present, + while vifs are. *) + match dev with + | Device.TAP -> + ok () + | Device.VIF -> + iter + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "priority=%d" (Const.rule_prio - 1) + ; udp + ; p "dl_src=%s" mac + ; nw_dst + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.client2server Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ; [ + p "cookie=%Ld" id + ; p "priority=%d" Const.rule_prio + ; udp + ; p "dl_src=%s" proxy_mac + ; p "dl_dst=%s" mac + ; nw_src + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.proxy2client Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ; [ + p "cookie=%Ld" id + ; p "priority=%d" (Const.rule_prio - 1) + ; udp + ; p "dl_dst=%s" mac + ; nw_src + ; p "actions=load:%d->%s[]" id_lsb Const.nxm_client_lsb + ; p "load:%d->%s[]" id_msb Const.nxm_client_msb + ; p "load:%d->%s[]" Const.server2client Const.nxm_dir + ; p "resubmit(,%d)" Const.port_table + ] + ] + in + (* The following rules are independent of the in_port, so we'll + need just one copy per VIF. We'll only apply them if the script is + called for a vif interface, not for a tap interface, because tap + interfaces are not always present, while vifs are. *) + match dev with + | Device.TAP -> + ok () + | Device.VIF -> + let* () = + seq port_lo port_hi @@ fun port -> + (* flows per port *) + iter (* over flows below *) + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "table=%d" Const.port_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; "udp" + ; p "tp_dst=%d" port + ; p "actions=resubmit(,%d)" Const.action_table + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.port_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; "udp6" + ; p "tp_dst=%d" port + ; p "actions=resubmit(,%d)" Const.action_table + ] + ] + in + + iter (* over flows below *) + (OVS.add_flow ~debug bridge) + [ + [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" (Const.rule_prio - 1) + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.client2server + ; p "actions=%d" proxy_port + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" Const.rule_prio + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.proxy2client + ; "actions=NORMAL" + ] + ; [ + p "cookie=%Ld" id + ; p "table=%d" Const.action_table + ; p "priority=%d" (Const.rule_prio - 1) + ; p "%s=%d" Const.client_lsb id_lsb + ; p "%s=%d" Const.client_msb id_msb + ; p "%s=%d" Const.dir Const.server2client + ; p "actions=%d" proxy_port + ] + ] + in + + (* now add flows *) + let* () = + iter (* over flows below *) + (OVS.add_flow ~debug pvs_bridge) + [ + ["priority=0"; p "table=%d" Const.port_table; "actions=NORMAL"] + ; ["priority=0"; p "table=%d" Const.action_table; "actions=NORMAL"] + ] + in + (* add flows for each server 0..n *) + let* () = seq 0 (pvs_server_count - 1) (add_pvs_server pvs_bridge) in + (* announce that we are ready *) + let* () = + let path = hotplug // "pvs-rules-active" in + match debug with + | true -> + p "setup complete: %s" path |> print_endline ; + ok () + | false -> + write path "" + in + ok () + +let remove dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let p = Printf.sprintf in + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* bridge = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge in + let path = hotplug // p "%s-ifindex" (Device.to_string dev) in + let* str = read path in + let* intf = + try Scanf.sscanf str "%d" ok + with _e -> error "%s: can't parse %s as a number" __FUNCTION__ str + in + let* () = OVS.del_flows pvs_bridge [p "cookie=%d/-1" intf] in + match dev with TAP -> ok () | VIF -> rm (hotplug // "pvs-rules-active") + +let reset _dev vif priv hotplug = + D.debug "%s: %s %s %s" __FUNCTION__ vif priv hotplug ; + let* vif_uuid = read (priv // "vif-uuid") in + let* () = pvs_is_running vif_uuid in + let* bridge = read (priv // "bridge") in + let* pvs_bridge = OVS.pvs_bridge bridge in + let* () = OVS.del_flows pvs_bridge [] in + OVS.add_flow pvs_bridge ["priority=0"; "actions=NORMAL"] + +(* Command line parsing is delegated to Cmdliner and confined to the CLI + module. *) +module CLI = struct + let build = Xapi_version.version + + let man = + [ + `P "These options are common to all commands." + ; `S "MORE HELP" + ; `P "Use `$(mname) $(i,COMMAND) --help' for help on a single command." + ; `S "BUILD DETAILS" + ; `P build + ] + + (** debug flag for development *) + let debug = + let doc = "Emit to stdout; don't change anything" in + C.Arg.(value & flag & info ["debug"; "d"] ~docv:"DEBUG" ~doc) + + (** device type argument *) + let device = + let dev = C.Arg.enum [("vif", Device.VIF); ("tap", Device.TAP)] in + C.Arg.( + required + & pos 0 (some dev) None + & info [] ~docv:"vif|tap" ~doc:"Device type: vif|tap" + ) + + (** device name, a string *) + let device_name = + C.Arg.( + required + & pos 1 (some string) None + & info [] ~docv:"vif4.0" ~doc:"Device name, e.g. vif4.0" + ) + + (** xenstore path *) + let private_path = + C.Arg.( + required + & pos 2 (some string) None + & info [] ~docv:"PATH" ~doc:"private xenstore path" + ) + + (** xenstore path *) + let hotplug_path = + C.Arg.( + required + & pos 3 (some string) None + & info [] ~docv:"HOTPLUG" ~doc:"hotplug xenstore path" + ) + + (** CLI sub command with args *) + let add = + let doc = "add OVS rules for Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P "Add OVS rules for an interface to use the PVS Proxy." + ; `Blocks man + ] + in + let info = C.Cmd.info "add" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const add + $ debug + $ device + $ device_name + $ private_path + $ hotplug_path + ) + ) + ) + + (** CLI sub command with args *) + let remove = + let doc = "remove OVS rules for a Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P + "Remove OVS rules for an interface such that it no longer uses the \ + PVS proxy." + ; `Blocks man + ] + in + let info = C.Cmd.info "remove" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const remove $ device $ device_name $ private_path $ hotplug_path) + ) + ) + + (** CLI sub command with args *) + let reset = + let doc = "remove OVS rules for all Virtual Interface (VIF) or TAP" in + let man = + [ + `S C.Manpage.s_description + ; `P "Remove OVS rules for any interface." + ; `Blocks man + ] + in + let info = C.Cmd.info "reset" ~doc ~man in + C.( + Cmd.v info + Term.( + term_result + (const reset $ device $ device_name $ private_path $ hotplug_path) + ) + ) + + (** sub commands; each sub command can take different arguments *) + let cmds = [add; remove; reset] + + let cmd = + let help = `Help (`Pager, None) in + let doc = "set up OVS rules for PVS proxy" in + let info = C.Cmd.info name ~doc ~man in + let default = C.Term.(ret @@ const help) in + C.Cmd.group info ~default cmds +end + +(* Program entry is below and starts with command line parsing, which + invokes the command implementation *) +let main () = + let this = Sys.argv.(0) in + Debug.set_facility Syslog.User ; + D.info "%s %s" this CLI.build ; + C.Cmd.eval CLI.cmd + +let () = if !Sys.interactive then () else main () |> exit diff --git a/ocaml/xenopsd/pvs/pvs_proxy_setup.mli b/ocaml/xenopsd/pvs/pvs_proxy_setup.mli new file mode 100644 index 00000000000..8ba3226d529 --- /dev/null +++ b/ocaml/xenopsd/pvs/pvs_proxy_setup.mli @@ -0,0 +1 @@ +(* empty - this is not a library *) diff --git a/ocaml/xenopsd/scripts/setup-pvs-proxy-rules b/ocaml/xenopsd/scripts/setup-pvs-proxy-rules deleted file mode 100755 index 6351346b8b9..00000000000 --- a/ocaml/xenopsd/scripts/setup-pvs-proxy-rules +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/bash - -if [[ $# -ne 5 ]] -then - echo "Usage: $0 " >&2 - echo "where is [add|remove|reset]" - echo "where is [vif|tap]" - echo "where is the VIF's device name, e.g. vif2.1" - echo "where is the private xenstore path for the VIF" - echo "where is the xenstore path for the VIF used by the udev script" - exit 1; -fi - -ACTION=$1 -TYPE=$2 -PVS_VM_INTERFACE=$3 -PRIVATE_PATH=$4 -HOTPLUG_PATH=$5 - -VSCTL=/usr/bin/ovs-vsctl -OFCTL=/usr/bin/ovs-ofctl -XSREAD=/usr/bin/xenstore-read -XSWRITE=/usr/bin/xenstore-write -XSRM=/usr/bin/xenstore-rm -XSLIST=/usr/bin/xenstore-list - -LOG_TAG="setup-pvs-proxy-rules" - -handle_error() -{ - echo "$1" >&2 - logger -t "$LOG_TAG" "$1" - exit 1 -} - -handle_xs_error() -{ - handle_error "Failed to read $1 from xenstore" -} - -logger -t "$LOG_TAG" "Called as $0 $*" - -path="${PRIVATE_PATH}/vif-uuid" -VIF=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$VIF" ]; then - handle_xs_error "$path" -fi - -# Only continue if the proxy state is "started". -# According to comments in CA-227626: -# The VIF UUID is maintained across the migration. -# Furthermore, a proxied VIF can be in only one PVS site at once. -pvs_prefix="/xapi/pvs-proxy" -started="false" -for path in $($XSLIST -p "$pvs_prefix"); do - PVS_PROXY_STATE=$($XSREAD "$path/$VIF/state") - if [ $? -eq 0 ] && [ "$PVS_PROXY_STATE" = "started" ]; then - started="true" - break - fi -done -if [ "$started" = "false" ]; then - handle_error "PVS proxy daemon not configured for this proxy - not installing OVS rules." -fi - -path="${PRIVATE_PATH}/pvs-interface" -PVS_PROXY_INTERFACE=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_PROXY_INTERFACE" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/mac" -PVS_VM_MAC=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_VM_MAC" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/pvs-server-num" -PVS_SERVER_NUM=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$PVS_SERVER_NUM" ]; then - handle_xs_error "$path" -fi - -path="${PRIVATE_PATH}/bridge" -bridge=$($XSREAD "$path") -if [ $? -ne 0 ] || [ -z "$bridge" ]; then - handle_xs_error "$path" -fi -PVS_BRIDGE=$($VSCTL br-to-parent "$bridge") - -PVS_RULE_PRIO=1000 -DIR_CLIENT_TO_SERVER=1 -DIR_PROXY_TO_CLIENT=2 -DIR_SERVER_TO_CLIENT=3 - -PVS_PORT_TABLE=101 -PVS_ACTION_TABLE=102 - -PVS_DIR_REG=reg0 -PVS_CLIENT_ID_REG_MSB=reg1 -PVS_CLIENT_ID_REG_LSB=reg2 -PVS_NXM_NX_DIR_REG=NXM_NX_REG0 -PVS_NXM_NX_CLIENT_ID_REG_MSB=NXM_NX_REG1 -PVS_NXM_NX_CLIENT_ID_REG_LSB=NXM_NX_REG2 - -case $ACTION in - add) - PVS_PROXY_MAC=$($VSCTL get interface "$PVS_PROXY_INTERFACE" mac_in_use | tr -d '"') - PVS_PROXY_OFPORT=$($VSCTL get interface "$PVS_PROXY_INTERFACE" ofport) - PVS_VM_OFPORT=$($VSCTL get interface "$PVS_VM_INTERFACE" ofport) - PVS_IF_ID=$(cat /sys/class/net/"$PVS_VM_INTERFACE"/ifindex) - PVS_IF_ID_LSB=$((PVS_IF_ID&0xffff)) - PVS_IF_ID_MSB=$((PVS_IF_ID>>32)) - - if [ $? -ne 0 ] || [ -z "$PVS_VM_OFPORT" ]; then - handle_error "The $PVS_VM_INTERFACE interface was not found on a bridge" - fi - - logger -t "$LOG_TAG" "Adding rules to $PVS_BRIDGE for $PVS_PROXY_INTERFACE $PVS_PROXY_OFPORT/$PVS_PROXY_MAC and $PVS_VM_INTERFACE/$PVS_IF_ID $PVS_VM_OFPORT/$PVS_VM_MAC" - - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,table=$((PVS_PORT_TABLE)),actions=NORMAL - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,table=$((PVS_ACTION_TABLE)),actions=NORMAL - - for ((j=0; jserver that need to be proxied. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO-1)),udp,dl_src="$PVS_VM_MAC",nw_dst="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_CLIENT_TO_SERVER"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - # Packets from proxy->client. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO)),udp,dl_src="$PVS_PROXY_MAC",dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_PROXY_TO_CLIENT"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - # Packets from server->client that need to be proxied. - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,priority=$((PVS_RULE_PRIO-1)),udp,dl_dst="$PVS_VM_MAC",nw_src="$PVS_SERVER_IP",actions=load:"$PVS_IF_ID_LSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_LSB"[],load:"$PVS_IF_ID_MSB"-\>"$PVS_NXM_NX_CLIENT_ID_REG_MSB"[],load:"$DIR_SERVER_TO_CLIENT"-\>"$PVS_NXM_NX_DIR_REG"[],resubmit\(,"$PVS_PORT_TABLE"\) - fi - done - # The following rules are independent of the in_port, so we'll - # need just one copy per VIF. We'll only apply them if the - # script is called for a vif interface, not for a tap interface, - # because tap interfaces are not always present, while vifs are. - if [ "${TYPE}" = "vif" ]; then - for ((i=PVS_STARTPORT; i<=PVS_ENDPORT; i++)) do - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_PORT_TABLE)),priority=$((PVS_RULE_PRIO)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB",udp,tp_dst="$i",actions=resubmit\(,"$PVS_ACTION_TABLE"\) - done - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO-1)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_CLIENT_TO_SERVER",actions="$PVS_PROXY_OFPORT" - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_PROXY_TO_CLIENT",actions=NORMAL - $OFCTL --strict add-flow "$PVS_BRIDGE" cookie=$PVS_IF_ID,table=$((PVS_ACTION_TABLE)),priority=$((PVS_RULE_PRIO-1)),"$PVS_CLIENT_ID_REG_LSB"="$PVS_IF_ID_LSB","$PVS_CLIENT_ID_REG_MSB"="$PVS_IF_ID_MSB","$PVS_DIR_REG"="$DIR_SERVER_TO_CLIENT",actions="$PVS_PROXY_OFPORT" - - fi - unset IFS - done - # Announce that on the OVS we have set up the rules for this VIF's pvs-proxy - $XSWRITE "${HOTPLUG_PATH}/pvs-rules-active" '' - ;; - remove) - path="${HOTPLUG_PATH}/$TYPE-ifindex" - PVS_IF_ID=$($XSREAD "$path") - if [ $? -ne 0 ] || [ -z "$PVS_IF_ID" ]; then - handle_error "The $PVS_VM_INTERFACE's ID[$PVS_IF_ID] not found" - fi - - logger -t "$LOG_TAG" "Removing rules from $PVS_BRIDGE for $PVS_PROXY_INTERFACE $PVS_VM_INTERFACE/$PVS_IF_ID" - - $OFCTL del-flows "$PVS_BRIDGE" cookie="$PVS_IF_ID"/-1 - if [ "${TYPE}" = "vif" ]; then - # Again, don't do the following when a tap goes away, because - # vif may still be there. - - # Announce that on the OVS we have removed the rules for this VIF's pvs-proxy. - $XSRM "${HOTPLUG_PATH}/pvs-rules-active" - fi - ;; - reset) - $OFCTL del-flows "$PVS_BRIDGE" - $OFCTL --strict add-flow "$PVS_BRIDGE" priority=0,actions=NORMAL - ;; - *) - handle_error "Unknown command '$1'" - ;; -esac diff --git a/ocaml/xenopsd/xc/xc_resources.ml b/ocaml/xenopsd/xc/xc_resources.ml index 5f9dba7f79f..2199fb04ab6 100644 --- a/ocaml/xenopsd/xc/xc_resources.ml +++ b/ocaml/xenopsd/xc/xc_resources.ml @@ -37,7 +37,7 @@ let ionice = ref "ionice" let setup_vif_rules = ref "setup-vif-rules" -let setup_pvs_proxy_rules = ref "setup-pvs-proxy-rules" +let setup_pvs_proxy_rules = ref "pvs-proxy-ovs-setup" let vgpu = ref "vgpu" diff --git a/ocaml/xenopsd/xenopsd.conf b/ocaml/xenopsd/xenopsd.conf index 46f95207c7d..353e758c0d3 100644 --- a/ocaml/xenopsd/xenopsd.conf +++ b/ocaml/xenopsd/xenopsd.conf @@ -79,8 +79,8 @@ disable-logging-for=http tracing # Path to the setup-vif-rules script # setup-vif-rules=/opt/xensource/libexec/setup-vif-rules -# Path to the setup-pvs-proxy-rules script -# setup-pvs-proxy-rules=/usr/libexec/xenopsd/setup-pvs-proxy-rules +# Path to the pvs-setup script +# setup-pvs-proxy-rules=/usr/libexec/xenopsd/pvs-proxy-ovs-setup # Paths to standard system utilities: # tune2fs=/sbin/tune2fs