From c4572a465b731a60e7dfc76e1c29640da4025640 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 30 May 2019 22:11:14 -0500 Subject: [PATCH 1/4] rdma: Add an option to query,set net namespace sharing sys parameter Enrich rdmatool with an option to query rdma subsystem parameter whether rdma devices are shared among multiple network namespaces or exclusive to single network namespace. rdma tool command examples and output. $ rdma system show netns shared $ rdma system set netns exclusive $ rdma system show netns exclusive Reviewed-by: Leon Romanovsky Signed-off-by: Parav Pandit Signed-off-by: David Ahern --- rdma/Makefile | 2 +- rdma/rdma.c | 3 +- rdma/rdma.h | 1 + rdma/sys.c | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++ rdma/utils.c | 1 + 5 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 rdma/sys.c diff --git a/rdma/Makefile b/rdma/Makefile index 6a424234..4847f27e 100644 --- a/rdma/Makefile +++ b/rdma/Makefile @@ -7,7 +7,7 @@ ifeq ($(HAVE_MNL),y) CFLAGS += -I./include/uapi/ RDMA_OBJ = rdma.o utils.o dev.o link.o res.o res-pd.o res-mr.o res-cq.o \ - res-cmid.o res-qp.o + res-cmid.o res-qp.o sys.o TARGETS += rdma endif diff --git a/rdma/rdma.c b/rdma/rdma.c index 676e03c2..e9f1b4bb 100644 --- a/rdma/rdma.c +++ b/rdma/rdma.c @@ -11,7 +11,7 @@ static void help(char *name) { pr_out("Usage: %s [ OPTIONS ] OBJECT { COMMAND | help }\n" " %s [ -f[orce] ] -b[atch] filename\n" - "where OBJECT := { dev | link | resource | help }\n" + "where OBJECT := { dev | link | resource | system | help }\n" " OPTIONS := { -V[ersion] | -d[etails] | -j[son] | -p[retty]}\n", name, name); } @@ -29,6 +29,7 @@ static int rd_cmd(struct rd *rd, int argc, char **argv) { "dev", cmd_dev }, { "link", cmd_link }, { "resource", cmd_res }, + { "system", cmd_sys }, { 0 } }; diff --git a/rdma/rdma.h b/rdma/rdma.h index 9ed9e045..885a751e 100644 --- a/rdma/rdma.h +++ b/rdma/rdma.h @@ -93,6 +93,7 @@ char *rd_argv(struct rd *rd); int cmd_dev(struct rd *rd); int cmd_link(struct rd *rd); int cmd_res(struct rd *rd); +int cmd_sys(struct rd *rd); int rd_exec_cmd(struct rd *rd, const struct rd_cmd *c, const char *str); int rd_exec_dev(struct rd *rd, int (*cb)(struct rd *rd)); int rd_exec_require_dev(struct rd *rd, int (*cb)(struct rd *rd)); diff --git a/rdma/sys.c b/rdma/sys.c new file mode 100644 index 00000000..cef39081 --- /dev/null +++ b/rdma/sys.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * sys.c RDMA tool + */ + +#include "rdma.h" + +static int sys_help(struct rd *rd) +{ + pr_out("Usage: %s system show [ netns ]\n", rd->filename); + pr_out(" %s system set netns { shared | exclusive }\n", rd->filename); + return 0; +} + +static const char *netns_modes_str[] = { + "exclusive", + "shared", +}; + +static int sys_show_parse_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {}; + struct rd *rd = data; + + mnl_attr_parse(nlh, 0, rd_attr_cb, tb); + + if (tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]) { + const char *mode_str; + uint8_t netns_mode; + + netns_mode = + mnl_attr_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]); + + if (netns_mode <= ARRAY_SIZE(netns_modes_str)) + mode_str = netns_modes_str[netns_mode]; + else + mode_str = "unknown"; + + if (rd->json_output) + jsonw_string_field(rd->jw, "netns", mode_str); + else + pr_out("netns %s\n", mode_str); + } + return MNL_CB_OK; +} + +static int sys_show_no_args(struct rd *rd) +{ + uint32_t seq; + int ret; + + rd_prepare_msg(rd, RDMA_NLDEV_CMD_SYS_GET, + &seq, (NLM_F_REQUEST | NLM_F_ACK)); + ret = rd_send_msg(rd); + if (ret) + return ret; + + return rd_recv_msg(rd, sys_show_parse_cb, rd, seq); +} + +static int sys_show(struct rd *rd) +{ + const struct rd_cmd cmds[] = { + { NULL, sys_show_no_args}, + { "netns", sys_show_no_args}, + { 0 } + }; + + return rd_exec_cmd(rd, cmds, "parameter"); +} + +static int sys_set_netns_cmd(struct rd *rd, bool enable) +{ + uint32_t seq; + + rd_prepare_msg(rd, RDMA_NLDEV_CMD_SYS_SET, + &seq, (NLM_F_REQUEST | NLM_F_ACK)); + mnl_attr_put_u8(rd->nlh, RDMA_NLDEV_SYS_ATTR_NETNS_MODE, enable); + + return rd_sendrecv_msg(rd, seq); +} + +static bool sys_valid_netns_cmd(const char *cmd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(netns_modes_str); i++) { + if (!strcmp(cmd, netns_modes_str[i])) + return true; + } + return false; +} + +static int sys_set_netns_args(struct rd *rd) +{ + bool cmd; + + if (rd_no_arg(rd) || !sys_valid_netns_cmd(rd_argv(rd))) { + pr_err("valid options are: { shared | exclusive }\n"); + return -EINVAL; + } + + cmd = (strcmp(rd_argv(rd), "shared") == 0) ? true : false; + + return sys_set_netns_cmd(rd, cmd); +} + +static int sys_set_help(struct rd *rd) +{ + pr_out("Usage: %s system set [PARAM] value\n", rd->filename); + pr_out(" system set netns { shared | exclusive }\n"); + return 0; +} + +static int sys_set(struct rd *rd) +{ + const struct rd_cmd cmds[] = { + { NULL, sys_set_help }, + { "help", sys_set_help }, + { "netns", sys_set_netns_args}, + { 0 } + }; + + return rd_exec_cmd(rd, cmds, "parameter"); +} + +int cmd_sys(struct rd *rd) +{ + const struct rd_cmd cmds[] = { + { NULL, sys_show }, + { "show", sys_show }, + { "set", sys_set }, + { "help", sys_help }, + { 0 } + }; + + return rd_exec_cmd(rd, cmds, "system command"); +} diff --git a/rdma/utils.c b/rdma/utils.c index 11ed8a73..558d1c29 100644 --- a/rdma/utils.c +++ b/rdma/utils.c @@ -435,6 +435,7 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_DRIVER_U32] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_DRIVER_S64] = MNL_TYPE_U64, [RDMA_NLDEV_ATTR_DRIVER_U64] = MNL_TYPE_U64, + [RDMA_NLDEV_SYS_ATTR_NETNS_MODE] = MNL_TYPE_U8, }; int rd_attr_check(const struct nlattr *attr, int *typep) From e86127201504d00987d89c9c48eebc6b0c7eb2c8 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 30 May 2019 22:11:15 -0500 Subject: [PATCH 2/4] rdma: Add man pages for rdma system commands Reviewed-by: Leon Romanovsky Signed-off-by: Parav Pandit Signed-off-by: David Ahern --- man/man8/rdma-system.8 | 82 ++++++++++++++++++++++++++++++++++++++++++ man/man8/rdma.8 | 7 +++- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 man/man8/rdma-system.8 diff --git a/man/man8/rdma-system.8 b/man/man8/rdma-system.8 new file mode 100644 index 00000000..ab1d89fd --- /dev/null +++ b/man/man8/rdma-system.8 @@ -0,0 +1,82 @@ +.TH RDMA\-SYSTEM 8 "06 Jul 2017" "iproute2" "Linux" +.SH NAME +rdma-system \- RDMA subsystem configuration +.SH SYNOPSIS +.sp +.ad l +.in +8 +.ti -8 +.B rdma +.RI "[ " OPTIONS " ]" +.B sys +.RI " { " COMMAND " | " +.BR help " }" +.sp + +.ti -8 +.IR OPTIONS " := { " +\fB\-V\fR[\fIersion\fR] | +\fB\-d\fR[\fIetails\fR] } + +.ti -8 +.B rdma system show + +.ti -8 +.B rdma system set +.BR netns +.BR NEWMODE + +.ti -8 +.B rdma system help + +.SH "DESCRIPTION" +.SS rdma system set - set RDMA subsystem network namespace mode + +.SS rdma system show - display RDMA subsystem network namespace mode + +.PP +.I "NEWMODE" +- specifies the RDMA subsystem mode. Either exclusive or shared. +When user wants to assign dedicated RDMA device to a particular +network namespace, exclusive mode should be set before creating +any network namespace. If there are active network namespaces and if +one or more RDMA devices exist, changing mode from shared to +exclusive returns error code EBUSY. + +When RDMA subsystem is in shared mode, RDMA device is accessible in +all network namespace. When RDMA device isolation among multiple +network namespaces is not needed, shared mode can be used. + +It is preferred to not change the subsystem mode when there is active +RDMA traffic running, even though it is supported. + +.SH "EXAMPLES" +.PP +rdma system show +.RS 4 +Shows the state of RDMA subsystem network namespace mode on the system. +.RE +.PP +rdma system set netns exclusive +.RS 4 +Sets the RDMA subsystem in network namespace exclusive mode. In this mode RDMA devices +are visible only in single network namespace. +.RE +.PP +rdma system set netns shared +.RS 4 +Sets the RDMA subsystem in network namespace shared mode. In this mode RDMA devices +are shared among network namespaces. +.RE +.PP + +.SH SEE ALSO +.BR rdma (8), +.BR rdma-link (8), +.BR rdma-resource (8), +.BR network_namespaces (7), +.BR namespaces (7), +.br + +.SH AUTHOR +Parav Pandit diff --git a/man/man8/rdma.8 b/man/man8/rdma.8 index b2b5aef8..3ae33987 100644 --- a/man/man8/rdma.8 +++ b/man/man8/rdma.8 @@ -19,7 +19,7 @@ rdma \- RDMA tool .ti -8 .IR OBJECT " := { " -.BR dev " | " link " }" +.BR dev " | " link " | " system " }" .sp .ti -8 @@ -70,6 +70,10 @@ Generate JSON output. .B link - RDMA port related. +.TP +.B sys +- RDMA subsystem related. + .PP The names of all objects may be written in full or abbreviated form, for example @@ -107,6 +111,7 @@ Exit status is 0 if command was successful or a positive integer upon failure. .BR rdma-dev (8), .BR rdma-link (8), .BR rdma-resource (8), +.BR rdma-system (8), .br .SH REPORTING BUGS From d17a0248a287cabf59936bc34b0e1b960ae83f46 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 30 May 2019 22:11:16 -0500 Subject: [PATCH 3/4] rdma: Add an option to set net namespace of rdma device Enrich rdmatool with an option to set network namespace of RDMA device. After successful execution of it, rdma device will be accessible only in assigned network namespace. rdma tool command examples and output. First set netns mode to exclusive. $ rdma system set netns exclusive Now create network namespace and assign RDMA device to this network namespace. $ ip netns add foo $ rdma dev set mlx5_1 netns foo Reviewed-by: Leon Romanovsky Signed-off-by: Parav Pandit Signed-off-by: David Ahern --- rdma/dev.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/rdma/dev.c b/rdma/dev.c index 90483622..d28bf6b3 100644 --- a/rdma/dev.c +++ b/rdma/dev.c @@ -4,12 +4,14 @@ * Authors: Leon Romanovsky */ +#include #include "rdma.h" static int dev_help(struct rd *rd) { pr_out("Usage: %s dev show [DEV]\n", rd->filename); pr_out(" %s dev set [DEV] name DEVNAME\n", rd->filename); + pr_out(" %s dev set [DEV] netns NSNAME\n", rd->filename); return 0; } @@ -272,11 +274,46 @@ static int dev_set_name(struct rd *rd) return rd_sendrecv_msg(rd, seq); } +static int dev_set_netns(struct rd *rd) +{ + char *netns_path; + uint32_t seq; + int netns; + int ret; + + if (rd_no_arg(rd)) { + pr_err("Please provide device name.\n"); + return -EINVAL; + } + + if (asprintf(&netns_path, "%s/%s", NETNS_RUN_DIR, rd_argv(rd)) < 0) + return -ENOMEM; + + netns = open(netns_path, O_RDONLY | O_CLOEXEC); + if (netns < 0) { + fprintf(stderr, "Cannot open network namespace \"%s\": %s\n", + rd_argv(rd), strerror(errno)); + ret = -EINVAL; + goto done; + } + + rd_prepare_msg(rd, RDMA_NLDEV_CMD_SET, + &seq, (NLM_F_REQUEST | NLM_F_ACK)); + mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_ATTR_DEV_INDEX, rd->dev_idx); + mnl_attr_put_u32(rd->nlh, RDMA_NLDEV_NET_NS_FD, netns); + ret = rd_sendrecv_msg(rd, seq); + close(netns); +done: + free(netns_path); + return ret; +} + static int dev_one_set(struct rd *rd) { const struct rd_cmd cmds[] = { { NULL, dev_help}, { "name", dev_set_name}, + { "netns", dev_set_netns}, { 0 } }; From c2ffce5d397ae53ab825134cf55d716858407437 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Thu, 30 May 2019 22:11:17 -0500 Subject: [PATCH 4/4] rdma: Add man page for rdma dev set netns command Add man page to describe additional set netns command for rdma device. Reviewed-by: Leon Romanovsky Signed-off-by: Parav Pandit Signed-off-by: David Ahern --- man/man8/rdma-dev.8 | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/man/man8/rdma-dev.8 b/man/man8/rdma-dev.8 index 069f4717..38e34b3b 100644 --- a/man/man8/rdma-dev.8 +++ b/man/man8/rdma-dev.8 @@ -28,13 +28,19 @@ rdma-dev \- RDMA device configuration .BR name .BR NEWNAME +.ti -8 +.B rdma dev set +.RI "[ " DEV " ]" +.BR netns +.BR NSNAME + .ti -8 .B rdma dev help .SH "DESCRIPTION" -.SS rdma dev set - rename rdma device +.SS rdma dev set - rename RDMA device or set network namespace -.SS rdma dev show - display rdma device attributes +.SS rdma dev show - display RDMA device attributes .PP .I "DEV" @@ -58,11 +64,19 @@ rdma dev set mlx5_3 name rdma_0 Renames the mlx5_3 device to rdma_0. .RE .PP +rdma dev set mlx5_3 netns foo +.RS 4 +Changes the network namespace of RDMA device to foo where foo is +previously created using iproute2 ip command. +.RE +.PP .SH SEE ALSO +.BR ip (8), .BR rdma (8), .BR rdma-link (8), .BR rdma-resource (8), +.BR rdma-system (8), .br .SH AUTHOR