From 5a93929d9f9a1d82946ddd49e260b6dd1756ad6d Mon Sep 17 00:00:00 2001 From: Santosh Pradhan Date: Thu, 18 Aug 2022 12:52:39 +0200 Subject: RDMA/rtrs-clt: Add event tracing support Add event tracing mechanism for following routines: - rtrs_clt_reconnect_work() - rtrs_clt_close_conns() - rtrs_rdma_error_recovery() How to use: 1. Load the rtrs_client module 2. cd /sys/kernel/debug/tracing 3. If all the events need to be enabled: echo 1 > events/rtrs_clt/enable 4. OR only speific routine/event needs to be enabled e.g. echo 1 > events/rtrs_clt/rtrs_clt_close_conns/enable 5. cat trace 6. Run some workload which can trigger rtrs_clt_close_conns() Link: https://lore.kernel.org/r/20220818105240.110234-2-haris.iqbal@ionos.com Signed-off-by: Santosh Pradhan Signed-off-by: Jack Wang Signed-off-by: Md Haris Iqbal Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/rtrs/Makefile | 5 +- drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c | 15 +++++ drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h | 86 ++++++++++++++++++++++++++++ drivers/infiniband/ulp/rtrs/rtrs-clt.c | 7 +++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h (limited to 'drivers/infiniband/ulp/rtrs') diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile index 3898509be270..1fdf918b37eb 100644 --- a/drivers/infiniband/ulp/rtrs/Makefile +++ b/drivers/infiniband/ulp/rtrs/Makefile @@ -1,8 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-or-later +CFLAGS_rtrs-clt-trace.o = -I$(src) + rtrs-client-y := rtrs-clt.o \ rtrs-clt-stats.o \ - rtrs-clt-sysfs.o + rtrs-clt-sysfs.o \ + rtrs-clt-trace.o rtrs-server-y := rtrs-srv.o \ rtrs-srv-stats.o \ diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c new file mode 100644 index 000000000000..f14fa1f36ce8 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#include "rtrs.h" +#include "rtrs-clt.h" + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "rtrs-clt-trace.h" diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h new file mode 100644 index 000000000000..7738e2676855 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rtrs_clt + +#if !defined(_TRACE_RTRS_CLT_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RTRS_CLT_H + +#include + +struct rtrs_clt_path; +struct rtrs_clt_sess; + +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING); +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING_ERR); +TRACE_DEFINE_ENUM(RTRS_CLT_RECONNECTING); +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTED); +TRACE_DEFINE_ENUM(RTRS_CLT_CLOSING); +TRACE_DEFINE_ENUM(RTRS_CLT_CLOSED); +TRACE_DEFINE_ENUM(RTRS_CLT_DEAD); + +#define show_rtrs_clt_state(x) \ + __print_symbolic(x, \ + { RTRS_CLT_CONNECTING, "CONNECTING" }, \ + { RTRS_CLT_CONNECTING_ERR, "CONNECTING_ERR" }, \ + { RTRS_CLT_RECONNECTING, "RECONNECTING" }, \ + { RTRS_CLT_CONNECTED, "CONNECTED" }, \ + { RTRS_CLT_CLOSING, "CLOSING" }, \ + { RTRS_CLT_CLOSED, "CLOSED" }, \ + { RTRS_CLT_DEAD, "DEAD" }) + +DECLARE_EVENT_CLASS(rtrs_clt_conn_class, + TP_PROTO(struct rtrs_clt_path *clt_path), + + TP_ARGS(clt_path), + + TP_STRUCT__entry( + __field(int, state) + __field(int, reconnect_attempts) + __field(int, max_reconnect_attempts) + __field(int, fail_cnt) + __field(int, success_cnt) + __array(char, sessname, NAME_MAX) + ), + + TP_fast_assign( + struct rtrs_clt_sess *clt = clt_path->clt; + + __entry->state = clt_path->state; + __entry->reconnect_attempts = clt_path->reconnect_attempts; + __entry->max_reconnect_attempts = clt->max_reconnect_attempts; + __entry->fail_cnt = clt_path->stats->reconnects.fail_cnt; + __entry->success_cnt = clt_path->stats->reconnects.successful_cnt; + memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX); + ), + + TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'", + __entry->sessname, + show_rtrs_clt_state(__entry->state), + __entry->reconnect_attempts, + __entry->max_reconnect_attempts, + __entry->fail_cnt, + __entry->success_cnt + ) +); + +#define DEFINE_CLT_CONN_EVENT(name) \ +DEFINE_EVENT(rtrs_clt_conn_class, rtrs_##name, \ + TP_PROTO(struct rtrs_clt_path *clt_path), \ + TP_ARGS(clt_path)) + +DEFINE_CLT_CONN_EVENT(clt_reconnect_work); +DEFINE_CLT_CONN_EVENT(clt_close_conns); +DEFINE_CLT_CONN_EVENT(rdma_error_recovery); + +#endif /* _TRACE_RTRS_CLT_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE rtrs-clt-trace +#include + diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index baecde41d126..5219bb10777a 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -16,6 +16,7 @@ #include "rtrs-clt.h" #include "rtrs-log.h" +#include "rtrs-clt-trace.h" #define RTRS_CONNECT_TIMEOUT_MS 30000 /* @@ -302,6 +303,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) { struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + trace_rtrs_rdma_error_recovery(clt_path); + if (rtrs_clt_change_state_from_to(clt_path, RTRS_CLT_CONNECTED, RTRS_CLT_RECONNECTING)) { @@ -1942,6 +1945,8 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) { + trace_rtrs_clt_close_conns(clt_path); + if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL)) queue_work(rtrs_wq, &clt_path->close_work); if (wait) @@ -2648,6 +2653,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work) reconnect_dwork); clt = clt_path->clt; + trace_rtrs_clt_reconnect_work(clt_path); + if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING) return; -- cgit v1.2.3