aboutsummaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
diff options
context:
space:
mode:
authorGravatar Aya Levin <ayal@nvidia.com> 2021-02-12 14:30:42 -0800
committerGravatar Saeed Mahameed <saeedm@nvidia.com> 2021-02-16 14:04:54 -0800
commit432119de33d9013467371fc85238d623f64ff67e (patch)
tree353a4da5fd72b04f474690795ce7a8c167969802 /drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
parentnet/mlx5: Move some PPS logic into helper functions (diff)
downloadlinux-432119de33d9013467371fc85238d623f64ff67e.tar.gz
linux-432119de33d9013467371fc85238d623f64ff67e.tar.bz2
linux-432119de33d9013467371fc85238d623f64ff67e.zip
net/mlx5: Add cyc2time HW translation mode support
Device timestamp can be in real time mode (cycles to time translation is offloaded into the Hardware). With real time mode, HW provides timestamp which is already translated into nanoseconds. With this mode, driver adjusts both the HW and timecounter (to keep clock_info_page updated) using callbacks: adjfreq, adjtime and settime. HW clock modifications are done via MTUTC access reg commands. Driver is allowed to modify HW real time clock only if MCAM ptpcyc2realtime_modify capability is set. Add MTUTC set function to be used for configuring the HW real time clock. Modify existing code to support both internal timer (with conversion via timecounter_cyc2time() and real time (no conversions). Align the signatures of the helpers converting from timestamp to nanoseconds. With that, when allocating a queue assign the corresponding callback with respect to the capability. Adjust 1PPS timestamp calculation flows based on the timestamp mode. Cyc2time offload brings two major advantages: - Improve MTAE (Max Time Absolute Error) for HW TS by up to 160 ns over a 100% loaded CPU. - Faster data-path timestamp to nanoseconds, as translation is lock-less and done in HW. On real time mode, timestamp format is 32 high bits of seconds and 32 low bits of nanoseconds. On some flows, driver shall convert this format into nanoseconds wall-clock with REAL_TIME_TO_NS macro. HW supports a single clock, and it is shared by all functions on a device. In case real time clock is used, it is recommended to use a single GM to all device's functions. Signed-off-by: Eran Ben Elisha <eranbe@mellanox.com> Signed-off-by: Aya Levin <ayal@nvidia.com> Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c')
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c197
1 files changed, 171 insertions, 26 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index dbb6138db096..b0e129d0f6d8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -67,24 +67,51 @@ enum {
MLX5_MTPPS_FS_ENH_OUT_PER_ADJ = BIT(0x7),
};
-static u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev,
- struct ptp_system_timestamp *sts)
+static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev)
+{
+ return (mlx5_is_real_time_rq(mdev) || mlx5_is_real_time_sq(mdev));
+}
+
+static bool mlx5_modify_mtutc_allowed(struct mlx5_core_dev *mdev)
+{
+ return MLX5_CAP_MCAM_FEATURE(mdev, ptpcyc2realtime_modify);
+}
+
+static int mlx5_set_mtutc(struct mlx5_core_dev *dev, u32 *mtutc, u32 size)
+{
+ u32 out[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+
+ if (!MLX5_CAP_MCAM_REG(dev, mtutc))
+ return -EOPNOTSUPP;
+
+ return mlx5_core_access_reg(dev, mtutc, size, out, sizeof(out),
+ MLX5_REG_MTUTC, 0, 1);
+}
+
+static u64 mlx5_read_time(struct mlx5_core_dev *dev,
+ struct ptp_system_timestamp *sts,
+ bool real_time)
{
u32 timer_h, timer_h1, timer_l;
- timer_h = ioread32be(&dev->iseg->internal_timer_h);
+ timer_h = ioread32be(real_time ? &dev->iseg->real_time_h :
+ &dev->iseg->internal_timer_h);
ptp_read_system_prets(sts);
- timer_l = ioread32be(&dev->iseg->internal_timer_l);
+ timer_l = ioread32be(real_time ? &dev->iseg->real_time_l :
+ &dev->iseg->internal_timer_l);
ptp_read_system_postts(sts);
- timer_h1 = ioread32be(&dev->iseg->internal_timer_h);
+ timer_h1 = ioread32be(real_time ? &dev->iseg->real_time_h :
+ &dev->iseg->internal_timer_h);
if (timer_h != timer_h1) {
/* wrap around */
ptp_read_system_prets(sts);
- timer_l = ioread32be(&dev->iseg->internal_timer_l);
+ timer_l = ioread32be(real_time ? &dev->iseg->real_time_l :
+ &dev->iseg->internal_timer_l);
ptp_read_system_postts(sts);
}
- return (u64)timer_l | (u64)timer_h1 << 32;
+ return real_time ? REAL_TIME_TO_NS(timer_h1, timer_l) :
+ (u64)timer_l | (u64)timer_h1 << 32;
}
static u64 read_internal_timer(const struct cyclecounter *cc)
@@ -94,7 +121,7 @@ static u64 read_internal_timer(const struct cyclecounter *cc)
struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev,
clock);
- return mlx5_read_internal_timer(mdev, NULL) & cc->mask;
+ return mlx5_read_time(mdev, NULL, false) & cc->mask;
}
static void mlx5_update_clock_info_page(struct mlx5_core_dev *mdev)
@@ -169,23 +196,58 @@ static void mlx5_timestamp_overflow(struct work_struct *work)
schedule_delayed_work(&timer->overflow_work, timer->overflow_period);
}
+static int mlx5_ptp_settime_real_time(struct mlx5_core_dev *mdev,
+ const struct timespec64 *ts)
+{
+ u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+
+ if (!mlx5_modify_mtutc_allowed(mdev))
+ return 0;
+
+ if (ts->tv_sec < 0 || ts->tv_sec > U32_MAX ||
+ ts->tv_nsec < 0 || ts->tv_nsec > NSEC_PER_SEC)
+ return -EINVAL;
+
+ MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_SET_TIME_IMMEDIATE);
+ MLX5_SET(mtutc_reg, in, utc_sec, ts->tv_sec);
+ MLX5_SET(mtutc_reg, in, utc_nsec, ts->tv_nsec);
+
+ return mlx5_set_mtutc(mdev, in, sizeof(in));
+}
+
static int mlx5_ptp_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts)
{
struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info);
struct mlx5_timer *timer = &clock->timer;
- u64 ns = timespec64_to_ns(ts);
struct mlx5_core_dev *mdev;
unsigned long flags;
+ int err;
mdev = container_of(clock, struct mlx5_core_dev, clock);
+ err = mlx5_ptp_settime_real_time(mdev, ts);
+ if (err)
+ return err;
+
write_seqlock_irqsave(&clock->lock, flags);
- timecounter_init(&timer->tc, &timer->cycles, ns);
+ timecounter_init(&timer->tc, &timer->cycles, timespec64_to_ns(ts));
mlx5_update_clock_info_page(mdev);
write_sequnlock_irqrestore(&clock->lock, flags);
return 0;
}
+static
+struct timespec64 mlx5_ptp_gettimex_real_time(struct mlx5_core_dev *mdev,
+ struct ptp_system_timestamp *sts)
+{
+ struct timespec64 ts;
+ u64 time;
+
+ time = mlx5_read_time(mdev, sts, true);
+ ts = ns_to_timespec64(time);
+ return ts;
+}
+
static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts,
struct ptp_system_timestamp *sts)
{
@@ -196,24 +258,57 @@ static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts,
u64 cycles, ns;
mdev = container_of(clock, struct mlx5_core_dev, clock);
+ if (mlx5_real_time_mode(mdev)) {
+ *ts = mlx5_ptp_gettimex_real_time(mdev, sts);
+ goto out;
+ }
+
write_seqlock_irqsave(&clock->lock, flags);
- cycles = mlx5_read_internal_timer(mdev, sts);
+ cycles = mlx5_read_time(mdev, sts, false);
ns = timecounter_cyc2time(&timer->tc, cycles);
write_sequnlock_irqrestore(&clock->lock, flags);
-
*ts = ns_to_timespec64(ns);
-
+out:
return 0;
}
+static int mlx5_ptp_adjtime_real_time(struct mlx5_core_dev *mdev, s64 delta)
+{
+ u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+
+ if (!mlx5_modify_mtutc_allowed(mdev))
+ return 0;
+
+ /* HW time adjustment range is s16. If out of range, settime instead */
+ if (delta < S16_MIN || delta > S16_MAX) {
+ struct timespec64 ts;
+ s64 ns;
+
+ ts = mlx5_ptp_gettimex_real_time(mdev, NULL);
+ ns = timespec64_to_ns(&ts) + delta;
+ ts = ns_to_timespec64(ns);
+ return mlx5_ptp_settime_real_time(mdev, &ts);
+ }
+
+ MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_TIME);
+ MLX5_SET(mtutc_reg, in, time_adjustment, delta);
+
+ return mlx5_set_mtutc(mdev, in, sizeof(in));
+}
+
static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
{
struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info);
struct mlx5_timer *timer = &clock->timer;
struct mlx5_core_dev *mdev;
unsigned long flags;
+ int err;
mdev = container_of(clock, struct mlx5_core_dev, clock);
+
+ err = mlx5_ptp_adjtime_real_time(mdev, delta);
+ if (err)
+ return err;
write_seqlock_irqsave(&clock->lock, flags);
timecounter_adjtime(&timer->tc, delta);
mlx5_update_clock_info_page(mdev);
@@ -222,6 +317,19 @@ static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
return 0;
}
+static int mlx5_ptp_adjfreq_real_time(struct mlx5_core_dev *mdev, s32 freq)
+{
+ u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+
+ if (!mlx5_modify_mtutc_allowed(mdev))
+ return 0;
+
+ MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_FREQ_UTC);
+ MLX5_SET(mtutc_reg, in, freq_adjustment, freq);
+
+ return mlx5_set_mtutc(mdev, in, sizeof(in));
+}
+
static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta)
{
struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info);
@@ -231,6 +339,12 @@ static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta)
int neg_adj = 0;
u32 diff;
u64 adj;
+ int err;
+
+ mdev = container_of(clock, struct mlx5_core_dev, clock);
+ err = mlx5_ptp_adjfreq_real_time(mdev, delta);
+ if (err)
+ return err;
if (delta < 0) {
neg_adj = 1;
@@ -241,7 +355,6 @@ static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta)
adj *= delta;
diff = div_u64(adj, 1000000000ULL);
- mdev = container_of(clock, struct mlx5_core_dev, clock);
write_seqlock_irqsave(&clock->lock, flags);
timecounter_read(&timer->tc);
timer->cycles.mult = neg_adj ? timer->nominal_c_mult - diff :
@@ -324,7 +437,7 @@ static u64 find_target_cycles(struct mlx5_core_dev *mdev, s64 target_ns)
timer = &clock->timer;
- cycles_now = mlx5_read_internal_timer(mdev, NULL);
+ cycles_now = mlx5_read_time(mdev, NULL, false);
write_seqlock_irqsave(&clock->lock, flags);
nsec_now = timecounter_cyc2time(&timer->tc, cycles_now);
nsec_delta = target_ns - nsec_now;
@@ -348,6 +461,11 @@ static u64 perout_conf_internal_timer(struct mlx5_core_dev *mdev,
return find_target_cycles(mdev, target_ns);
}
+static u64 perout_conf_real_time(s64 sec, u32 nsec)
+{
+ return (u64)nsec | (u64)sec << 32;
+}
+
static int mlx5_perout_configure(struct ptp_clock_info *ptp,
struct ptp_clock_request *rq,
int on)
@@ -378,6 +496,7 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp,
field_select = MLX5_MTPPS_FS_ENABLE;
if (on) {
+ bool rt_mode = mlx5_real_time_mode(mdev);
u32 nsec;
s64 sec;
@@ -397,7 +516,11 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp,
nsec = rq->perout.start.nsec;
sec = rq->perout.start.sec;
- time_stamp = perout_conf_internal_timer(mdev, sec, nsec);
+ if (rt_mode && sec > U32_MAX)
+ return -EINVAL;
+
+ time_stamp = rt_mode ? perout_conf_real_time(sec, nsec) :
+ perout_conf_internal_timer(mdev, sec, nsec);
field_select |= MLX5_MTPPS_FS_PIN_MODE |
MLX5_MTPPS_FS_PATTERN |
@@ -578,17 +701,23 @@ static void ts_next_sec(struct timespec64 *ts)
ts->tv_nsec = 0;
}
-static u64 perout_conf_next_event_inernal_timer(struct mlx5_core_dev *mdev,
- struct mlx5_clock *clock)
+static u64 perout_conf_next_event_timer(struct mlx5_core_dev *mdev,
+ struct mlx5_clock *clock)
{
+ bool rt_mode = mlx5_real_time_mode(mdev);
struct timespec64 ts;
s64 target_ns;
- mlx5_ptp_gettimex(&clock->ptp_info, &ts, NULL);
+ if (rt_mode)
+ ts = mlx5_ptp_gettimex_real_time(mdev, NULL);
+ else
+ mlx5_ptp_gettimex(&clock->ptp_info, &ts, NULL);
+
ts_next_sec(&ts);
target_ns = timespec64_to_ns(&ts);
- return find_target_cycles(mdev, target_ns);
+ return rt_mode ? perout_conf_real_time(ts.tv_sec, ts.tv_nsec) :
+ find_target_cycles(mdev, target_ns);
}
static int mlx5_pps_event(struct notifier_block *nb,
@@ -607,7 +736,9 @@ static int mlx5_pps_event(struct notifier_block *nb,
switch (clock->ptp_info.pin_config[pin].func) {
case PTP_PF_EXTTS:
ptp_event.index = pin;
- ptp_event.timestamp =
+ ptp_event.timestamp = mlx5_real_time_mode(mdev) ?
+ mlx5_real_time_cyc2time(clock,
+ be64_to_cpu(eqe->data.pps.time_stamp)) :
mlx5_timecounter_cyc2time(clock,
be64_to_cpu(eqe->data.pps.time_stamp));
if (clock->pps_info.enabled) {
@@ -621,7 +752,7 @@ static int mlx5_pps_event(struct notifier_block *nb,
ptp_clock_event(clock->ptp, &ptp_event);
break;
case PTP_PF_PEROUT:
- ns = perout_conf_next_event_inernal_timer(mdev, clock);
+ ns = perout_conf_next_event_timer(mdev, clock);
write_seqlock_irqsave(&clock->lock, flags);
clock->pps_info.start[pin] = ns;
write_sequnlock_irqrestore(&clock->lock, flags);
@@ -711,6 +842,23 @@ static void mlx5_init_clock_info(struct mlx5_core_dev *mdev)
info->frac = timer->tc.frac;
}
+static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev)
+{
+ struct mlx5_clock *clock = &mdev->clock;
+
+ mlx5_timecounter_init(mdev);
+ mlx5_init_clock_info(mdev);
+ mlx5_init_overflow_period(clock);
+ clock->ptp_info = mlx5_ptp_clock_info;
+
+ if (mlx5_real_time_mode(mdev)) {
+ struct timespec64 ts;
+
+ ktime_get_real_ts64(&ts);
+ mlx5_ptp_settime(&clock->ptp_info, &ts);
+ }
+}
+
void mlx5_init_clock(struct mlx5_core_dev *mdev)
{
struct mlx5_clock *clock = &mdev->clock;
@@ -721,10 +869,7 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev)
}
seqlock_init(&clock->lock);
-
- mlx5_timecounter_init(mdev);
- mlx5_init_clock_info(mdev);
- mlx5_init_overflow_period(clock);
+ mlx5_init_timer_clock(mdev);
INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out);
/* Configure the PHC */