diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox')
62 files changed, 1392 insertions, 785 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 6152f77dcfd8..277738c50c56 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -4031,7 +4031,6 @@ static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) goto err_params_unregister; pci_save_state(pdev); - devlink_set_features(devlink, DEVLINK_F_RELOAD); devl_unlock(devlink); devlink_register(devlink); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 382d02f6619c..b00e33ed05e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -104,6 +104,7 @@ static bool mlx5_cmd_is_throttle_opcode(u16 op) case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: + case MLX5_CMD_OP_SYNC_CRYPTO: return true; } return false; @@ -523,6 +524,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE: case MLX5_CMD_OP_SAVE_VHCA_STATE: case MLX5_CMD_OP_LOAD_VHCA_STATE: + case MLX5_CMD_OP_SYNC_CRYPTO: *status = MLX5_DRIVER_STATUS_ABORTED; *synd = MLX5_DRIVER_SYND; return -ENOLINK; @@ -725,6 +727,7 @@ const char *mlx5_command_str(int command) MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE); MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE); MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE); + MLX5_COMMAND_STR_CASE(SYNC_CRYPTO); default: return "unknown command opcode"; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index 3e232a65a0c3..bb95b40d25eb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -245,8 +245,9 @@ void mlx5_pages_debugfs_init(struct mlx5_core_dev *dev) pages = dev->priv.dbg.pages_debugfs; debugfs_create_u32("fw_pages_total", 0400, pages, &dev->priv.fw_pages); - debugfs_create_u32("fw_pages_vfs", 0400, pages, &dev->priv.vfs_pages); - debugfs_create_u32("fw_pages_host_pf", 0400, pages, &dev->priv.host_pf_pages); + debugfs_create_u32("fw_pages_vfs", 0400, pages, &dev->priv.page_counters[MLX5_VF]); + debugfs_create_u32("fw_pages_sfs", 0400, pages, &dev->priv.page_counters[MLX5_SF]); + debugfs_create_u32("fw_pages_host_pf", 0400, pages, &dev->priv.page_counters[MLX5_HOST_PF]); debugfs_create_u32("fw_pages_alloc_failed", 0400, pages, &dev->priv.fw_pages_alloc_failed); debugfs_create_u32("fw_pages_give_dropped", 0400, pages, &dev->priv.give_pages_dropped); debugfs_create_u32("fw_pages_reclaim_discard", 0400, pages, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 95a69544a685..b742e04deec1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -156,6 +156,11 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, return -EOPNOTSUPP; } + if (mlx5_core_is_mp_slave(dev)) { + NL_SET_ERR_MSG_MOD(extack, "reload is unsupported for multi port slave"); + return -EOPNOTSUPP; + } + if (pci_num_vf(pdev)) { NL_SET_ERR_MSG_MOD(extack, "reload while VFs are present is unfavorable"); } @@ -263,9 +268,10 @@ static int mlx5_devlink_trap_action_set(struct devlink *devlink, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_devlink_trap_event_ctx trap_event_ctx; enum devlink_trap_action action_orig; struct mlx5_devlink_trap *dl_trap; - int err = 0; + int err; if (is_mdev_switchdev_mode(dev)) { NL_SET_ERR_MSG_MOD(extack, "Devlink traps can't be set in switchdev mode"); @@ -275,26 +281,25 @@ static int mlx5_devlink_trap_action_set(struct devlink *devlink, dl_trap = mlx5_find_trap_by_id(dev, trap->id); if (!dl_trap) { mlx5_core_err(dev, "Devlink trap: Set action on invalid trap id 0x%x", trap->id); - err = -EINVAL; - goto out; + return -EINVAL; } - if (action != DEVLINK_TRAP_ACTION_DROP && action != DEVLINK_TRAP_ACTION_TRAP) { - err = -EOPNOTSUPP; - goto out; - } + if (action != DEVLINK_TRAP_ACTION_DROP && action != DEVLINK_TRAP_ACTION_TRAP) + return -EOPNOTSUPP; if (action == dl_trap->trap.action) - goto out; + return 0; action_orig = dl_trap->trap.action; dl_trap->trap.action = action; + trap_event_ctx.trap = &dl_trap->trap; + trap_event_ctx.err = 0; err = mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_TYPE_TRAP, - &dl_trap->trap); - if (err) + &trap_event_ctx); + if (err == NOTIFY_BAD) dl_trap->trap.action = action_orig; -out: - return err; + + return trap_event_ctx.err; } static const struct devlink_ops mlx5_devlink_ops = { @@ -744,7 +749,6 @@ void mlx5_devlink_traps_unregister(struct devlink *devlink) int mlx5_devlink_params_register(struct devlink *devlink) { - struct mlx5_core_dev *dev = devlink_priv(devlink); int err; err = devl_params_register(devlink, mlx5_devlink_params, @@ -762,9 +766,6 @@ int mlx5_devlink_params_register(struct devlink *devlink) if (err) goto max_uc_list_err; - if (!mlx5_core_is_mp_slave(dev)) - devlink_set_features(devlink, DEVLINK_F_RELOAD); - return 0; max_uc_list_err: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index 1c1b62ee84bb..b561107e0df1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -24,6 +24,11 @@ struct mlx5_devlink_trap { struct list_head list; }; +struct mlx5_devlink_trap_event_ctx { + struct mlx5_trap_ctx *trap; + int err; +}; + struct mlx5_core_dev; void mlx5_devlink_trap_report(struct mlx5_core_dev *dev, int trap_id, struct sk_buff *skb, struct devlink_port *dl_port); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index 21831386b26e..f40497823e65 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -64,6 +64,7 @@ static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer) MLX5_GET(mtrc_cap, out, num_string_trace); tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db); tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); + tracer->str_db.loaded = false; for (i = 0; i < tracer->str_db.num_string_db; i++) { mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]); @@ -233,6 +234,8 @@ static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer) int i; for (i = 0; i < num_string_db; i++) { + if (!string_db_size_out[i]) + continue; tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL); if (!tracer->str_db.buffer[i]) goto free_strings_db; @@ -278,6 +281,8 @@ static void mlx5_tracer_read_strings_db(struct work_struct *work) } for (i = 0; i < num_string_db; i++) { + if (!tracer->str_db.size_out[i]) + continue; offset = 0; MLX5_SET(mtrc_stdb, in, string_db_index, i); num_of_reads = tracer->str_db.size_out[i] / @@ -384,6 +389,8 @@ static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer str_ptr = tracer_event->string_event.string_param; for (i = 0; i < tracer->str_db.num_string_db; i++) { + if (!tracer->str_db.size_out[i]) + continue; if (str_ptr > tracer->str_db.base_address_out[i] && str_ptr < tracer->str_db.base_address_out[i] + tracer->str_db.size_out[i]) { @@ -459,6 +466,7 @@ static void poll_trace(struct mlx5_fw_tracer *tracer, tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id); tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost); + tracer_event->out = trace; switch (tracer_event->event_id) { case TRACER_EVENT_TYPE_TIMESTAMP: @@ -581,6 +589,26 @@ void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt, mlx5_tracer_clean_message(str_frmt); } +static int mlx5_tracer_handle_raw_string(struct mlx5_fw_tracer *tracer, + struct tracer_event *tracer_event) +{ + struct tracer_string_format *cur_string; + + cur_string = mlx5_tracer_message_insert(tracer, tracer_event); + if (!cur_string) + return -1; + + cur_string->event_id = tracer_event->event_id; + cur_string->timestamp = tracer_event->string_event.timestamp; + cur_string->lost = tracer_event->lost_event; + cur_string->string = "0x%08x%08x"; + cur_string->num_of_params = 2; + cur_string->params[0] = upper_32_bits(*tracer_event->out); + cur_string->params[1] = lower_32_bits(*tracer_event->out); + list_add_tail(&cur_string->list, &tracer->ready_strings_list); + return 0; +} + static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, struct tracer_event *tracer_event) { @@ -589,7 +617,7 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, if (tracer_event->string_event.tdsn == 0) { cur_string = mlx5_tracer_get_string(tracer, tracer_event); if (!cur_string) - return -1; + return mlx5_tracer_handle_raw_string(tracer, tracer_event); cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); cur_string->last_param_num = 0; @@ -602,9 +630,9 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, } else { cur_string = mlx5_tracer_message_get(tracer, tracer_event); if (!cur_string) { - pr_debug("%s Got string event for unknown string tdsm: %d\n", + pr_debug("%s Got string event for unknown string tmsn: %d\n", __func__, tracer_event->string_event.tmsn); - return -1; + return mlx5_tracer_handle_raw_string(tracer, tracer_event); } cur_string->last_param_num += 1; if (cur_string->last_param_num > TRACER_MAX_PARAMS) { @@ -756,6 +784,7 @@ static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer) if (err) mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err); + tracer->buff.consumer_index = 0; return err; } @@ -820,7 +849,6 @@ static void mlx5_fw_tracer_ownership_change(struct work_struct *work) mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner); if (tracer->owner) { tracer->owner = false; - tracer->buff.consumer_index = 0; return; } @@ -930,6 +958,14 @@ unlock: return err; } +static void mlx5_fw_tracer_update_db(struct work_struct *work) +{ + struct mlx5_fw_tracer *tracer = + container_of(work, struct mlx5_fw_tracer, update_db_work); + + mlx5_fw_tracer_reload(tracer); +} + /* Create software resources (Buffers, etc ..) */ struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) { @@ -957,6 +993,8 @@ struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change); INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db); INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces); + INIT_WORK(&tracer->update_db_work, mlx5_fw_tracer_update_db); + mutex_init(&tracer->state_lock); err = mlx5_query_mtrc_caps(tracer); @@ -1003,11 +1041,15 @@ int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) if (IS_ERR_OR_NULL(tracer)) return 0; - dev = tracer->dev; - if (!tracer->str_db.loaded) queue_work(tracer->work_queue, &tracer->read_fw_strings_work); + mutex_lock(&tracer->state_lock); + if (test_and_set_bit(MLX5_TRACER_STATE_UP, &tracer->state)) + goto unlock; + + dev = tracer->dev; + err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn); if (err) { mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err); @@ -1028,6 +1070,8 @@ int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) mlx5_core_warn(dev, "FWTracer: Failed to start tracer %d\n", err); goto err_notifier_unregister; } +unlock: + mutex_unlock(&tracer->state_lock); return 0; err_notifier_unregister: @@ -1037,6 +1081,7 @@ err_dealloc_pd: mlx5_core_dealloc_pd(dev, tracer->buff.pdn); err_cancel_work: cancel_work_sync(&tracer->read_fw_strings_work); + mutex_unlock(&tracer->state_lock); return err; } @@ -1046,17 +1091,27 @@ void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) if (IS_ERR_OR_NULL(tracer)) return; + mutex_lock(&tracer->state_lock); + if (!test_and_clear_bit(MLX5_TRACER_STATE_UP, &tracer->state)) + goto unlock; + mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n", tracer->owner); mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb); cancel_work_sync(&tracer->ownership_change_work); cancel_work_sync(&tracer->handle_traces_work); + /* It is valid to get here from update_db_work. Hence, don't wait for + * update_db_work to finished. + */ + cancel_work(&tracer->update_db_work); if (tracer->owner) mlx5_fw_tracer_ownership_release(tracer); mlx5_core_destroy_mkey(tracer->dev, tracer->buff.mkey); mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn); +unlock: + mutex_unlock(&tracer->state_lock); } /* Free software resources (Buffers, etc ..) */ @@ -1073,6 +1128,7 @@ void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer) mlx5_fw_tracer_clean_saved_traces_array(tracer); mlx5_fw_tracer_free_strings_db(tracer); mlx5_fw_tracer_destroy_log_buf(tracer); + mutex_destroy(&tracer->state_lock); destroy_workqueue(tracer->work_queue); kvfree(tracer); } @@ -1082,6 +1138,8 @@ static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer) struct mlx5_core_dev *dev; int err; + if (test_and_set_bit(MLX5_TRACER_RECREATE_DB, &tracer->state)) + return 0; cancel_work_sync(&tracer->read_fw_strings_work); mlx5_fw_tracer_clean_ready_list(tracer); mlx5_fw_tracer_clean_print_hash(tracer); @@ -1092,17 +1150,18 @@ static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer) err = mlx5_query_mtrc_caps(tracer); if (err) { mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); - return err; + goto out; } err = mlx5_fw_tracer_allocate_strings_db(tracer); if (err) { mlx5_core_warn(dev, "FWTracer: Allocate strings DB failed %d\n", err); - return err; + goto out; } mlx5_fw_tracer_init_saved_traces_array(tracer); - - return 0; +out: + clear_bit(MLX5_TRACER_RECREATE_DB, &tracer->state); + return err; } int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer) @@ -1142,6 +1201,9 @@ static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE: queue_work(tracer->work_queue, &tracer->handle_traces_work); break; + case MLX5_TRACER_SUBTYPE_STRINGS_DB_UPDATE: + queue_work(tracer->work_queue, &tracer->update_db_work); + break; default: mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n", eqe->sub_type); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h index 4762b55b0b0e..5c548bb74f07 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h @@ -63,6 +63,11 @@ struct mlx5_fw_trace_data { char msg[TRACE_STR_MSG]; }; +enum mlx5_fw_tracer_state { + MLX5_TRACER_STATE_UP = BIT(0), + MLX5_TRACER_RECREATE_DB = BIT(1), +}; + struct mlx5_fw_tracer { struct mlx5_core_dev *dev; struct mlx5_nb nb; @@ -104,6 +109,9 @@ struct mlx5_fw_tracer { struct work_struct handle_traces_work; struct hlist_head hash[MESSAGE_HASH_SIZE]; struct list_head ready_strings_list; + struct work_struct update_db_work; + struct mutex state_lock; /* Synchronize update work with reload flows */ + unsigned long state; }; struct tracer_string_format { @@ -158,6 +166,7 @@ struct tracer_event { struct tracer_string_event string_event; struct tracer_timestamp_event timestamp_event; }; + u64 *out; }; struct mlx5_ifc_tracer_event_bits { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c index b70e36025d92..9a3878f9e582 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ecpf.c @@ -95,7 +95,7 @@ void mlx5_ec_cleanup(struct mlx5_core_dev *dev) mlx5_host_pf_cleanup(dev); - err = mlx5_wait_for_pages(dev, &dev->priv.host_pf_pages); + err = mlx5_wait_for_pages(dev, &dev->priv.page_counters[MLX5_HOST_PF]); if (err) mlx5_core_warn(dev, "Timeout reclaiming external host PF pages err(%d)\n", err); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 6f8723cc6874..125c7cb7d839 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -454,6 +454,7 @@ struct mlx5e_txqsq { struct mlx5_clock *clock; struct net_device *netdev; struct mlx5_core_dev *mdev; + struct mlx5e_channel *channel; struct mlx5e_priv *priv; /* control path */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c index 57f4b1b50421..7ac1ad9c46de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port_buffer.c @@ -314,11 +314,11 @@ static int port_set_buffer(struct mlx5e_priv *priv, err = port_update_shared_buffer(priv->mdev, current_headroom_size, new_headroom_size); if (err) - return err; + goto out; err = port_update_pool_cfg(priv->mdev, port_buffer); if (err) - return err; + goto out; err = mlx5e_port_set_pbmc(mdev, in); out: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 8469e9c38670..9a1bc93b7dc6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -771,8 +771,8 @@ void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c) if (test_bit(MLX5E_PTP_STATE_RX, c->state)) { mlx5e_ptp_rx_set_fs(c->priv); mlx5e_activate_rq(&c->rq); - mlx5e_trigger_napi_sched(&c->napi); } + mlx5e_trigger_napi_sched(&c->napi); } void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index 8099a21e674c..ce85b48d327d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -438,10 +438,6 @@ static int mlx5_esw_bridge_switchdev_event(struct notifier_block *nb, switch (event) { case SWITCHDEV_FDB_ADD_TO_BRIDGE: - /* only handle the event on native eswtich of representor */ - if (!mlx5_esw_bridge_is_local(dev, rep, esw)) - break; - fdb_info = container_of(info, struct switchdev_notifier_fdb_info, info); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index b195dbbf6c90..41e356d9d785 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -81,6 +81,10 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) sq->stats->recover++; clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); mlx5e_activate_txqsq(sq); + if (sq->channel) + mlx5e_trigger_napi_icosq(sq->channel); + else + mlx5e_trigger_napi_sched(sq->cq.napi); return 0; out: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c index 78c427b38048..c095a12346de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/mirred.c @@ -232,9 +232,9 @@ parse_mirred(struct mlx5e_tc_act_parse_state *parse_state, parse_state->ifindexes[if_count] = out_dev->ifindex; parse_state->if_count++; is_uplink_rep = mlx5e_eswitch_uplink_rep(out_dev); - err = mlx5_lag_do_mirred(priv->mdev, out_dev); - if (err) - return err; + + if (mlx5_lag_mpesw_do_mirred(priv->mdev, out_dev, extack)) + return -EOPNOTSUPP; out_dev = get_fdb_out_dev(uplink_dev, out_dev); if (!out_dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c index b86ac604d0c2..2e0d88b513aa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/vlan.c @@ -44,19 +44,17 @@ parse_tc_vlan_action(struct mlx5e_priv *priv, return -EOPNOTSUPP; } + if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, vlan_idx)) { + NL_SET_ERR_MSG_MOD(extack, "firmware vlan actions is not supported"); + return -EOPNOTSUPP; + } + switch (act->id) { case FLOW_ACTION_VLAN_POP: - if (vlan_idx) { - if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, - MLX5_FS_VLAN_DEPTH)) { - NL_SET_ERR_MSG_MOD(extack, "vlan pop action is not supported"); - return -EOPNOTSUPP; - } - + if (vlan_idx) *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2; - } else { + else *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; - } break; case FLOW_ACTION_VLAN_PUSH: attr->vlan_vid[vlan_idx] = act->vlan.vid; @@ -65,25 +63,10 @@ parse_tc_vlan_action(struct mlx5e_priv *priv, if (!attr->vlan_proto[vlan_idx]) attr->vlan_proto[vlan_idx] = htons(ETH_P_8021Q); - if (vlan_idx) { - if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, - MLX5_FS_VLAN_DEPTH)) { - NL_SET_ERR_MSG_MOD(extack, - "vlan push action is not supported for vlan depth > 1"); - return -EOPNOTSUPP; - } - + if (vlan_idx) *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; - } else { - if (!mlx5_eswitch_vlan_actions_supported(priv->mdev, 1) && - (act->vlan.proto != htons(ETH_P_8021Q) || - act->vlan.prio)) { - NL_SET_ERR_MSG_MOD(extack, "vlan push action is not supported"); - return -EOPNOTSUPP; - } - + else *action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; - } break; case FLOW_ACTION_VLAN_POP_ETH: parse_state->eth_pop = true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index 7758a425bfa8..8218c892b161 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -204,13 +204,15 @@ mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_ u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; struct mlx5_core_dev *mdev = flow_meters->mdev; - void *obj; + void *obj, *param; int err; MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); - MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_create_param, param, log_obj_range, + flow_meters->log_granularity); obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 313df8232db7..193562c14c44 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -1073,12 +1073,16 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; struct flow_action_entry *meta_action; unsigned long cookie = flow->cookie; + enum ip_conntrack_info ctinfo; struct mlx5_ct_entry *entry; int err; meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule); if (!meta_action) return -EOPNOTSUPP; + ctinfo = meta_action->ct_metadata.cookie & NFCT_INFOMASK; + if (ctinfo == IP_CT_NEW) + return -EOPNOTSUPP; spin_lock_bh(&ct_priv->ht_lock); entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index e84c3400ba1d..7b0d3de0ec6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -158,6 +158,11 @@ void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->family = x->props.family; attrs->type = x->xso.type; attrs->reqid = x->props.reqid; + attrs->upspec.dport = ntohs(x->sel.dport); + attrs->upspec.dport_mask = ntohs(x->sel.dport_mask); + attrs->upspec.sport = ntohs(x->sel.sport); + attrs->upspec.sport_mask = ntohs(x->sel.sport_mask); + attrs->upspec.proto = x->sel.proto; mlx5e_ipsec_init_limits(sa_entry, attrs); } @@ -221,6 +226,13 @@ static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev, NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with geniv other than seqiv"); return -EINVAL; } + + if (x->sel.proto != IPPROTO_IP && + (x->sel.proto != IPPROTO_UDP || x->xso.dir != XFRM_DEV_OFFLOAD_OUT)) { + NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than UDP, and only Tx direction"); + return -EINVAL; + } + switch (x->xso.type) { case XFRM_DEV_OFFLOAD_CRYPTO: if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_CRYPTO)) { @@ -517,6 +529,12 @@ static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x, return -EINVAL; } + if (x->selector.proto != IPPROTO_IP && + (x->selector.proto != IPPROTO_UDP || x->xdo.dir != XFRM_DEV_OFFLOAD_OUT)) { + NL_SET_ERR_MSG_MOD(extack, "Device does not support upper protocol other than UDP, and only Tx direction"); + return -EINVAL; + } + return 0; } @@ -537,6 +555,11 @@ mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, attrs->action = x->action; attrs->type = XFRM_DEV_OFFLOAD_PACKET; attrs->reqid = x->xfrm_vec[0].reqid; + attrs->upspec.dport = ntohs(sel->dport); + attrs->upspec.dport_mask = ntohs(sel->dport_mask); + attrs->upspec.sport = ntohs(sel->sport); + attrs->upspec.sport_mask = ntohs(sel->sport_mask); + attrs->upspec.proto = sel->proto; } static int mlx5e_xfrm_add_policy(struct xfrm_policy *x, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 8bed9c361075..b387adca9c20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -52,6 +52,14 @@ struct aes_gcm_keymat { u32 aes_key[256 / 32]; }; +struct upspec { + u16 dport; + u16 dport_mask; + u16 sport; + u16 sport_mask; + u8 proto; +}; + struct mlx5_accel_esp_xfrm_attrs { u32 esn; u32 spi; @@ -68,6 +76,7 @@ struct mlx5_accel_esp_xfrm_attrs { __be32 a6[4]; } daddr; + struct upspec upspec; u8 dir : 2; u8 esn_overlap : 1; u8 esn_trigger : 1; @@ -181,6 +190,7 @@ struct mlx5_accel_pol_xfrm_attrs { __be32 a6[4]; } daddr; + struct upspec upspec; u8 family; u8 action; u8 type : 2; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 9f19f4b59a70..5da6fe68eea6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -467,6 +467,27 @@ static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) misc_parameters_2.metadata_reg_c_0, reqid); } +static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) +{ + if (upspec->proto != IPPROTO_UDP) + return; + + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); + if (upspec->dport) { + MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, + upspec->dport_mask); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, upspec->dport); + } + + if (upspec->sport) { + MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, + upspec->sport_mask); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, upspec->sport); + } +} + static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, struct mlx5_flow_act *flow_act) { @@ -654,6 +675,7 @@ static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); setup_fte_no_frags(spec); + setup_fte_upper_proto_match(spec, &attrs->upspec); switch (attrs->type) { case XFRM_DEV_OFFLOAD_CRYPTO: @@ -728,6 +750,7 @@ static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); setup_fte_no_frags(spec); + setup_fte_upper_proto_match(spec, &attrs->upspec); err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT, &flow_act); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 2461462b7b99..7fb3835befbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -4,7 +4,7 @@ #include "mlx5_core.h" #include "en.h" #include "ipsec.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" enum { MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET, @@ -92,7 +92,6 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn, MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt, lower_32_bits(attrs->hard_packet_limit)); MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1); - MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1); } if (attrs->soft_packet_limit != XFRM_INF) { @@ -329,8 +328,7 @@ static void mlx5e_ipsec_handle_event(struct work_struct *_work) if (attrs->soft_packet_limit != XFRM_INF) if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) || - !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) || - !MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable)) + !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm)) xfrm_state_check_expire(sa_entry->x); unlock: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index eb5b09f81dec..cf704f106b7c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -4,16 +4,16 @@ #include <linux/debugfs.h> #include "en.h" #include "lib/mlx5.h" +#include "lib/crypto.h" #include "en_accel/ktls.h" #include "en_accel/ktls_utils.h" #include "en_accel/fs_tcp.h" -int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, - struct tls_crypto_info *crypto_info, - u32 *p_key_id) +struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, + struct tls_crypto_info *crypto_info) { + const void *key; u32 sz_bytes; - void *key; switch (crypto_info->cipher_type) { case TLS_CIPHER_AES_GCM_128: { @@ -33,17 +33,16 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, break; } default: - return -EINVAL; + return ERR_PTR(-EINVAL); } - return mlx5_create_encryption_key(mdev, key, sz_bytes, - MLX5_ACCEL_OBJ_TLS_KEY, - p_key_id); + return mlx5_crypto_dek_create(dek_pool, key, sz_bytes); } -void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id) +void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek) { - mlx5_destroy_encryption_key(mdev, key_id); + mlx5_crypto_dek_destroy(dek_pool, dek); } static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, @@ -189,6 +188,7 @@ static void mlx5e_tls_debugfs_init(struct mlx5e_tls *tls, int mlx5e_ktls_init(struct mlx5e_priv *priv) { + struct mlx5_crypto_dek_pool *dek_pool; struct mlx5e_tls *tls; if (!mlx5e_is_ktls_device(priv->mdev)) @@ -197,9 +197,15 @@ int mlx5e_ktls_init(struct mlx5e_priv *priv) tls = kzalloc(sizeof(*tls), GFP_KERNEL); if (!tls) return -ENOMEM; + tls->mdev = priv->mdev; + dek_pool = mlx5_crypto_dek_pool_create(priv->mdev, MLX5_ACCEL_OBJ_TLS_KEY); + if (IS_ERR(dek_pool)) { + kfree(tls); + return PTR_ERR(dek_pool); + } + tls->dek_pool = dek_pool; priv->tls = tls; - priv->tls->mdev = priv->mdev; mlx5e_tls_debugfs_init(tls, priv->dfs_root); @@ -216,6 +222,7 @@ void mlx5e_ktls_cleanup(struct mlx5e_priv *priv) debugfs_remove_recursive(tls->debugfs.dfs); tls->debugfs.dfs = NULL; + mlx5_crypto_dek_pool_destroy(tls->dek_pool); kfree(priv->tls); priv->tls = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index fccf995ee16d..f11075e67658 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -10,10 +10,12 @@ #include "en.h" #ifdef CONFIG_MLX5_EN_TLS -int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, - struct tls_crypto_info *crypto_info, - u32 *p_key_id); -void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id); +#include "lib/crypto.h" + +struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, + struct tls_crypto_info *crypto_info); +void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek); static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev) { @@ -83,6 +85,7 @@ struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; struct workqueue_struct *rx_wq; struct mlx5e_tls_tx_pool *tx_pool; + struct mlx5_crypto_dek_pool *dek_pool; struct mlx5e_tls_debugfs debugfs; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 3e54834747ce..4be770443b0c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -50,7 +50,7 @@ struct mlx5e_ktls_offload_context_rx { struct mlx5e_tls_sw_stats *sw_stats; struct completion add_ctx; struct mlx5e_tir tir; - u32 key_id; + struct mlx5_crypto_dek *dek; u32 rxq; DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); @@ -148,7 +148,8 @@ post_static_params(struct mlx5e_icosq *sq, wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, mlx5e_tir_get_tirn(&priv_rx->tir), - priv_rx->key_id, priv_rx->resync.seq, false, + mlx5_crypto_dek_get_id(priv_rx->dek), + priv_rx->resync.seq, false, TLS_OFFLOAD_CTX_DIR_RX); wi = (struct mlx5e_icosq_wqe_info) { .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, @@ -610,20 +611,22 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, struct mlx5e_ktls_offload_context_rx *priv_rx; struct mlx5e_ktls_rx_resync_ctx *resync; struct tls_context *tls_ctx; - struct mlx5_core_dev *mdev; + struct mlx5_crypto_dek *dek; struct mlx5e_priv *priv; int rxq, err; tls_ctx = tls_get_ctx(sk); priv = netdev_priv(netdev); - mdev = priv->mdev; priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL); if (unlikely(!priv_rx)) return -ENOMEM; - err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id); - if (err) + dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info); + if (IS_ERR(dek)) { + err = PTR_ERR(dek); goto err_create_key; + } + priv_rx->dek = dek; INIT_LIST_HEAD(&priv_rx->list); spin_lock_init(&priv_rx->lock); @@ -673,7 +676,7 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, err_post_wqes: mlx5e_tir_destroy(&priv_rx->tir); err_create_tir: - mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); err_create_key: kfree(priv_rx); return err; @@ -683,11 +686,9 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) { struct mlx5e_ktls_offload_context_rx *priv_rx; struct mlx5e_ktls_rx_resync_ctx *resync; - struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; priv = netdev_priv(netdev); - mdev = priv->mdev; priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); @@ -707,7 +708,7 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) mlx5e_accel_fs_del_sk(priv_rx->rule.rule); mlx5e_tir_destroy(&priv_rx->tir); - mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); /* priv_rx should normally be freed here, but if there is an outstanding * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is * processed. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 6db27062b765..60b3e08a1028 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -98,7 +98,7 @@ struct mlx5e_ktls_offload_context_tx { struct tls_offload_context_tx *tx_ctx; struct mlx5_core_dev *mdev; struct mlx5e_tls_sw_stats *sw_stats; - u32 key_id; + struct mlx5_crypto_dek *dek; u8 create_err : 1; }; @@ -457,6 +457,7 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, struct mlx5e_ktls_offload_context_tx *priv_tx; struct mlx5e_tls_tx_pool *pool; struct tls_context *tls_ctx; + struct mlx5_crypto_dek *dek; struct mlx5e_priv *priv; int err; @@ -468,9 +469,12 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, if (IS_ERR(priv_tx)) return PTR_ERR(priv_tx); - err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id); - if (err) + dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info); + if (IS_ERR(dek)) { + err = PTR_ERR(dek); goto err_create_key; + } + priv_tx->dek = dek; priv_tx->expected_seq = start_offload_tcp_sn; switch (crypto_info->cipher_type) { @@ -512,7 +516,7 @@ void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) pool = priv->tls->tx_pool; atomic64_inc(&priv_tx->sw_stats->tx_tls_del); - mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_tx->dek); pool_push(pool, priv_tx); } @@ -551,8 +555,9 @@ post_static_params(struct mlx5e_txqsq *sq, pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, - priv_tx->tisn, priv_tx->key_id, 0, fence, - TLS_OFFLOAD_CTX_DIR_TX); + priv_tx->tisn, + mlx5_crypto_dek_get_id(priv_tx->dek), + 0, fence, TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; } @@ -894,8 +899,6 @@ static void mlx5e_tls_tx_debugfs_init(struct mlx5e_tls *tls, return; tls->debugfs.dfs_tx = debugfs_create_dir("tx", dfs_root); - if (!tls->debugfs.dfs_tx) - return; debugfs_create_size_t("pool_size", 0400, tls->debugfs.dfs_tx, &tls->tx_pool->size); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 7f6b940830b3..08d0929e8260 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -7,7 +7,7 @@ #include "en.h" #include "lib/aso.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" #include "en_accel/macsec.h" #include "en_accel/macsec_fs.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 68f19324db93..4c9a3210600c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -31,6 +31,7 @@ */ #include "en.h" +#include "lib/crypto.h" /* mlx5e global resources should be placed in this file. * Global resources are common to all the netdevices created on the same nic. @@ -104,6 +105,13 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); + mdev->mlx5e_res.dek_priv = mlx5_crypto_dek_init(mdev); + if (IS_ERR(mdev->mlx5e_res.dek_priv)) { + mlx5_core_err(mdev, "crypto dek init failed, %ld\n", + PTR_ERR(mdev->mlx5e_res.dek_priv)); + mdev->mlx5e_res.dek_priv = NULL; + } + return 0; err_destroy_mkey: @@ -119,6 +127,8 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; + mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); + mdev->mlx5e_res.dek_priv = NULL; mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 7298fe782e9e..05796f8b1d7c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -450,7 +450,7 @@ void mlx5e_enable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc) void mlx5e_disable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc) { - if (fs->vlan->cvlan_filter_disabled) + if (!fs->vlan || fs->vlan->cvlan_filter_disabled) return; fs->vlan->cvlan_filter_disabled = true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index e4996ef04d86..ec81d935262f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -39,6 +39,7 @@ #include <linux/if_bridge.h> #include <linux/filter.h> #include <net/page_pool.h> +#include <net/pkt_sched.h> #include <net/xdp_sock_drv.h> #include "eswitch.h" #include "en.h" @@ -180,17 +181,21 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv) static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) { struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); + struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data; int err; switch (event) { case MLX5_DRIVER_EVENT_TYPE_TRAP: - err = mlx5e_handle_trap_event(priv, data); + err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap); + if (err) { + trap_event_ctx->err = err; + return NOTIFY_BAD; + } break; default: - netdev_warn(priv->netdev, "Sync event: Unknown event %ld\n", event); - err = -EINVAL; + return NOTIFY_DONE; } - return err; + return NOTIFY_OK; } static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) @@ -592,7 +597,8 @@ static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *param rq->ix = c->ix; rq->channel = c; rq->mdev = mdev; - rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); + rq->hw_mtu = + MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en; rq->xdpsq = &c->rq_xdpsq; rq->stats = &c->priv->channel_stats[c->ix]->rq; rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); @@ -1015,35 +1021,6 @@ int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) return mlx5e_rq_to_ready(rq, curr_state); } -static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) -{ - struct mlx5_core_dev *mdev = rq->mdev; - - void *in; - void *rqc; - int inlen; - int err; - - inlen = MLX5_ST_SZ_BYTES(modify_rq_in); - in = kvzalloc(inlen, GFP_KERNEL); - if (!in) - return -ENOMEM; - - rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); - - MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); - MLX5_SET64(modify_rq_in, in, modify_bitmask, - MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS); - MLX5_SET(rqc, rqc, scatter_fcs, enable); - MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); - - err = mlx5_core_modify_rq(mdev, rq->rqn, in); - - kvfree(in); - - return err; -} - static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1470,6 +1447,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, sq->mkey_be = c->mkey_be; sq->netdev = c->netdev; sq->mdev = c->mdev; + sq->channel = c; sq->priv = c->priv; sq->ch_ix = c->ix; sq->txq_ix = txq_ix; @@ -2482,8 +2460,6 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) mlx5e_activate_xsk(c); else mlx5e_activate_rq(&c->rq); - - mlx5e_trigger_napi_icosq(c); } static void mlx5e_deactivate_channel(struct mlx5e_channel *c) @@ -2575,13 +2551,19 @@ err_free: return err; } -static void mlx5e_activate_channels(struct mlx5e_channels *chs) +static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { int i; for (i = 0; i < chs->num; i++) mlx5e_activate_channel(chs->c[i]); + if (priv->htb) + mlx5e_qos_activate_queues(priv); + + for (i = 0; i < chs->num; i++) + mlx5e_trigger_napi_icosq(chs->c[i]); + if (chs->ptp) mlx5e_ptp_activate_channel(chs->ptp); } @@ -2888,9 +2870,7 @@ out: void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) { mlx5e_build_txq_maps(priv); - mlx5e_activate_channels(&priv->channels); - if (priv->htb) - mlx5e_qos_activate_queues(priv); + mlx5e_activate_channels(priv, &priv->channels); mlx5e_xdp_tx_enable(priv); /* dev_watchdog() wants all TX queues to be started when the carrier is @@ -2998,32 +2978,37 @@ int mlx5e_safe_switch_params(struct mlx5e_priv *priv, mlx5e_fp_preactivate preactivate, void *context, bool reset) { - struct mlx5e_channels new_chs = {}; + struct mlx5e_channels *new_chs; int err; reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); if (!reset) return mlx5e_switch_priv_params(priv, params, preactivate, context); - new_chs.params = *params; + new_chs = kzalloc(sizeof(*new_chs), GFP_KERNEL); + if (!new_chs) + return -ENOMEM; + new_chs->params = *params; - mlx5e_selq_prepare_params(&priv->selq, &new_chs.params); + mlx5e_selq_prepare_params(&priv->selq, &new_chs->params); - err = mlx5e_open_channels(priv, &new_chs); + err = mlx5e_open_channels(priv, new_chs); if (err) goto err_cancel_selq; - err = mlx5e_switch_priv_channels(priv, &new_chs, preactivate, context); + err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context); if (err) goto err_close; + kfree(new_chs); return 0; err_close: - mlx5e_close_channels(&new_chs); + mlx5e_close_channels(new_chs); err_cancel_selq: mlx5e_selq_cancel(&priv->selq); + kfree(new_chs); return err; } @@ -3315,20 +3300,6 @@ static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) mlx5e_destroy_tises(priv); } -static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable) -{ - int err = 0; - int i; - - for (i = 0; i < chs->num; i++) { - err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable); - if (err) - return err; - } - - return 0; -} - static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) { int err; @@ -3904,41 +3875,27 @@ static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) return mlx5_set_ports_check(mdev, in, sizeof(in)); } +static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx) +{ + struct mlx5_core_dev *mdev = priv->mdev; + bool enable = *(bool *)ctx; + + return mlx5e_set_rx_port_ts(mdev, enable); +} + static int set_feature_rx_fcs(struct net_device *netdev, bool enable) { struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_channels *chs = &priv->channels; - struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_params new_params; int err; mutex_lock(&priv->state_lock); - if (enable) { - err = mlx5e_set_rx_port_ts(mdev, false); - if (err) - goto out; - - chs->params.scatter_fcs_en = true; - err = mlx5e_modify_channels_scatter_fcs(chs, true); - if (err) { - chs->params.scatter_fcs_en = false; - mlx5e_set_rx_port_ts(mdev, true); - } - } else { - chs->params.scatter_fcs_en = false; - err = mlx5e_modify_channels_scatter_fcs(chs, false); - if (err) { - chs->params.scatter_fcs_en = true; - goto out; - } - err = mlx5e_set_rx_port_ts(mdev, true); - if (err) { - mlx5_core_warn(mdev, "Failed to set RX port timestamp %d\n", err); - err = 0; - } - } - -out: + new_params = chs->params; + new_params.scatter_fcs_en = enable; + err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap, + &new_params.scatter_fcs_en, true); mutex_unlock(&priv->state_lock); return err; } @@ -4075,6 +4032,10 @@ static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev if (netdev->features & NETIF_F_GRO_HW) netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); + features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; + if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) + netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); + return features; } @@ -5986,7 +5947,7 @@ static int mlx5e_probe(struct auxiliary_device *adev, } mlx5e_dcbnl_init_app(priv); - mlx5_uplink_netdev_set(mdev, netdev); + mlx5_core_uplink_netdev_set(mdev, netdev); mlx5e_params_print_info(mdev, &priv->channels.params); return 0; @@ -6010,6 +5971,7 @@ static void mlx5e_remove(struct auxiliary_device *adev) struct mlx5e_priv *priv = mlx5e_dev->priv; pm_message_t state = {}; + mlx5_core_uplink_netdev_set(priv->mdev, NULL); mlx5e_dcbnl_delete_app(priv); unregister_netdev(priv->netdev); mlx5e_suspend(adev, state); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 4e6f5caf8ab6..e2ec80ebde58 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1124,8 +1124,6 @@ static void mlx5e_tc_debugfs_init(struct mlx5e_tc_table *tc, return; tc->dfs_root = debugfs_create_dir("tc", dfs_root); - if (!tc->dfs_root) - return; debugfs_create_file("hairpin_num_queues", 0644, tc->dfs_root, &tc->hairpin_params, &fops_hairpin_queues); @@ -1884,7 +1882,6 @@ post_process_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr, struct netlink_ext_ack *extack) { - struct mlx5_eswitch *esw = flow->priv->mdev->priv.eswitch; bool vf_tun; int err = 0; @@ -1896,12 +1893,6 @@ post_process_attr(struct mlx5e_tc_flow *flow, if (err) goto err_out; - if (mlx5e_is_eswitch_flow(flow)) { - err = mlx5_eswitch_add_vlan_action(esw, attr); - if (err) - goto err_out; - } - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { err = mlx5e_tc_attach_mod_hdr(flow->priv, flow, attr); if (err) @@ -2105,8 +2096,6 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, if (mlx5_flow_has_geneve_opt(flow)) mlx5_geneve_tlv_option_del(priv->mdev->geneve); - mlx5_eswitch_del_vlan_action(esw, attr); - if (flow->decap_route) mlx5e_detach_decap_route(priv, flow); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 9b44557e7271..66ec7932f008 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -817,9 +817,12 @@ static void comp_irqs_release(struct mlx5_core_dev *dev) static int comp_irqs_request(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = dev->priv.eq_table; + const struct cpumask *prev = cpu_none_mask; + const struct cpumask *mask; int ncomp_eqs = table->num_comp_eqs; u16 *cpus; int ret; + int cpu; int i; ncomp_eqs = table->num_comp_eqs; @@ -838,8 +841,19 @@ static int comp_irqs_request(struct mlx5_core_dev *dev) ret = -ENOMEM; goto free_irqs; } - for (i = 0; i < ncomp_eqs; i++) - cpus[i] = cpumask_local_spread(i, dev->priv.numa_node); + + i = 0; + rcu_read_lock(); + for_each_numa_hop_mask(mask, dev->priv.numa_node) { + for_each_cpu_andnot(cpu, mask, prev) { + cpus[i] = cpu; + if (++i == ncomp_eqs) + goto spread_done; + } + prev = mask; + } +spread_done: + rcu_read_unlock(); ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs); kfree(cpus); if (ret < 0) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index b176648d1343..3cdcb0e0b20f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1715,7 +1715,7 @@ void mlx5_esw_bridge_fdb_update_used(struct net_device *dev, u16 vport_num, u16 struct mlx5_esw_bridge *bridge; port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads); - if (!port || port->flags & MLX5_ESW_BRIDGE_PORT_FLAG_PEER) + if (!port) return; bridge = port->bridge; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 5b5a215a7dc5..fd03f076551b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -222,7 +222,6 @@ struct mlx5_eswitch_fdb { struct mlx5_flow_handle **send_to_vport_meta_rules; struct mlx5_flow_handle *miss_rule_uni; struct mlx5_flow_handle *miss_rule_multi; - int vlan_push_pop_refcount; struct mlx5_fs_chains *esw_chains_priv; struct { @@ -520,10 +519,6 @@ int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, struct netlink_ext_ack *extack); void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type); -int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, - struct mlx5_flow_attr *attr); -int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, - struct mlx5_flow_attr *attr); int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, u16 vport, u16 vlan, u8 qos, u8 set_flags); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 3a82e385544d..8fb09143e9e8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -579,16 +579,16 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, if (esw->mode != MLX5_ESWITCH_OFFLOADS) return ERR_PTR(-EOPNOTSUPP); + if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) + return ERR_PTR(-EOPNOTSUPP); + dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); if (!dest) return ERR_PTR(-ENOMEM); flow_act.action = attr->action; - /* if per flow vlan pop/push is emulated, don't set that into the firmware */ - if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) - flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | - MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); - else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { + + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; @@ -829,204 +829,6 @@ mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, __mlx5_eswitch_del_rule(esw, rule, attr, true); } -static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) -{ - struct mlx5_eswitch_rep *rep; - unsigned long i; - int err = 0; - - esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); - mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) { - if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) - continue; - - err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); - if (err) - goto out; - } - -out: - return err; -} - -static struct mlx5_eswitch_rep * -esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) -{ - struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; - - in_rep = attr->in_rep; - out_rep = attr->dests[0].rep; - - if (push) - vport = in_rep; - else if (pop) - vport = out_rep; - else - vport = in_rep; - - return vport; -} - -static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, - bool push, bool pop, bool fwd) -{ - struct mlx5_eswitch_rep *in_rep, *out_rep; - - if ((push || pop) && !fwd) - goto out_notsupp; - - in_rep = attr->in_rep; - out_rep = attr->dests[0].rep; - - if (push && in_rep->vport == MLX5_VPORT_UPLINK) - goto out_notsupp; - - if (pop && out_rep->vport == MLX5_VPORT_UPLINK) - goto out_notsupp; - - /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ - if (!push && !pop && fwd) - if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) - goto out_notsupp; - - /* protects against (1) setting rules with different vlans to push and - * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) - */ - if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) - goto out_notsupp; - - return 0; - -out_notsupp: - return -EOPNOTSUPP; -} - -int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, - struct mlx5_flow_attr *attr) -{ - struct offloads_fdb *offloads = &esw->fdb_table.offloads; - struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; - struct mlx5_eswitch_rep *vport = NULL; - bool push, pop, fwd; - int err = 0; - - /* nop if we're on the vlan push/pop non emulation mode */ - if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) - return 0; - - push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); - pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); - fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && - !attr->dest_chain); - - mutex_lock(&esw->state_lock); - - err = esw_add_vlan_action_check(esw_attr, push, pop, fwd); - if (err) - goto unlock; - - attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED; - - vport = esw_vlan_action_get_vport(esw_attr, push, pop); - - if (!push && !pop && fwd) { - /* tracks VF --> wire rules without vlan push action */ - if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { - vport->vlan_refcount++; - attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; - } - - goto unlock; - } - - if (!push && !pop) - goto unlock; - - if (!(offloads->vlan_push_pop_refcount)) { - /* it's the 1st vlan rule, apply global vlan pop policy */ - err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); - if (err) - goto out; - } - offloads->vlan_push_pop_refcount++; - - if (push) { - if (vport->vlan_refcount) - goto skip_set_push; - - err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0], - 0, SET_VLAN_INSERT | SET_VLAN_STRIP); - if (err) - goto out; - vport->vlan = esw_attr->vlan_vid[0]; -skip_set_push: - vport->vlan_refcount++; - } -out: - if (!err) - attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; -unlock: - mutex_unlock(&esw->state_lock); - return err; -} - -int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, - struct mlx5_flow_attr *attr) -{ - struct offloads_fdb *offloads = &esw->fdb_table.offloads; - struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; - struct mlx5_eswitch_rep *vport = NULL; - bool push, pop, fwd; - int err = 0; - - /* nop if we're on the vlan push/pop non emulation mode */ - if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) - return 0; - - if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED)) - return 0; - - push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); - pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); - fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); - - mutex_lock(&esw->state_lock); - - vport = esw_vlan_action_get_vport(esw_attr, push, pop); - - if (!push && !pop && fwd) { - /* tracks VF --> wire rules without vlan push action */ - if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) - vport->vlan_refcount--; - - goto out; - } - - if (push) { - vport->vlan_refcount--; - if (vport->vlan_refcount) - goto skip_unset_push; - - vport->vlan = 0; - err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, - 0, 0, SET_VLAN_STRIP); - if (err) - goto out; - } - -skip_unset_push: - offloads->vlan_push_pop_refcount--; - if (offloads->vlan_push_pop_refcount) - goto out; - - /* no more vlan rules, stop global vlan pop policy */ - err = esw_set_global_vlan_pop(esw, 0); - -out: - mutex_unlock(&esw->state_lock); - return err; -} - struct mlx5_flow_handle * mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, struct mlx5_eswitch *from_esw, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 9459e56ee90a..718cf09c28ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -424,6 +424,7 @@ int mlx5_blocking_notifier_register(struct mlx5_core_dev *dev, struct notifier_b return blocking_notifier_chain_register(&events->sw_nh, nb); } +EXPORT_SYMBOL(mlx5_blocking_notifier_register); int mlx5_blocking_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb) { @@ -431,6 +432,7 @@ int mlx5_blocking_notifier_unregister(struct mlx5_core_dev *dev, struct notifier return blocking_notifier_chain_unregister(&events->sw_nh, nb); } +EXPORT_SYMBOL(mlx5_blocking_notifier_unregister); int mlx5_blocking_notifier_call_chain(struct mlx5_core_dev *dev, unsigned int event, void *data) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 32d4c967469c..374c17445e54 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -272,8 +272,6 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, unsigned int size; int err; - if (ft_attr->max_fte != POOL_NEXT_SIZE) - size = roundup_pow_of_two(ft_attr->max_fte); size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte); if (!size) return -ENOSPC; @@ -412,11 +410,6 @@ static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns, MLX5_CMD_OP_CREATE_FLOW_GROUP); MLX5_SET(create_flow_group_in, in, table_type, ft->type); MLX5_SET(create_flow_group_in, in, table_id, ft->id); - if (ft->vport) { - MLX5_SET(create_flow_group_in, in, vport_number, ft->vport); - MLX5_SET(create_flow_group_in, in, other_vport, 1); - } - MLX5_SET(create_flow_group_in, in, vport_number, ft->vport); MLX5_SET(create_flow_group_in, in, other_vport, !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index dd43a940499b..fad479df12e4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -1776,7 +1776,6 @@ static int build_match_list(struct match_list *match_head, { struct rhlist_head *tmp, *list; struct mlx5_flow_group *g; - int err = 0; rcu_read_lock(); INIT_LIST_HEAD(&match_head->list); @@ -1802,7 +1801,7 @@ static int build_match_list(struct match_list *match_head, list_add_tail(&curr_match->list, &match_head->list); } rcu_read_unlock(); - return err; + return 0; } static u64 matched_fgs_get_version(struct list_head *match_head) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index f34e758a2f1f..7bb7be01225a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -267,6 +267,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, crypto)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_CRYPTO); + if (err) + return err; + } + if (MLX5_CAP_GEN(dev, shampo)) { err = mlx5_core_get_caps(dev, MLX5_CAP_DEV_SHAMPO); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 1da4da564e6d..63290da84010 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -371,6 +371,7 @@ static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n", reg16, mlx5_tout_ms(dev, PCI_TOGGLE)); err = -ETIMEDOUT; + goto restore; } do { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 879555ba847d..1e8bee906c31 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -62,7 +62,7 @@ enum { }; enum { - MLX5_DROP_NEW_HEALTH_WORK, + MLX5_DROP_HEALTH_WORK, }; enum { @@ -675,7 +675,7 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) devlink = priv_to_devlink(dev); mutex_lock(&dev->intf_state_mutex); - if (test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) { + if (test_bit(MLX5_DROP_HEALTH_WORK, &health->flags)) { mlx5_core_err(dev, "health works are not permitted at this stage\n"); mutex_unlock(&dev->intf_state_mutex); return; @@ -771,14 +771,8 @@ static unsigned long get_next_poll_jiffies(struct mlx5_core_dev *dev) void mlx5_trigger_health_work(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; - unsigned long flags; - spin_lock_irqsave(&health->wq_lock, flags); - if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) - queue_work(health->wq, &health->fatal_report_work); - else - mlx5_core_err(dev, "new health works are not permitted at this stage\n"); - spin_unlock_irqrestore(&health->wq_lock, flags); + queue_work(health->wq, &health->fatal_report_work); } #define MLX5_MSEC_PER_HOUR (MSEC_PER_SEC * 60 * 60) @@ -858,7 +852,7 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) timer_setup(&health->timer, poll_health, 0); health->fatal_error = MLX5_SENSOR_NO_ERR; - clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); + clear_bit(MLX5_DROP_HEALTH_WORK, &health->flags); health->health = &dev->iseg->health; health->health_counter = &dev->iseg->health_counter; @@ -869,13 +863,9 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) { struct mlx5_core_health *health = &dev->priv.health; - unsigned long flags; - if (disable_health) { - spin_lock_irqsave(&health->wq_lock, flags); - set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); - spin_unlock_irqrestore(&health->wq_lock, flags); - } + if (disable_health) + set_bit(MLX5_DROP_HEALTH_WORK, &health->flags); del_timer_sync(&health->timer); } @@ -891,11 +881,8 @@ void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev) void mlx5_drain_health_wq(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; - unsigned long flags; - spin_lock_irqsave(&health->wq_lock, flags); - set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); - spin_unlock_irqrestore(&health->wq_lock, flags); + set_bit(MLX5_DROP_HEALTH_WORK, &health->flags); cancel_delayed_work_sync(&health->update_fw_log_ts_work); cancel_work_sync(&health->report_work); cancel_work_sync(&health->fatal_report_work); @@ -928,7 +915,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev) kfree(name); if (!health->wq) goto out_err; - spin_lock_init(&health->wq_lock); INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work); INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work); INIT_DELAYED_WORK(&health->update_fw_log_ts_work, mlx5_health_log_ts_update); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index eff92dc0927c..779d92b762d3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -172,6 +172,7 @@ enum mlx5_ptys_rate { MLX5_PTYS_RATE_EDR = 1 << 5, MLX5_PTYS_RATE_HDR = 1 << 6, MLX5_PTYS_RATE_NDR = 1 << 7, + MLX5_PTYS_RATE_XDR = 1 << 8, }; static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate) @@ -185,20 +186,21 @@ static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate) case MLX5_PTYS_RATE_EDR: return 25000; case MLX5_PTYS_RATE_HDR: return 50000; case MLX5_PTYS_RATE_NDR: return 100000; + case MLX5_PTYS_RATE_XDR: return 200000; default: return -1; } } -static int mlx5i_get_speed_settings(u16 ib_link_width_oper, u16 ib_proto_oper) +static u32 mlx5i_get_speed_settings(u16 ib_link_width_oper, u16 ib_proto_oper) { int rate, width; rate = mlx5_ptys_rate_enum_to_int(ib_proto_oper); if (rate < 0) - return -EINVAL; + return SPEED_UNKNOWN; width = mlx5_ptys_width_enum_to_int(ib_link_width_oper); if (width < 0) - return -EINVAL; + return SPEED_UNKNOWN; return rate * width; } @@ -221,16 +223,13 @@ static int mlx5i_get_link_ksettings(struct net_device *netdev, ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising); speed = mlx5i_get_speed_settings(ib_link_width_oper, ib_proto_oper); - if (speed < 0) - return -EINVAL; + link_ksettings->base.speed = speed; + link_ksettings->base.duplex = speed == SPEED_UNKNOWN ? DUPLEX_UNKNOWN : DUPLEX_FULL; - link_ksettings->base.duplex = DUPLEX_FULL; link_ksettings->base.port = PORT_OTHER; link_ksettings->base.autoneg = AUTONEG_DISABLE; - link_ksettings->base.speed = speed; - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c index b8feaf0f5c4c..f4b777d4e108 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c @@ -22,7 +22,7 @@ static int type_show(struct seq_file *file, void *priv) struct mlx5_lag *ldev; char *mode = NULL; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); if (__mlx5_lag_is_active(ldev)) mode = get_str_mode_type(ldev); @@ -41,7 +41,7 @@ static int port_sel_mode_show(struct seq_file *file, void *priv) int ret = 0; char *mode; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); if (__mlx5_lag_is_active(ldev)) mode = mlx5_get_str_port_sel_mode(ldev->mode, ldev->mode_flags); @@ -61,7 +61,7 @@ static int state_show(struct seq_file *file, void *priv) struct mlx5_lag *ldev; bool active; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); active = __mlx5_lag_is_active(ldev); mutex_unlock(&ldev->lock); @@ -77,7 +77,7 @@ static int flags_show(struct seq_file *file, void *priv) bool shared_fdb; bool lag_active; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); lag_active = __mlx5_lag_is_active(ldev); if (!lag_active) @@ -108,7 +108,7 @@ static int mapping_show(struct seq_file *file, void *priv) int num_ports; int i; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); lag_active = __mlx5_lag_is_active(ldev); if (lag_active) { @@ -142,7 +142,7 @@ static int members_show(struct seq_file *file, void *priv) struct mlx5_lag *ldev; int i; - ldev = dev->priv.lag; + ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); for (i = 0; i < ldev->ports; i++) { if (!ldev->pf[i].dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index ad32b80e8501..dbf218cac535 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1187,7 +1187,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) tmp_dev = mlx5_get_next_phys_dev_lag(dev); if (tmp_dev) - ldev = tmp_dev->priv.lag; + ldev = mlx5_lag_dev(tmp_dev); if (!ldev) { ldev = mlx5_lag_dev_alloc(dev); @@ -1386,8 +1386,7 @@ bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev) spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); - res = ldev && __mlx5_lag_is_sriov(ldev) && - test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags); + res = ldev && test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags); spin_unlock_irqrestore(&lag_lock, flags); return res; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index f30ac2de639f..66013bef9939 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -50,19 +50,6 @@ struct lag_tracker { enum netdev_lag_hash hash_type; }; -enum mpesw_op { - MLX5_MPESW_OP_ENABLE, - MLX5_MPESW_OP_DISABLE, -}; - -struct mlx5_mpesw_work_st { - struct work_struct work; - struct mlx5_lag *lag; - enum mpesw_op op; - struct completion comp; - int result; -}; - /* LAG data of a ConnectX card. * It serves both its phys functions. */ @@ -124,8 +111,6 @@ int mlx5_activate_lag(struct mlx5_lag *ldev, int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev, struct net_device *ndev); bool mlx5_shared_fdb_supported(struct mlx5_lag *ldev); -void mlx5_lag_del_mpesw_rule(struct mlx5_core_dev *dev); -int mlx5_lag_add_mpesw_rule(struct mlx5_core_dev *dev); char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags); void mlx5_infer_tx_enabled(struct lag_tracker *tracker, u8 num_ports, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c index d9fcb9ed726f..d85a8dfc153d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c @@ -28,13 +28,9 @@ static bool mlx5_lag_multipath_check_prereq(struct mlx5_lag *ldev) bool mlx5_lag_is_multipath(struct mlx5_core_dev *dev) { - struct mlx5_lag *ldev; - bool res; - - ldev = mlx5_lag_dev(dev); - res = ldev && __mlx5_lag_is_multipath(ldev); + struct mlx5_lag *ldev = mlx5_lag_dev(dev); - return res; + return ldev && __mlx5_lag_is_multipath(ldev); } /** diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index c17e8f1ec914..3799f89ed1a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -58,7 +58,7 @@ static void mlx5_mpesw_work(struct work_struct *work) static int mlx5_lag_mpesw_queue_work(struct mlx5_core_dev *dev, enum mpesw_op op) { - struct mlx5_lag *ldev = dev->priv.lag; + struct mlx5_lag *ldev = mlx5_lag_dev(dev); struct mlx5_mpesw_work_st *work; int err = 0; @@ -96,25 +96,27 @@ int mlx5_lag_add_mpesw_rule(struct mlx5_core_dev *dev) return mlx5_lag_mpesw_queue_work(dev, MLX5_MPESW_OP_ENABLE); } -int mlx5_lag_do_mirred(struct mlx5_core_dev *mdev, struct net_device *out_dev) +int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev, + struct net_device *out_dev, + struct netlink_ext_ack *extack) { - struct mlx5_lag *ldev = mdev->priv.lag; + struct mlx5_lag *ldev = mlx5_lag_dev(mdev); if (!netif_is_bond_master(out_dev) || !ldev) return 0; - if (ldev->mode == MLX5_LAG_MODE_MPESW) - return -EOPNOTSUPP; + if (ldev->mode != MLX5_LAG_MODE_MPESW) + return 0; - return 0; + NL_SET_ERR_MSG_MOD(extack, "can't forward to bond in mpesw mode"); + return -EOPNOTSUPP; } bool mlx5_lag_mpesw_is_activated(struct mlx5_core_dev *dev) { - bool ret; + struct mlx5_lag *ldev = mlx5_lag_dev(dev); - ret = dev->priv.lag && dev->priv.lag->mode == MLX5_LAG_MODE_MPESW; - return ret; + return ldev && ldev->mode == MLX5_LAG_MODE_MPESW; } void mlx5_lag_mpesw_init(struct mlx5_lag *ldev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h index 88e8daffcf92..818f19b5a984 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h @@ -12,8 +12,25 @@ struct lag_mpesw { atomic_t mpesw_rule_count; }; -int mlx5_lag_do_mirred(struct mlx5_core_dev *mdev, struct net_device *out_dev); +enum mpesw_op { + MLX5_MPESW_OP_ENABLE, + MLX5_MPESW_OP_DISABLE, +}; + +struct mlx5_mpesw_work_st { + struct work_struct work; + struct mlx5_lag *lag; + enum mpesw_op op; + struct completion comp; + int result; +}; + +int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev, + struct net_device *out_dev, + struct netlink_ext_ack *extack); bool mlx5_lag_mpesw_is_activated(struct mlx5_core_dev *dev); +void mlx5_lag_del_mpesw_rule(struct mlx5_core_dev *dev); +int mlx5_lag_add_mpesw_rule(struct mlx5_core_dev *dev); #if IS_ENABLED(CONFIG_MLX5_ESWITCH) void mlx5_lag_mpesw_init(struct mlx5_lag *ldev); void mlx5_lag_mpesw_cleanup(struct mlx5_lag *ldev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index 75510a12ab02..4c9a40211059 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -362,7 +362,7 @@ static int mlx5_ptp_adjphase(struct ptp_clock_info *ptp, s32 delta) return mlx5_ptp_adjtime(ptp, delta); } -static int mlx5_ptp_adjfreq_real_time(struct mlx5_core_dev *mdev, s32 freq) +static int mlx5_ptp_freq_adj_real_time(struct mlx5_core_dev *mdev, long scaled_ppm) { u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {}; @@ -370,7 +370,15 @@ static int mlx5_ptp_adjfreq_real_time(struct mlx5_core_dev *mdev, s32 freq) return 0; MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_FREQ_UTC); - MLX5_SET(mtutc_reg, in, freq_adjustment, freq); + + if (MLX5_CAP_MCAM_FEATURE(mdev, mtutc_freq_adj_units)) { + MLX5_SET(mtutc_reg, in, freq_adj_units, + MLX5_MTUTC_FREQ_ADJ_UNITS_SCALED_PPM); + MLX5_SET(mtutc_reg, in, freq_adjustment, scaled_ppm); + } else { + MLX5_SET(mtutc_reg, in, freq_adj_units, MLX5_MTUTC_FREQ_ADJ_UNITS_PPB); + MLX5_SET(mtutc_reg, in, freq_adjustment, scaled_ppm_to_ppb(scaled_ppm)); + } return mlx5_set_mtutc(mdev, in, sizeof(in)); } @@ -385,7 +393,8 @@ static int mlx5_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) int err; mdev = container_of(clock, struct mlx5_core_dev, clock); - err = mlx5_ptp_adjfreq_real_time(mdev, scaled_ppm_to_ppb(scaled_ppm)); + + err = mlx5_ptp_freq_adj_real_time(mdev, scaled_ppm); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index e995f8378df7..3a94b8f8031e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -2,53 +2,253 @@ // Copyright (c) 2019 Mellanox Technologies. #include "mlx5_core.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" -int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, - u32 key_type, u32 *p_key_id) -{ - u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; - u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; - u32 sz_bits = sz_bytes * BITS_PER_BYTE; - u8 general_obj_key_size; - u64 general_obj_types; - void *obj, *key_p; - int err; +#define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) +#define type2idx(type) ((type) - 1) - obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); - key_p = MLX5_ADDR_OF(encryption_key_obj, obj, key); +#define MLX5_CRYPTO_DEK_POOL_SYNC_THRESH 128 - general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); - if (!(general_obj_types & - MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) - return -EINVAL; +/* calculate the num of DEKs, which are freed by any user + * (for example, TLS) after last revalidation in a pool or a bulk. + */ +#define MLX5_CRYPTO_DEK_CALC_FREED(a) \ + ({ typeof(a) _a = (a); \ + _a->num_deks - _a->avail_deks - _a->in_use_deks; }) + +#define MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) MLX5_CRYPTO_DEK_CALC_FREED(pool) +#define MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk) MLX5_CRYPTO_DEK_CALC_FREED(bulk) + +#define MLX5_CRYPTO_DEK_BULK_IDLE(bulk) \ + ({ typeof(bulk) _bulk = (bulk); \ + _bulk->avail_deks == _bulk->num_deks; }) + +enum { + MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0), +}; + +struct mlx5_crypto_dek_pool { + struct mlx5_core_dev *mdev; + u32 key_purpose; + int num_deks; /* the total number of keys in this pool */ + int avail_deks; /* the number of available keys in this pool */ + int in_use_deks; /* the number of being used keys in this pool */ + struct mutex lock; /* protect the following lists, and the bulks */ + struct list_head partial_list; /* some of keys are available */ + struct list_head full_list; /* no available keys */ + struct list_head avail_list; /* all keys are available to use */ + + /* No in-used keys, and all need to be synced. + * These bulks will be put to avail list after sync. + */ + struct list_head sync_list; + + bool syncing; + struct list_head wait_for_free; + struct work_struct sync_work; + + spinlock_t destroy_lock; /* protect destroy_list */ + struct list_head destroy_list; + struct work_struct destroy_work; +}; + +struct mlx5_crypto_dek_bulk { + struct mlx5_core_dev *mdev; + int base_obj_id; + int avail_start; /* the bit to start search */ + int num_deks; /* the total number of keys in a bulk */ + int avail_deks; /* the number of keys available, with need_sync bit 0 */ + int in_use_deks; /* the number of keys being used, with in_use bit 1 */ + struct list_head entry; + + /* 0: not being used by any user, 1: otherwise */ + unsigned long *in_use; + + /* The bits are set when they are used, and reset after crypto_sync + * is executed. So, the value 0 means the key is newly created, or not + * used after sync, and 1 means it is in use, or freed but not synced + */ + unsigned long *need_sync; +}; + +struct mlx5_crypto_dek_priv { + struct mlx5_core_dev *mdev; + int log_dek_obj_range; +}; + +struct mlx5_crypto_dek { + struct mlx5_crypto_dek_bulk *bulk; + struct list_head entry; + u32 obj_id; +}; + +u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek) +{ + return dek->obj_id; +} + +static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev, + u32 sz_bytes, u8 *key_sz_p) +{ + u32 sz_bits = sz_bytes * BITS_PER_BYTE; switch (sz_bits) { case 128: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; - key_p += sz_bytes; + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; break; case 256: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; break; default: + mlx5_core_err(mdev, "Crypto offload error, invalid key size (%u bits)\n", + sz_bits); return -EINVAL; } - memcpy(key_p, key, sz_bytes); + return 0; +} + +static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, + const void *key, u32 sz_bytes) +{ + void *dst; + u8 key_sz; + int err; + + err = mlx5_crypto_dek_get_key_sz(mdev, sz_bytes, &key_sz); + if (err) + return err; + + MLX5_SET(encryption_key_obj, key_obj, key_size, key_sz); + + if (sz_bytes == 16) + /* For key size of 128b the MSBs are reserved. */ + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key[1]); + else + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key); + + memcpy(dst, key, sz_bytes); + + return 0; +} + +static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev, + int crypto_type) +{ + u32 in[MLX5_ST_SZ_DW(sync_crypto_in)] = {}; + int err; + + mlx5_core_dbg(mdev, + "Execute SYNC_CRYPTO command with crypto_type(0x%x)\n", + crypto_type); + + MLX5_SET(sync_crypto_in, in, opcode, MLX5_CMD_OP_SYNC_CRYPTO); + MLX5_SET(sync_crypto_in, in, crypto_type, crypto_type); + + err = mlx5_cmd_exec_in(mdev, sync_crypto, in); + if (err) + mlx5_core_err(mdev, + "Failed to exec sync crypto, type=%d, err=%d\n", + crypto_type, err); + + return err; +} + +static int mlx5_crypto_create_dek_bulk(struct mlx5_core_dev *mdev, + u32 key_purpose, int log_obj_range, + u32 *obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; - MLX5_SET(encryption_key_obj, obj, key_size, general_obj_key_size); - MLX5_SET(encryption_key_obj, obj, key_type, key_type); MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_create_param, param, log_obj_range, log_obj_range); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + mlx5_core_dbg(mdev, "DEK objects created, bulk=%d, obj_id=%d\n", + 1 << log_obj_range, *obj_id); + + return 0; +} + +static int mlx5_crypto_modify_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, u32 key_purpose, + u32 obj_id, u32 obj_offset) +{ + u32 in[MLX5_ST_SZ_DW(modify_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); + + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_query_param, param, obj_offset, obj_offset); + + obj = MLX5_ADDR_OF(modify_encryption_key_in, in, encryption_key_object); + MLX5_SET64(encryption_key_obj, obj, modify_field_select, 1); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + + /* avoid leaking key on the stack */ + memzero_explicit(in, sizeof(in)); + + return err; +} + +static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_purpose, u32 *p_key_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + u64 general_obj_types; + void *obj; + int err; + + general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); + if (!(general_obj_types & + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) + return -EINVAL; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) *p_key_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); @@ -58,7 +258,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, return err; } -void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +static void mlx5_crypto_destroy_dek_key(struct mlx5_core_dev *mdev, u32 key_id) { u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; @@ -71,3 +271,504 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); } + +int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id) +{ + return mlx5_crypto_create_dek_key(mdev, key, sz_bytes, key_type, p_key_id); +} + +void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +{ + mlx5_crypto_destroy_dek_key(mdev, key_id); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_bulk_create(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_priv *dek_priv = pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = pool->mdev; + struct mlx5_crypto_dek_bulk *bulk; + int num_deks, base_obj_id; + int err; + + bulk = kzalloc(sizeof(*bulk), GFP_KERNEL); + if (!bulk) + return ERR_PTR(-ENOMEM); + + num_deks = 1 << dek_priv->log_dek_obj_range; + bulk->need_sync = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->need_sync) { + err = -ENOMEM; + goto err_out; + } + + bulk->in_use = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->in_use) { + err = -ENOMEM; + goto err_out; + } + + err = mlx5_crypto_create_dek_bulk(mdev, pool->key_purpose, + dek_priv->log_dek_obj_range, + &base_obj_id); + if (err) + goto err_out; + + bulk->base_obj_id = base_obj_id; + bulk->num_deks = num_deks; + bulk->avail_deks = num_deks; + bulk->mdev = mdev; + + return bulk; + +err_out: + bitmap_free(bulk->in_use); + bitmap_free(bulk->need_sync); + kfree(bulk); + return ERR_PTR(err); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_add_bulk(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk; + + bulk = mlx5_crypto_dek_bulk_create(pool); + if (IS_ERR(bulk)) + return bulk; + + pool->avail_deks += bulk->num_deks; + pool->num_deks += bulk->num_deks; + list_add(&bulk->entry, &pool->partial_list); + + return bulk; +} + +static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk) +{ + mlx5_crypto_destroy_dek_key(bulk->mdev, bulk->base_obj_id); + bitmap_free(bulk->need_sync); + bitmap_free(bulk->in_use); + kfree(bulk); +} + +static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk, + bool delay) +{ + pool->num_deks -= bulk->num_deks; + pool->avail_deks -= bulk->avail_deks; + pool->in_use_deks -= bulk->in_use_deks; + list_del(&bulk->entry); + if (!delay) + mlx5_crypto_dek_bulk_free(bulk); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) +{ + struct mlx5_crypto_dek_bulk *bulk; + int pos; + + mutex_lock(&pool->lock); + bulk = list_first_entry_or_null(&pool->partial_list, + struct mlx5_crypto_dek_bulk, entry); + + if (bulk) { + pos = find_next_zero_bit(bulk->need_sync, bulk->num_deks, + bulk->avail_start); + if (pos == bulk->num_deks) { + mlx5_core_err(pool->mdev, "Wrong DEK bulk avail_start.\n"); + pos = find_first_zero_bit(bulk->need_sync, bulk->num_deks); + } + WARN_ON(pos == bulk->num_deks); + } else { + bulk = list_first_entry_or_null(&pool->avail_list, + struct mlx5_crypto_dek_bulk, + entry); + if (bulk) { + list_move(&bulk->entry, &pool->partial_list); + } else { + bulk = mlx5_crypto_dek_pool_add_bulk(pool); + if (IS_ERR(bulk)) + goto out; + } + pos = 0; + } + + *obj_offset = pos; + bitmap_set(bulk->need_sync, pos, 1); + bitmap_set(bulk->in_use, pos, 1); + bulk->in_use_deks++; + bulk->avail_deks--; + if (!bulk->avail_deks) { + list_move(&bulk->entry, &pool->full_list); + bulk->avail_start = bulk->num_deks; + } else { + bulk->avail_start = pos + 1; + } + pool->avail_deks--; + pool->in_use_deks++; + +out: + mutex_unlock(&pool->lock); + return bulk; +} + +static bool mlx5_crypto_dek_need_sync(struct mlx5_crypto_dek_pool *pool) +{ + return !pool->syncing && + MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) > MLX5_CRYPTO_DEK_POOL_SYNC_THRESH; +} + +static int mlx5_crypto_dek_free_locked(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_crypto_dek_bulk *bulk = dek->bulk; + int obj_offset; + bool old_val; + int err = 0; + + obj_offset = dek->obj_id - bulk->base_obj_id; + old_val = test_and_clear_bit(obj_offset, bulk->in_use); + WARN_ON_ONCE(!old_val); + if (!old_val) { + err = -ENOENT; + goto out_free; + } + pool->in_use_deks--; + bulk->in_use_deks--; + if (!bulk->avail_deks && !bulk->in_use_deks) + list_move(&bulk->entry, &pool->sync_list); + + if (mlx5_crypto_dek_need_sync(pool) && schedule_work(&pool->sync_work)) + pool->syncing = true; + +out_free: + kfree(dek); + return err; +} + +static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + int err = 0; + + mutex_lock(&pool->lock); + if (pool->syncing) + list_add(&dek->entry, &pool->wait_for_free); + else + err = mlx5_crypto_dek_free_locked(pool, dek); + mutex_unlock(&pool->lock); + + return err; +} + +/* Update the bits for a bulk while sync, and avail_next for search. + * As the combinations of (need_sync, in_use) of one DEK are + * - (0,0) means the key is ready for use, + * - (1,1) means the key is currently being used by a user, + * - (1,0) means the key is freed, and waiting for being synced, + * - (0,1) is invalid state. + * the number of revalidated DEKs can be calculated by + * hweight_long(need_sync XOR in_use), and the need_sync bits can be reset + * by simply copying from in_use bits. + */ +static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk) +{ + unsigned long *need_sync = bulk->need_sync; + unsigned long *in_use = bulk->in_use; + int i, freed, reused, avail_next; + bool first = true; + + freed = MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk); + + for (i = 0; freed && i < BITS_TO_LONGS(bulk->num_deks); + i++, need_sync++, in_use++) { + reused = hweight_long((*need_sync) ^ (*in_use)); + if (!reused) + continue; + + bulk->avail_deks += reused; + pool->avail_deks += reused; + *need_sync = *in_use; + if (first) { + avail_next = i * BITS_PER_TYPE(long); + if (bulk->avail_start > avail_next) + bulk->avail_start = avail_next; + first = false; + } + + freed -= reused; + } +} + +/* Return true if the bulk is reused, false if destroyed with delay */ +static bool mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk, + struct list_head *destroy_list) +{ + if (list_empty(&pool->avail_list)) { + list_move(&bulk->entry, &pool->avail_list); + return true; + } + + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true); + list_add(&bulk->entry, destroy_list); + return false; +} + +static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool, + struct list_head *list, + struct list_head *head) +{ + spin_lock(&pool->destroy_lock); + list_splice_init(list, head); + spin_unlock(&pool->destroy_lock); +} + +static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek *dek, *next; + + list_for_each_entry_safe(dek, next, &pool->wait_for_free, entry) { + list_del(&dek->entry); + mlx5_crypto_dek_free_locked(pool, dek); + } +} + +/* For all the bulks in each list, reset the bits while sync. + * Move them to different lists according to the number of available DEKs. + * Destrory all the idle bulks, except one for quick service. + * And free DEKs in the waiting list at the end of this func. + */ +static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + LIST_HEAD(destroy_list); + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + + if (!bulk->avail_deks) + continue; + + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); + else + list_move(&bulk->entry, &pool->partial_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) { + bulk->avail_deks = bulk->num_deks; + pool->avail_deks += bulk->num_deks; + if (mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list)) { + memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); + bulk->avail_start = 0; + } + } + + mlx5_crypto_dek_pool_free_wait_keys(pool); + + if (!list_empty(&destroy_list)) { + mlx5_crypto_dek_pool_splice_destroy_list(pool, &destroy_list, + &pool->destroy_list); + schedule_work(&pool->destroy_work); + } +} + +static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, sync_work); + int err; + + err = mlx5_crypto_cmd_sync_crypto(pool->mdev, BIT(pool->key_purpose)); + mutex_lock(&pool->lock); + if (!err) + mlx5_crypto_dek_pool_reset_synced(pool); + pool->syncing = false; + mutex_unlock(&pool->lock); +} + +struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, + const void *key, u32 sz_bytes) +{ + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = dek_pool->mdev; + u32 key_purpose = dek_pool->key_purpose; + struct mlx5_crypto_dek_bulk *bulk; + struct mlx5_crypto_dek *dek; + int obj_offset; + int err; + + dek = kzalloc(sizeof(*dek), GFP_KERNEL); + if (!dek) + return ERR_PTR(-ENOMEM); + + if (!dek_priv) { + err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, + key_purpose, &dek->obj_id); + goto out; + } + + bulk = mlx5_crypto_dek_pool_pop(dek_pool, &obj_offset); + if (IS_ERR(bulk)) { + err = PTR_ERR(bulk); + goto out; + } + + dek->bulk = bulk; + dek->obj_id = bulk->base_obj_id + obj_offset; + err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose, + bulk->base_obj_id, obj_offset); + if (err) { + mlx5_crypto_dek_pool_push(dek_pool, dek); + return ERR_PTR(err); + } + +out: + if (err) { + kfree(dek); + return ERR_PTR(err); + } + + return dek; +} + +void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = dek_pool->mdev; + + if (!dek_priv) { + mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + kfree(dek); + } else { + mlx5_crypto_dek_pool_push(dek_pool, dek); + } +} + +static void mlx5_crypto_dek_free_destroy_list(struct list_head *destroy_list) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + list_for_each_entry_safe(bulk, tmp, destroy_list, entry) + mlx5_crypto_dek_bulk_free(bulk); +} + +static void mlx5_crypto_dek_destroy_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, destroy_work); + LIST_HEAD(destroy_list); + + mlx5_crypto_dek_pool_splice_destroy_list(pool, &pool->destroy_list, + &destroy_list); + mlx5_crypto_dek_free_destroy_list(&destroy_list); +} + +struct mlx5_crypto_dek_pool * +mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) +{ + struct mlx5_crypto_dek_pool *pool; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return ERR_PTR(-ENOMEM); + + pool->mdev = mdev; + pool->key_purpose = key_purpose; + + mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->avail_list); + INIT_LIST_HEAD(&pool->partial_list); + INIT_LIST_HEAD(&pool->full_list); + INIT_LIST_HEAD(&pool->sync_list); + INIT_LIST_HEAD(&pool->wait_for_free); + INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn); + spin_lock_init(&pool->destroy_lock); + INIT_LIST_HEAD(&pool->destroy_list); + INIT_WORK(&pool->destroy_work, mlx5_crypto_dek_destroy_work_fn); + + return pool; +} + +void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + cancel_work_sync(&pool->sync_work); + cancel_work_sync(&pool->destroy_work); + + mlx5_crypto_dek_pool_free_wait_keys(pool); + + list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + mlx5_crypto_dek_free_destroy_list(&pool->destroy_list); + + mutex_destroy(&pool->lock); + + kfree(pool); +} + +void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) +{ + if (!dek_priv) + return; + + kfree(dek_priv); +} + +struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_crypto_dek_priv *dek_priv; + int err; + + if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)) + return NULL; + + dek_priv = kzalloc(sizeof(*dek_priv), GFP_KERNEL); + if (!dek_priv) + return ERR_PTR(-ENOMEM); + + dek_priv->mdev = mdev; + dek_priv->log_dek_obj_range = min_t(int, 12, + MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)); + + /* sync all types of objects */ + err = mlx5_crypto_cmd_sync_crypto(mdev, MLX5_CRYPTO_DEK_ALL_TYPE); + if (err) + goto err_sync_crypto; + + mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n", + 1 << dek_priv->log_dek_obj_range, + 1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc), + 1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks)); + + return dek_priv; + +err_sync_crypto: + kfree(dek_priv); + return ERR_PTR(err); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h new file mode 100644 index 000000000000..c819c047bb9c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LIB_CRYPTO_H__ +#define __MLX5_LIB_CRYPTO_H__ + +enum { + MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS, + MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC, + MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC, + MLX5_ACCEL_OBJ_TYPE_KEY_NUM, +}; + +int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id); + +void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); + +struct mlx5_crypto_dek_pool; +struct mlx5_crypto_dek; + +struct mlx5_crypto_dek_pool *mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, + int key_purpose); +void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool); +struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, + const void *key, u32 sz_bytes); +void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek); +u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek); + +struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev); +void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv); +#endif /* __MLX5_LIB_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 032adb21ad4b..ccf12f7db6f0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -79,28 +79,11 @@ struct mlx5_pme_stats { void mlx5_get_pme_stats(struct mlx5_core_dev *dev, struct mlx5_pme_stats *stats); int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, void *data); -/* Crypto */ -enum { - MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, - MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, - MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC, -}; - -int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, - u32 key_type, u32 *p_key_id); -void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); - static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) { return devlink_net(priv_to_devlink(dev)); } -static inline void mlx5_uplink_netdev_set(struct mlx5_core_dev *mdev, struct net_device *netdev) -{ - mdev->mlx5e_res.uplink_netdev = netdev; -} - static inline struct net_device *mlx5_uplink_netdev_get(struct mlx5_core_dev *mdev) { return mdev->mlx5e_res.uplink_netdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 8823f20d2122..26e1057845fe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -336,6 +336,24 @@ static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) } } +void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev) +{ + mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); + dev->mlx5e_res.uplink_netdev = netdev; + mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, + netdev); + mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); +} + +void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev) +{ + mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); + mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV, + dev->mlx5e_res.uplink_netdev); + mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); +} +EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay); + static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, enum mlx5_cap_mode cap_mode) @@ -1555,6 +1573,7 @@ static const int types[] = { MLX5_CAP_DEV_SHAMPO, MLX5_CAP_MACSEC, MLX5_CAP_ADV_VIRTUALIZATION, + MLX5_CAP_CRYPTO, }; static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) @@ -1608,6 +1627,7 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) lockdep_register_key(&dev->lock_key); mutex_init(&dev->intf_state_mutex); lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); + mutex_init(&dev->mlx5e_res.uplink_netdev_lock); mutex_init(&priv->bfregs.reg_head.lock); mutex_init(&priv->bfregs.wc_head.lock); @@ -1696,6 +1716,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) mutex_destroy(&priv->alloc_mutex); mutex_destroy(&priv->bfregs.wc_head.lock); mutex_destroy(&priv->bfregs.reg_head.lock); + mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock); mutex_destroy(&dev->intf_state_mutex); lockdep_unregister_key(&dev->lock_key); } @@ -2110,7 +2131,7 @@ static int __init mlx5_init(void) mlx5_core_verify_params(); mlx5_register_debugfs(); - err = pci_register_driver(&mlx5_core_driver); + err = mlx5e_init(); if (err) goto err_debug; @@ -2118,16 +2139,16 @@ static int __init mlx5_init(void) if (err) goto err_sf; - err = mlx5e_init(); + err = pci_register_driver(&mlx5_core_driver); if (err) - goto err_en; + goto err_pci; return 0; -err_en: +err_pci: mlx5_sf_driver_unregister(); err_sf: - pci_unregister_driver(&mlx5_core_driver); + mlx5e_cleanup(); err_debug: mlx5_unregister_debugfs(); return err; @@ -2135,9 +2156,9 @@ err_debug: static void __exit mlx5_cleanup(void) { - mlx5e_cleanup(); - mlx5_sf_driver_unregister(); pci_unregister_driver(&mlx5_core_driver); + mlx5_sf_driver_unregister(); + mlx5e_cleanup(); mlx5_unregister_debugfs(); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 60596357bfc7..64d4e7125e9b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -74,6 +74,14 @@ static u32 get_function(u16 func_id, bool ec_function) return (u32)func_id | (ec_function << 16); } +static u16 func_id_to_type(struct mlx5_core_dev *dev, u16 func_id, bool ec_function) +{ + if (!func_id) + return mlx5_core_is_ecpf(dev) && !ec_function ? MLX5_HOST_PF : MLX5_PF; + + return func_id <= mlx5_core_max_vfs(dev) ? MLX5_VF : MLX5_SF; +} + static struct rb_root *page_root_per_function(struct mlx5_core_dev *dev, u32 function) { struct rb_root *root; @@ -211,7 +219,8 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u32 function) n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); if (n >= MLX5_NUM_4K_IN_PAGE) { - mlx5_core_warn(dev, "alloc 4k bug\n"); + mlx5_core_warn(dev, "alloc 4k bug: fw page = 0x%llx, n = %u, bitmask: %lu, max num of 4K pages: %d\n", + fp->addr, n, fp->bitmask, MLX5_NUM_4K_IN_PAGE); return -ENOENT; } clear_bit(n, &fp->bitmask); @@ -332,6 +341,7 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); int notify_fail = event; + u16 func_type; u64 addr; int err; u32 *in; @@ -383,11 +393,9 @@ retry: goto out_dropped; } + func_type = func_id_to_type(dev, func_id, ec_function); + dev->priv.page_counters[func_type] += npages; dev->priv.fw_pages += npages; - if (func_id) - dev->priv.vfs_pages += npages; - else if (mlx5_core_is_ecpf(dev) && !ec_function) - dev->priv.host_pf_pages += npages; mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n", npages, ec_function, func_id, err); @@ -414,6 +422,7 @@ static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id, struct rb_root *root; struct rb_node *p; int npages = 0; + u16 func_type; root = xa_load(&dev->priv.page_root_xa, function); if (WARN_ON_ONCE(!root)) @@ -428,11 +437,9 @@ static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id, free_fwp(dev, fwp, fwp->free_count); } + func_type = func_id_to_type(dev, func_id, ec_function); + dev->priv.page_counters[func_type] -= npages; dev->priv.fw_pages -= npages; - if (func_id) - dev->priv.vfs_pages -= npages; - else if (mlx5_core_is_ecpf(dev) && !ec_function) - dev->priv.host_pf_pages -= npages; mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n", npages, ec_function, func_id); @@ -498,6 +505,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; int num_claimed; + u16 func_type; u32 *out; int err; int i; @@ -549,11 +557,9 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, if (nclaimed) *nclaimed = num_claimed; + func_type = func_id_to_type(dev, func_id, ec_function); + dev->priv.page_counters[func_type] -= num_claimed; dev->priv.fw_pages -= num_claimed; - if (func_id) - dev->priv.vfs_pages -= num_claimed; - else if (mlx5_core_is_ecpf(dev) && !ec_function) - dev->priv.host_pf_pages -= num_claimed; out_free: kvfree(out); @@ -706,12 +712,12 @@ int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) WARN(dev->priv.fw_pages, "FW pages counter is %d after reclaiming all pages\n", dev->priv.fw_pages); - WARN(dev->priv.vfs_pages, + WARN(dev->priv.page_counters[MLX5_VF], "VFs FW pages counter is %d after reclaiming all pages\n", - dev->priv.vfs_pages); - WARN(dev->priv.host_pf_pages, + dev->priv.page_counters[MLX5_VF]); + WARN(dev->priv.page_counters[MLX5_HOST_PF], "External host PF FW pages counter is %d after reclaiming all pages\n", - dev->priv.host_pf_pages); + dev->priv.page_counters[MLX5_HOST_PF]); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index c0e6c487c63c..3008e9ce2bbf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -147,7 +147,7 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) mlx5_eswitch_disable_sriov(dev->priv.eswitch, clear_vf); - if (mlx5_wait_for_pages(dev, &dev->priv.vfs_pages)) + if (mlx5_wait_for_pages(dev, &dev->priv.page_counters[MLX5_VF])) mlx5_core_warn(dev, "timeout reclaiming VFs pages\n"); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c index b851141e03de..042ca0349124 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c @@ -1138,12 +1138,14 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, rule->flow_source)) return 0; + mlx5dr_domain_nic_lock(nic_dmn); + ret = mlx5dr_matcher_select_builders(matcher, nic_matcher, dr_rule_get_ipv(¶m->outer), dr_rule_get_ipv(¶m->inner)); if (ret) - return ret; + goto err_unlock; hw_ste_arr_is_opt = nic_matcher->num_of_builders <= DR_RULE_MAX_STES_OPTIMIZED; if (likely(hw_ste_arr_is_opt)) { @@ -1152,12 +1154,12 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, hw_ste_arr = kzalloc((nic_matcher->num_of_builders + DR_ACTION_MAX_STES) * DR_STE_SIZE, GFP_KERNEL); - if (!hw_ste_arr) - return -ENOMEM; + if (!hw_ste_arr) { + ret = -ENOMEM; + goto err_unlock; + } } - mlx5dr_domain_nic_lock(nic_dmn); - ret = mlx5dr_matcher_add_to_tbl_nic(dmn, nic_matcher); if (ret) goto free_hw_ste; @@ -1223,7 +1225,10 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, mlx5dr_domain_nic_unlock(nic_dmn); - goto out; + if (unlikely(!hw_ste_arr_is_opt)) + kfree(hw_ste_arr); + + return 0; free_rule: dr_rule_clean_rule_members(rule, nic_rule); @@ -1238,12 +1243,12 @@ remove_from_nic_tbl: mlx5dr_matcher_remove_from_tbl_nic(dmn, nic_matcher); free_hw_ste: - mlx5dr_domain_nic_unlock(nic_dmn); - -out: - if (unlikely(!hw_ste_arr_is_opt)) + if (!hw_ste_arr_is_opt) kfree(hw_ste_arr); +err_unlock: + mlx5dr_domain_nic_unlock(nic_dmn); + return ret; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index f8623e8388c8..22db0bb15c45 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1750,29 +1750,12 @@ static const struct devlink_ops mlxsw_devlink_ops = { static int mlxsw_core_params_register(struct mlxsw_core *mlxsw_core) { - int err; - - err = mlxsw_core_fw_params_register(mlxsw_core); - if (err) - return err; - - if (mlxsw_core->driver->params_register) { - err = mlxsw_core->driver->params_register(mlxsw_core); - if (err) - goto err_params_register; - } - return 0; - -err_params_register: - mlxsw_core_fw_params_unregister(mlxsw_core); - return err; + return mlxsw_core_fw_params_register(mlxsw_core); } static void mlxsw_core_params_unregister(struct mlxsw_core *mlxsw_core) { mlxsw_core_fw_params_unregister(mlxsw_core); - if (mlxsw_core->driver->params_register) - mlxsw_core->driver->params_unregister(mlxsw_core); } struct mlxsw_core_health_event { @@ -2201,6 +2184,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, goto err_devlink_alloc; } devl_lock(devlink); + devl_register(devlink); } mlxsw_core = devlink_priv(devlink); @@ -2284,11 +2268,8 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, goto err_driver_init; } - if (!reload) { - devlink_set_features(devlink, DEVLINK_F_RELOAD); + if (!reload) devl_unlock(devlink); - devlink_register(devlink); - } return 0; err_driver_init: @@ -2320,6 +2301,7 @@ err_register_resources: err_bus_init: mlxsw_core_irq_event_handler_fini(mlxsw_core); if (!reload) { + devl_unregister(devlink); devl_unlock(devlink); devlink_free(devlink); } @@ -2358,10 +2340,8 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, { struct devlink *devlink = priv_to_devlink(mlxsw_core); - if (!reload) { - devlink_unregister(devlink); + if (!reload) devl_lock(devlink); - } if (devlink_is_reload_failed(devlink)) { if (!reload) @@ -2390,6 +2370,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, mlxsw_core->bus->fini(mlxsw_core->bus_priv); mlxsw_core_irq_event_handler_fini(mlxsw_core); if (!reload) { + devl_unregister(devlink); devl_unlock(devlink); devlink_free(devlink); } @@ -2399,6 +2380,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, reload_fail_deinit: mlxsw_core_params_unregister(mlxsw_core); devl_resources_unregister(devlink); + devl_unregister(devlink); devl_unlock(devlink); devlink_free(devlink); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index a77cb0be7108..e5474d3e34db 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -421,8 +421,6 @@ struct mlxsw_driver { const struct mlxsw_config_profile *profile, u64 *p_single_size, u64 *p_double_size, u64 *p_linear_size); - int (*params_register)(struct mlxsw_core *mlxsw_core); - void (*params_unregister)(struct mlxsw_core *mlxsw_core); /* Notify a driver that a timestamped packet was transmitted. Driver * is responsible for freeing the passed-in SKB. diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index b0bdb9640ebf..a8f94b7544ee 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -3861,62 +3861,6 @@ static int mlxsw_sp_kvd_sizes_get(struct mlxsw_core *mlxsw_core, return 0; } -static int -mlxsw_sp_params_acl_region_rehash_intrvl_get(struct devlink *devlink, u32 id, - struct devlink_param_gset_ctx *ctx) -{ - struct mlxsw_core *mlxsw_core = devlink_priv(devlink); - struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); - - ctx->val.vu32 = mlxsw_sp_acl_region_rehash_intrvl_get(mlxsw_sp); - return 0; -} - -static int -mlxsw_sp_params_acl_region_rehash_intrvl_set(struct devlink *devlink, u32 id, - struct devlink_param_gset_ctx *ctx) -{ - struct mlxsw_core *mlxsw_core = devlink_priv(devlink); - struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); - - return mlxsw_sp_acl_region_rehash_intrvl_set(mlxsw_sp, ctx->val.vu32); -} - -static const struct devlink_param mlxsw_sp2_devlink_params[] = { - DEVLINK_PARAM_DRIVER(MLXSW_DEVLINK_PARAM_ID_ACL_REGION_REHASH_INTERVAL, - "acl_region_rehash_interval", - DEVLINK_PARAM_TYPE_U32, - BIT(DEVLINK_PARAM_CMODE_RUNTIME), - mlxsw_sp_params_acl_region_rehash_intrvl_get, - mlxsw_sp_params_acl_region_rehash_intrvl_set, - NULL), -}; - -static int mlxsw_sp2_params_register(struct mlxsw_core *mlxsw_core) -{ - struct devlink *devlink = priv_to_devlink(mlxsw_core); - union devlink_param_value value; - int err; - - err = devl_params_register(devlink, mlxsw_sp2_devlink_params, - ARRAY_SIZE(mlxsw_sp2_devlink_params)); - if (err) - return err; - - value.vu32 = 0; - devl_param_driverinit_value_set(devlink, - MLXSW_DEVLINK_PARAM_ID_ACL_REGION_REHASH_INTERVAL, - value); - return 0; -} - -static void mlxsw_sp2_params_unregister(struct mlxsw_core *mlxsw_core) -{ - devl_params_unregister(priv_to_devlink(mlxsw_core), - mlxsw_sp2_devlink_params, - ARRAY_SIZE(mlxsw_sp2_devlink_params)); -} - static void mlxsw_sp_ptp_transmitted(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, u16 local_port) { @@ -3994,8 +3938,6 @@ static struct mlxsw_driver mlxsw_sp2_driver = { .trap_policer_counter_get = mlxsw_sp_trap_policer_counter_get, .txhdr_construct = mlxsw_sp_txhdr_construct, .resources_register = mlxsw_sp2_resources_register, - .params_register = mlxsw_sp2_params_register, - .params_unregister = mlxsw_sp2_params_unregister, .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp2_config_profile, @@ -4033,8 +3975,6 @@ static struct mlxsw_driver mlxsw_sp3_driver = { .trap_policer_counter_get = mlxsw_sp_trap_policer_counter_get, .txhdr_construct = mlxsw_sp_txhdr_construct, .resources_register = mlxsw_sp2_resources_register, - .params_register = mlxsw_sp2_params_register, - .params_unregister = mlxsw_sp2_params_unregister, .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp2_config_profile, @@ -4070,8 +4010,6 @@ static struct mlxsw_driver mlxsw_sp4_driver = { .trap_policer_counter_get = mlxsw_sp_trap_policer_counter_get, .txhdr_construct = mlxsw_sp_txhdr_construct, .resources_register = mlxsw_sp2_resources_register, - .params_register = mlxsw_sp2_params_register, - .params_unregister = mlxsw_sp2_params_unregister, .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, .profile = &mlxsw_sp4_config_profile, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index bbc73324451d..4c22f8004514 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -973,6 +973,7 @@ enum mlxsw_sp_acl_profile { }; struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl); +struct mlxsw_sp_acl_tcam *mlxsw_sp_acl_to_tcam(struct mlxsw_sp_acl *acl); int mlxsw_sp_acl_ruleset_bind(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_flow_block *block, @@ -1096,8 +1097,6 @@ mlxsw_sp_acl_act_cookie_lookup(struct mlxsw_sp *mlxsw_sp, u32 cookie_index) int mlxsw_sp_acl_init(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp_acl_fini(struct mlxsw_sp *mlxsw_sp); -u32 mlxsw_sp_acl_region_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp); -int mlxsw_sp_acl_region_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, u32 val); struct mlxsw_sp_acl_mangle_action; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index 6c5af018546f..0423ac262d89 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -40,6 +40,11 @@ struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl) return acl->afk; } +struct mlxsw_sp_acl_tcam *mlxsw_sp_acl_to_tcam(struct mlxsw_sp_acl *acl) +{ + return &acl->tcam; +} + struct mlxsw_sp_acl_ruleset_ht_key { struct mlxsw_sp_flow_block *block; u32 chain_index; @@ -1099,22 +1104,6 @@ void mlxsw_sp_acl_fini(struct mlxsw_sp *mlxsw_sp) kfree(acl); } -u32 mlxsw_sp_acl_region_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp) -{ - struct mlxsw_sp_acl *acl = mlxsw_sp->acl; - - return mlxsw_sp_acl_tcam_vregion_rehash_intrvl_get(mlxsw_sp, - &acl->tcam); -} - -int mlxsw_sp_acl_region_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, u32 val) -{ - struct mlxsw_sp_acl *acl = mlxsw_sp->acl; - - return mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(mlxsw_sp, - &acl->tcam, val); -} - struct mlxsw_sp_acl_rulei_ops mlxsw_sp1_acl_rulei_ops = { .act_mangle_field = mlxsw_sp1_acl_rulei_act_mangle_field, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c index 3b9ba8fa247a..d50786b0a6ce 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c @@ -9,6 +9,7 @@ #include <linux/rhashtable.h> #include <linux/netdevice.h> #include <linux/mutex.h> +#include <net/devlink.h> #include <trace/events/mlxsw.h> #include "reg.h" @@ -29,67 +30,6 @@ size_t mlxsw_sp_acl_tcam_priv_size(struct mlxsw_sp *mlxsw_sp) #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN 3000 /* ms */ #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS 100 /* number of entries */ -int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - u64 max_tcam_regions; - u64 max_regions; - u64 max_groups; - int err; - - mutex_init(&tcam->lock); - tcam->vregion_rehash_intrvl = - MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT; - INIT_LIST_HEAD(&tcam->vregion_list); - - max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, - ACL_MAX_TCAM_REGIONS); - max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); - - /* Use 1:1 mapping between ACL region and TCAM region */ - if (max_tcam_regions < max_regions) - max_regions = max_tcam_regions; - - tcam->used_regions = bitmap_zalloc(max_regions, GFP_KERNEL); - if (!tcam->used_regions) - return -ENOMEM; - tcam->max_regions = max_regions; - - max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS); - tcam->used_groups = bitmap_zalloc(max_groups, GFP_KERNEL); - if (!tcam->used_groups) { - err = -ENOMEM; - goto err_alloc_used_groups; - } - tcam->max_groups = max_groups; - tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, - ACL_MAX_GROUP_SIZE); - - err = ops->init(mlxsw_sp, tcam->priv, tcam); - if (err) - goto err_tcam_init; - - return 0; - -err_tcam_init: - bitmap_free(tcam->used_groups); -err_alloc_used_groups: - bitmap_free(tcam->used_regions); - return err; -} - -void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - - mutex_destroy(&tcam->lock); - ops->fini(mlxsw_sp, tcam->priv); - bitmap_free(tcam->used_groups); - bitmap_free(tcam->used_regions); -} - int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, u32 *priority, bool fillup_priority) @@ -893,41 +833,6 @@ mlxsw_sp_acl_tcam_vregion_destroy(struct mlxsw_sp *mlxsw_sp, kfree(vregion); } -u32 mlxsw_sp_acl_tcam_vregion_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - u32 vregion_rehash_intrvl; - - if (WARN_ON(!ops->region_rehash_hints_get)) - return 0; - vregion_rehash_intrvl = tcam->vregion_rehash_intrvl; - return vregion_rehash_intrvl; -} - -int mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam, - u32 val) -{ - const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; - struct mlxsw_sp_acl_tcam_vregion *vregion; - - if (val < MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN && val) - return -EINVAL; - if (WARN_ON(!ops->region_rehash_hints_get)) - return -EOPNOTSUPP; - tcam->vregion_rehash_intrvl = val; - mutex_lock(&tcam->lock); - list_for_each_entry(vregion, &tcam->vregion_list, tlist) { - if (val) - mlxsw_core_schedule_dw(&vregion->rehash.dw, 0); - else - cancel_delayed_work_sync(&vregion->rehash.dw); - } - mutex_unlock(&tcam->lock); - return 0; -} - static struct mlxsw_sp_acl_tcam_vregion * mlxsw_sp_acl_tcam_vregion_get(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_tcam_vgroup *vgroup, @@ -1542,6 +1447,153 @@ mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_acl_tcam_vregion_rehash_end(mlxsw_sp, vregion, ctx); } +static int +mlxsw_sp_acl_tcam_region_rehash_intrvl_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct mlxsw_core *mlxsw_core = devlink_priv(devlink); + struct mlxsw_sp_acl_tcam *tcam; + struct mlxsw_sp *mlxsw_sp; + + mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + tcam = mlxsw_sp_acl_to_tcam(mlxsw_sp->acl); + ctx->val.vu32 = tcam->vregion_rehash_intrvl; + + return 0; +} + +static int +mlxsw_sp_acl_tcam_region_rehash_intrvl_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct mlxsw_core *mlxsw_core = devlink_priv(devlink); + struct mlxsw_sp_acl_tcam_vregion *vregion; + struct mlxsw_sp_acl_tcam *tcam; + struct mlxsw_sp *mlxsw_sp; + u32 val = ctx->val.vu32; + + if (val < MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN && val) + return -EINVAL; + + mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + tcam = mlxsw_sp_acl_to_tcam(mlxsw_sp->acl); + tcam->vregion_rehash_intrvl = val; + mutex_lock(&tcam->lock); + list_for_each_entry(vregion, &tcam->vregion_list, tlist) { + if (val) + mlxsw_core_schedule_dw(&vregion->rehash.dw, 0); + else + cancel_delayed_work_sync(&vregion->rehash.dw); + } + mutex_unlock(&tcam->lock); + return 0; +} + +static const struct devlink_param mlxsw_sp_acl_tcam_rehash_params[] = { + DEVLINK_PARAM_DRIVER(MLXSW_DEVLINK_PARAM_ID_ACL_REGION_REHASH_INTERVAL, + "acl_region_rehash_interval", + DEVLINK_PARAM_TYPE_U32, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + mlxsw_sp_acl_tcam_region_rehash_intrvl_get, + mlxsw_sp_acl_tcam_region_rehash_intrvl_set, + NULL), +}; + +static int mlxsw_sp_acl_tcam_rehash_params_register(struct mlxsw_sp *mlxsw_sp) +{ + struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); + + if (!mlxsw_sp->acl_tcam_ops->region_rehash_hints_get) + return 0; + + return devl_params_register(devlink, mlxsw_sp_acl_tcam_rehash_params, + ARRAY_SIZE(mlxsw_sp_acl_tcam_rehash_params)); +} + +static void +mlxsw_sp_acl_tcam_rehash_params_unregister(struct mlxsw_sp *mlxsw_sp) +{ + struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); + + if (!mlxsw_sp->acl_tcam_ops->region_rehash_hints_get) + return; + + devl_params_unregister(devlink, mlxsw_sp_acl_tcam_rehash_params, + ARRAY_SIZE(mlxsw_sp_acl_tcam_rehash_params)); +} + +int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_tcam *tcam) +{ + const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; + u64 max_tcam_regions; + u64 max_regions; + u64 max_groups; + int err; + + mutex_init(&tcam->lock); + tcam->vregion_rehash_intrvl = + MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT; + INIT_LIST_HEAD(&tcam->vregion_list); + + err = mlxsw_sp_acl_tcam_rehash_params_register(mlxsw_sp); + if (err) + goto err_rehash_params_register; + + max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, + ACL_MAX_TCAM_REGIONS); + max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS); + + /* Use 1:1 mapping between ACL region and TCAM region */ + if (max_tcam_regions < max_regions) + max_regions = max_tcam_regions; + + tcam->used_regions = bitmap_zalloc(max_regions, GFP_KERNEL); + if (!tcam->used_regions) { + err = -ENOMEM; + goto err_alloc_used_regions; + } + tcam->max_regions = max_regions; + + max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS); + tcam->used_groups = bitmap_zalloc(max_groups, GFP_KERNEL); + if (!tcam->used_groups) { + err = -ENOMEM; + goto err_alloc_used_groups; + } + tcam->max_groups = max_groups; + tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, + ACL_MAX_GROUP_SIZE); + + err = ops->init(mlxsw_sp, tcam->priv, tcam); + if (err) + goto err_tcam_init; + + return 0; + +err_tcam_init: + bitmap_free(tcam->used_groups); +err_alloc_used_groups: + bitmap_free(tcam->used_regions); +err_alloc_used_regions: + mlxsw_sp_acl_tcam_rehash_params_unregister(mlxsw_sp); +err_rehash_params_register: + mutex_destroy(&tcam->lock); + return err; +} + +void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_tcam *tcam) +{ + const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops; + + ops->fini(mlxsw_sp, tcam->priv); + bitmap_free(tcam->used_groups); + bitmap_free(tcam->used_regions); + mlxsw_sp_acl_tcam_rehash_params_unregister(mlxsw_sp); + mutex_destroy(&tcam->lock); +} + static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = { MLXSW_AFK_ELEMENT_SRC_SYS_PORT, MLXSW_AFK_ELEMENT_DMAC_32_47, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h index edbbc89e7a71..462bf448497d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h @@ -29,11 +29,6 @@ int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_tcam *tcam); void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_tcam *tcam); -u32 mlxsw_sp_acl_tcam_vregion_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam); -int mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_tcam *tcam, - u32 val); int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei, u32 *priority, bool fillup_priority); |