diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 8d0c605d4cdb18..4f176bd8395a76 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -470,7 +470,7 @@ static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) */ return; - err = mlx5e_ipsec_aso_query(sa_entry); + err = mlx5e_ipsec_aso_query(sa_entry, NULL); if (err) return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index aac1e6a83631d2..e7f21e449268e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -34,8 +34,6 @@ #ifndef __MLX5E_IPSEC_H__ #define __MLX5E_IPSEC_H__ -#ifdef CONFIG_MLX5_EN_IPSEC - #include #include #include @@ -114,10 +112,21 @@ struct mlx5e_ipsec_sw_stats { struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_work { + struct work_struct work; + struct mlx5e_ipsec *ipsec; + u32 id; +}; + struct mlx5e_ipsec_aso { u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; dma_addr_t dma_addr; struct mlx5_aso *aso; + /* IPsec ASO caches data on every query call, + * so in nested calls, we can use this boolean to save + * recursive calls to mlx5e_ipsec_aso_query() + */ + u8 use_cache : 1; }; struct mlx5e_ipsec { @@ -131,6 +140,7 @@ struct mlx5e_ipsec { struct mlx5e_ipsec_rx *rx_ipv6; struct mlx5e_ipsec_tx *tx; struct mlx5e_ipsec_aso *aso; + struct notifier_block nb; }; struct mlx5e_ipsec_esn_state { @@ -188,6 +198,8 @@ struct mlx5e_ipsec_pol_entry { struct mlx5_accel_pol_xfrm_attrs attrs; }; +#ifdef CONFIG_MLX5_EN_IPSEC + void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); @@ -210,7 +222,8 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); -int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data); void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, u64 *packets); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 8790558ea8591f..1b5014ffa2574e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -260,6 +260,73 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs)); } +static void mlx5e_ipsec_handle_event(struct work_struct *_work) +{ + struct mlx5e_ipsec_work *work = + container_of(_work, struct mlx5e_ipsec_work, work); + struct mlx5_accel_esp_xfrm_attrs *attrs; + struct mlx5e_ipsec_sa_entry *sa_entry; + struct mlx5e_ipsec_aso *aso; + struct mlx5e_ipsec *ipsec; + int ret; + + sa_entry = xa_load(&work->ipsec->sadb, work->id); + if (!sa_entry) + goto out; + + ipsec = sa_entry->ipsec; + aso = ipsec->aso; + attrs = &sa_entry->attrs; + + spin_lock(&sa_entry->x->lock); + ret = mlx5e_ipsec_aso_query(sa_entry, NULL); + if (ret) + goto unlock; + + aso->use_cache = true; + if (attrs->soft_packet_limit != XFRM_INF) + if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) || + !MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable)) + xfrm_state_check_expire(sa_entry->x); + aso->use_cache = false; + +unlock: + spin_unlock(&sa_entry->x->lock); +out: + kfree(work); +} + +static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb); + struct mlx5_eqe_obj_change *object; + struct mlx5e_ipsec_work *work; + struct mlx5_eqe *eqe = data; + u16 type; + + if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE) + return NOTIFY_DONE; + + object = &eqe->data.obj_change; + type = be16_to_cpu(object->obj_type); + + if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5e_ipsec_handle_event); + work->ipsec = ipsec; + work->id = be32_to_cpu(object->obj_id); + + queue_work(ipsec->wq, &work->work); + return NOTIFY_OK; +} + int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) { struct mlx5_core_dev *mdev = ipsec->mdev; @@ -287,6 +354,9 @@ int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec) goto err_aso_create; } + ipsec->nb.notifier_call = mlx5e_ipsec_event; + mlx5_notifier_register(mdev, &ipsec->nb); + ipsec->aso = aso; return 0; @@ -307,13 +377,33 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec) aso = ipsec->aso; pdev = mlx5_core_dma_dev(mdev); + mlx5_notifier_unregister(mdev, &ipsec->nb); mlx5_aso_destroy(aso->aso); dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL); kfree(aso); } -int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) +static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl, + struct mlx5_wqe_aso_ctrl_seg *data) +{ + if (!data) + return; + + ctrl->data_mask_mode = data->data_mask_mode; + ctrl->condition_1_0_operand = data->condition_1_0_operand; + ctrl->condition_1_0_offset = data->condition_1_0_offset; + ctrl->data_offset_condition_operand = data->data_offset_condition_operand; + ctrl->condition_0_data = data->condition_0_data; + ctrl->condition_0_mask = data->condition_0_mask; + ctrl->condition_1_data = data->condition_1_data; + ctrl->condition_1_mask = data->condition_1_mask; + ctrl->bitwise_data = data->bitwise_data; + ctrl->data_mask = data->data_mask; +} + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data) { struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5e_ipsec_aso *aso = ipsec->aso; @@ -323,6 +413,10 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) struct mlx5_aso_wqe *wqe; u8 ds_cnt; + lockdep_assert_held(&sa_entry->x->lock); + if (aso->use_cache) + return 0; + res = &mdev->mlx5e_res.hw_objs; memset(aso->ctx, 0, sizeof(aso->ctx)); @@ -336,6 +430,7 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry) cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN); ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr)); ctrl->l_key = cpu_to_be32(res->mkey); + mlx5e_ipsec_aso_copy(ctrl, data); mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); return mlx5_aso_poll_cq(aso->aso, false); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index a0242dc15741c7..8f7580fec1937d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -19,6 +19,7 @@ #include "diag/fw_tracer.h" #include "mlx5_irq.h" #include "devlink.h" +#include "en_accel/ipsec.h" enum { MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -578,6 +579,10 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + async_event_mask |= + (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + mask[0] = async_event_mask; if (MLX5_CAP_GEN(dev, event_cap))