2951 			 struct ib_send_wr *wr, void *qend,  in set_eth_seg()  argument
2958 	if (wr->send_flags & IB_SEND_IP_CSUM)  in set_eth_seg()
2965 	if (wr->opcode == IB_WR_LSO) {  in set_eth_seg()
2966 		struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);  in set_eth_seg()
3004 			     struct ib_send_wr *wr)  in set_datagram_seg()  argument
3006 	memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));  in set_datagram_seg()
3007 	dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);  in set_datagram_seg()
3008 	dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);  in set_datagram_seg()
3163 				struct ib_send_wr *wr)  in set_reg_umr_segment()  argument
3165 	struct mlx5_umr_wr *umrwr = umr_wr(wr);  in set_reg_umr_segment()
3169 	if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)  in set_reg_umr_segment()
3174 	if (!(wr->send_flags & MLX5_IB_SEND_UMR_UNREG)) {  in set_reg_umr_segment()
3176 		if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT) {  in set_reg_umr_segment()
3181 		if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)  in set_reg_umr_segment()
3183 		if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_ACCESS)  in set_reg_umr_segment()
3185 		if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD)  in set_reg_umr_segment()
3193 	if (!wr->num_sge)  in set_reg_umr_segment()
3234 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr)  in set_reg_mkey_segment()  argument
3236 	struct mlx5_umr_wr *umrwr = umr_wr(wr);  in set_reg_mkey_segment()
3239 	if (wr->send_flags & MLX5_IB_SEND_UMR_UNREG) {  in set_reg_mkey_segment()
3245 	if (!(wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT)) {  in set_reg_mkey_segment()
3267 static __be32 send_ieth(struct ib_send_wr *wr)  in send_ieth()  argument
3269 	switch (wr->opcode) {  in send_ieth()
3272 		return wr->ex.imm_data;  in send_ieth()
3275 		return cpu_to_be32(wr->ex.invalidate_rkey);  in send_ieth()
3299 static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,  in set_data_inl_seg()  argument
3312 	for (i = 0; i < wr->num_sge; i++) {  in set_data_inl_seg()
3313 		addr = (void *)(unsigned long)(wr->sg_list[i].addr);  in set_data_inl_seg()
3314 		len  = wr->sg_list[i].length;  in set_data_inl_seg()
3445 static int set_sig_data_segment(struct ib_sig_handover_wr *wr,  in set_sig_data_segment()  argument
3448 	struct ib_sig_attrs *sig_attrs = wr->sig_attrs;  in set_sig_data_segment()
3449 	struct ib_mr *sig_mr = wr->sig_mr;  in set_sig_data_segment()
3451 	u32 data_len = wr->wr.sg_list->length;  in set_sig_data_segment()
3452 	u32 data_key = wr->wr.sg_list->lkey;  in set_sig_data_segment()
3453 	u64 data_va = wr->wr.sg_list->addr;  in set_sig_data_segment()
3457 	if (!wr->prot ||  in set_sig_data_segment()
3458 	    (data_key == wr->prot->lkey &&  in set_sig_data_segment()
3459 	     data_va == wr->prot->addr &&  in set_sig_data_segment()
3460 	     data_len == wr->prot->length)) {  in set_sig_data_segment()
3494 		u32 prot_key = wr->prot->lkey;  in set_sig_data_segment()
3495 		u64 prot_va = wr->prot->addr;  in set_sig_data_segment()
3547 				 struct ib_sig_handover_wr *wr, u32 nelements,  in set_sig_mkey_segment()  argument
3550 	struct ib_mr *sig_mr = wr->sig_mr;  in set_sig_mkey_segment()
3556 	seg->flags = get_umr_flags(wr->access_flags) |  in set_sig_mkey_segment()
3581 	struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);  in set_sig_umr_wr()  local
3582 	struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);  in set_sig_umr_wr()
3587 	if (unlikely(wr->wr.num_sge != 1) ||  in set_sig_umr_wr()
3588 	    unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) ||  in set_sig_umr_wr()
3594 	region_len = wr->wr.sg_list->length;  in set_sig_umr_wr()
3595 	if (wr->prot &&  in set_sig_umr_wr()
3596 	    (wr->prot->lkey != wr->wr.sg_list->lkey  ||  in set_sig_umr_wr()
3597 	     wr->prot->addr != wr->wr.sg_list->addr  ||  in set_sig_umr_wr()
3598 	     wr->prot->length != wr->wr.sg_list->length))  in set_sig_umr_wr()
3599 		region_len += wr->prot->length;  in set_sig_umr_wr()
3606 	klm_oct_size = wr->prot ? 3 : 1;  in set_sig_umr_wr()
3614 	set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);  in set_sig_umr_wr()
3620 	ret = set_sig_data_segment(wr, qp, seg, size);  in set_sig_umr_wr()
3655 		      struct ib_reg_wr *wr,  in set_reg_wr()  argument
3658 	struct mlx5_ib_mr *mr = to_mmr(wr->mr);  in set_reg_wr()
3661 	if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {  in set_reg_wr()
3673 	set_reg_mkey_seg(*seg, mr, wr->key, wr->access);  in set_reg_wr()
3738 static u8 get_fence(u8 fence, struct ib_send_wr *wr)  in get_fence()  argument
3740 	if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&  in get_fence()
3741 		     wr->send_flags & IB_SEND_FENCE))  in get_fence()
3745 		if (wr->send_flags & IB_SEND_FENCE)  in get_fence()
3749 	} else if (unlikely(wr->send_flags & IB_SEND_FENCE)) {  in get_fence()
3758 		     struct ib_send_wr *wr, unsigned *idx,  in begin_wqe()  argument
3768 	(*ctrl)->imm = send_ieth(wr);  in begin_wqe()
3770 		(wr->send_flags & IB_SEND_SIGNALED ?  in begin_wqe()
3772 		(wr->send_flags & IB_SEND_SOLICITED ?  in begin_wqe()
3805 int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,  in mlx5_ib_post_send()  argument
3830 		return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);  in mlx5_ib_post_send()
3840 		*bad_wr = wr;  in mlx5_ib_post_send()
3845 	for (nreq = 0; wr; nreq++, wr = wr->next) {  in mlx5_ib_post_send()
3846 		if (unlikely(wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {  in mlx5_ib_post_send()
3849 			*bad_wr = wr;  in mlx5_ib_post_send()
3854 		num_sge = wr->num_sge;  in mlx5_ib_post_send()
3858 			*bad_wr = wr;  in mlx5_ib_post_send()
3862 		err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);  in mlx5_ib_post_send()
3866 			*bad_wr = wr;  in mlx5_ib_post_send()
3877 			switch (wr->opcode) {  in mlx5_ib_post_send()
3881 				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,  in mlx5_ib_post_send()
3882 					      rdma_wr(wr)->rkey);  in mlx5_ib_post_send()
3892 				*bad_wr = wr;  in mlx5_ib_post_send()
3898 				ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);  in mlx5_ib_post_send()
3906 				ctrl->imm = cpu_to_be32(reg_wr(wr)->key);  in mlx5_ib_post_send()
3907 				err = set_reg_wr(qp, reg_wr(wr), &seg, &size);  in mlx5_ib_post_send()
3909 					*bad_wr = wr;  in mlx5_ib_post_send()
3917 				mr = to_mmr(sig_handover_wr(wr)->sig_mr);  in mlx5_ib_post_send()
3920 				err = set_sig_umr_wr(wr, qp, &seg, &size);  in mlx5_ib_post_send()
3923 					*bad_wr = wr;  in mlx5_ib_post_send()
3927 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,  in mlx5_ib_post_send()
3928 					   nreq, get_fence(fence, wr),  in mlx5_ib_post_send()
3934 				wr->send_flags &= ~IB_SEND_SIGNALED;  in mlx5_ib_post_send()
3935 				wr->send_flags |= IB_SEND_SOLICITED;  in mlx5_ib_post_send()
3936 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
3941 					*bad_wr = wr;  in mlx5_ib_post_send()
3945 				err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,  in mlx5_ib_post_send()
3950 					*bad_wr = wr;  in mlx5_ib_post_send()
3954 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,  in mlx5_ib_post_send()
3955 					   nreq, get_fence(fence, wr),  in mlx5_ib_post_send()
3957 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
3962 					*bad_wr = wr;  in mlx5_ib_post_send()
3967 				err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,  in mlx5_ib_post_send()
3972 					*bad_wr = wr;  in mlx5_ib_post_send()
3976 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,  in mlx5_ib_post_send()
3977 					   nreq, get_fence(fence, wr),  in mlx5_ib_post_send()
3988 			switch (wr->opcode) {  in mlx5_ib_post_send()
3991 				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,  in mlx5_ib_post_send()
3992 					      rdma_wr(wr)->rkey);  in mlx5_ib_post_send()
4004 			set_datagram_seg(seg, wr);  in mlx5_ib_post_send()
4011 			set_datagram_seg(seg, wr);  in mlx5_ib_post_send()
4027 				seg = set_eth_seg(seg, wr, qend, qp, &size);  in mlx5_ib_post_send()
4034 			if (wr->opcode != MLX5_IB_WR_UMR) {  in mlx5_ib_post_send()
4040 			ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);  in mlx5_ib_post_send()
4041 			set_reg_umr_segment(seg, wr);  in mlx5_ib_post_send()
4046 			set_reg_mkey_segment(seg, wr);  in mlx5_ib_post_send()
4057 		if (wr->send_flags & IB_SEND_INLINE && num_sge) {  in mlx5_ib_post_send()
4060 			err = set_data_inl_seg(qp, wr, seg, &sz);  in mlx5_ib_post_send()
4063 				*bad_wr = wr;  in mlx5_ib_post_send()
4075 				if (likely(wr->sg_list[i].length)) {  in mlx5_ib_post_send()
4076 					set_data_ptr_seg(dpseg, wr->sg_list + i);  in mlx5_ib_post_send()
4083 		finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,  in mlx5_ib_post_send()
4084 			   get_fence(fence, wr), next_fence,  in mlx5_ib_post_send()
4085 			   mlx5_ib_opcode[wr->opcode]);  in mlx5_ib_post_send()
4140 int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,  in mlx5_ib_post_recv()  argument
4155 		return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);  in mlx5_ib_post_recv()
4161 		*bad_wr = wr;  in mlx5_ib_post_recv()
4168 	for (nreq = 0; wr; nreq++, wr = wr->next) {  in mlx5_ib_post_recv()
4171 			*bad_wr = wr;  in mlx5_ib_post_recv()
4175 		if (unlikely(wr->num_sge > qp->rq.max_gs)) {  in mlx5_ib_post_recv()
4177 			*bad_wr = wr;  in mlx5_ib_post_recv()
4185 		for (i = 0; i < wr->num_sge; i++)  in mlx5_ib_post_recv()
4186 			set_data_ptr_seg(scat + i, wr->sg_list + i);  in mlx5_ib_post_recv()
4199 		qp->rq.wrid[ind] = wr->wr_id;  in mlx5_ib_post_recv()