Lines Matching refs:wr
2298 struct ib_ud_wr *wr, in build_sriov_qp0_header() argument
2305 struct mlx4_ib_ah *ah = to_mah(wr->ah); in build_sriov_qp0_header()
2313 if (wr->wr.opcode != IB_WR_SEND) in build_sriov_qp0_header()
2318 for (i = 0; i < wr->wr.num_sge; ++i) in build_sriov_qp0_header()
2319 send_size += wr->wr.sg_list[i].length; in build_sriov_qp0_header()
2344 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED); in build_sriov_qp0_header()
2348 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn); in build_sriov_qp0_header()
2427 static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr, in build_mlx_header() argument
2434 struct mlx4_ib_ah *ah = to_mah(wr->ah); in build_mlx_header()
2450 for (i = 0; i < wr->wr.num_sge; ++i) in build_mlx_header()
2451 send_size += wr->wr.sg_list[i].length; in build_mlx_header()
2571 switch (wr->wr.opcode) { in build_mlx_header()
2579 sqp->ud_header.immediate_data = wr->wr.ex.imm_data; in build_mlx_header()
2620 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED); in build_mlx_header()
2624 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, &pkey); in build_mlx_header()
2626 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn); in build_mlx_header()
2628 sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ? in build_mlx_header()
2629 sqp->qkey : wr->remote_qkey); in build_mlx_header()
2718 struct ib_reg_wr *wr) in set_reg_seg() argument
2720 struct mlx4_ib_mr *mr = to_mmr(wr->mr); in set_reg_seg()
2722 fseg->flags = convert_access(wr->access); in set_reg_seg()
2723 fseg->mem_key = cpu_to_be32(wr->key); in set_reg_seg()
2748 struct ib_atomic_wr *wr) in set_atomic_seg() argument
2750 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { in set_atomic_seg()
2751 aseg->swap_add = cpu_to_be64(wr->swap); in set_atomic_seg()
2752 aseg->compare = cpu_to_be64(wr->compare_add); in set_atomic_seg()
2753 } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) { in set_atomic_seg()
2754 aseg->swap_add = cpu_to_be64(wr->compare_add); in set_atomic_seg()
2755 aseg->compare = cpu_to_be64(wr->compare_add_mask); in set_atomic_seg()
2757 aseg->swap_add = cpu_to_be64(wr->compare_add); in set_atomic_seg()
2764 struct ib_atomic_wr *wr) in set_masked_atomic_seg() argument
2766 aseg->swap_add = cpu_to_be64(wr->swap); in set_masked_atomic_seg()
2767 aseg->swap_add_mask = cpu_to_be64(wr->swap_mask); in set_masked_atomic_seg()
2768 aseg->compare = cpu_to_be64(wr->compare_add); in set_masked_atomic_seg()
2769 aseg->compare_mask = cpu_to_be64(wr->compare_add_mask); in set_masked_atomic_seg()
2773 struct ib_ud_wr *wr) in set_datagram_seg() argument
2775 memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av)); in set_datagram_seg()
2776 dseg->dqpn = cpu_to_be32(wr->remote_qpn); in set_datagram_seg()
2777 dseg->qkey = cpu_to_be32(wr->remote_qkey); in set_datagram_seg()
2778 dseg->vlan = to_mah(wr->ah)->av.eth.vlan; in set_datagram_seg()
2779 memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6); in set_datagram_seg()
2784 struct ib_ud_wr *wr, in set_tunnel_datagram_seg() argument
2787 union mlx4_ext_av *av = &to_mah(wr->ah)->av; in set_tunnel_datagram_seg()
2806 static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len) in build_tunnel_header() argument
2810 struct mlx4_ib_ah *ah = to_mah(wr->ah); in build_tunnel_header()
2815 hdr.remote_qpn = cpu_to_be32(wr->remote_qpn); in build_tunnel_header()
2816 hdr.pkey_index = cpu_to_be16(wr->pkey_index); in build_tunnel_header()
2817 hdr.qkey = cpu_to_be32(wr->remote_qkey); in build_tunnel_header()
2889 static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr, in build_lso_seg() argument
2893 unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16); in build_lso_seg()
2899 wr->wr.num_sge > qp->sq.max_gs - (halign >> 4))) in build_lso_seg()
2902 memcpy(wqe->header, wr->header, wr->hlen); in build_lso_seg()
2904 *lso_hdr_sz = cpu_to_be32(wr->mss << 16 | wr->hlen); in build_lso_seg()
2909 static __be32 send_ieth(struct ib_send_wr *wr) in send_ieth() argument
2911 switch (wr->opcode) { in send_ieth()
2914 return wr->ex.imm_data; in send_ieth()
2917 return cpu_to_be32(wr->ex.invalidate_rkey); in send_ieth()
2931 int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, in mlx4_ib_post_send() argument
2956 struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah); in mlx4_ib_post_send()
2978 *bad_wr = wr; in mlx4_ib_post_send()
2985 for (nreq = 0; wr; ++nreq, wr = wr->next) { in mlx4_ib_post_send()
2991 *bad_wr = wr; in mlx4_ib_post_send()
2995 if (unlikely(wr->num_sge > qp->sq.max_gs)) { in mlx4_ib_post_send()
2997 *bad_wr = wr; in mlx4_ib_post_send()
3002 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id; in mlx4_ib_post_send()
3005 (wr->send_flags & IB_SEND_SIGNALED ? in mlx4_ib_post_send()
3007 (wr->send_flags & IB_SEND_SOLICITED ? in mlx4_ib_post_send()
3009 ((wr->send_flags & IB_SEND_IP_CSUM) ? in mlx4_ib_post_send()
3014 ctrl->imm = send_ieth(wr); in mlx4_ib_post_send()
3022 switch (wr->opcode) { in mlx4_ib_post_send()
3026 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, in mlx4_ib_post_send()
3027 atomic_wr(wr)->rkey); in mlx4_ib_post_send()
3030 set_atomic_seg(wqe, atomic_wr(wr)); in mlx4_ib_post_send()
3039 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr, in mlx4_ib_post_send()
3040 atomic_wr(wr)->rkey); in mlx4_ib_post_send()
3043 set_masked_atomic_seg(wqe, atomic_wr(wr)); in mlx4_ib_post_send()
3054 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr, in mlx4_ib_post_send()
3055 rdma_wr(wr)->rkey); in mlx4_ib_post_send()
3063 set_local_inv_seg(wqe, wr->ex.invalidate_rkey); in mlx4_ib_post_send()
3071 set_reg_seg(wqe, reg_wr(wr)); in mlx4_ib_post_send()
3083 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), in mlx4_ib_post_send()
3086 *bad_wr = wr; in mlx4_ib_post_send()
3095 set_datagram_seg(wqe, ud_wr(wr)); in mlx4_ib_post_send()
3102 set_datagram_seg(wqe, ud_wr(wr)); in mlx4_ib_post_send()
3106 if (wr->opcode == IB_WR_LSO) { in mlx4_ib_post_send()
3107 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen, in mlx4_ib_post_send()
3110 *bad_wr = wr; in mlx4_ib_post_send()
3120 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr), in mlx4_ib_post_send()
3123 *bad_wr = wr; in mlx4_ib_post_send()
3132 build_tunnel_header(ud_wr(wr), wqe, &seglen); in mlx4_ib_post_send()
3143 ud_wr(wr), in mlx4_ib_post_send()
3147 build_tunnel_header(ud_wr(wr), wqe, &seglen); in mlx4_ib_post_send()
3154 err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl, in mlx4_ib_post_send()
3157 *bad_wr = wr; in mlx4_ib_post_send()
3176 dseg += wr->num_sge - 1; in mlx4_ib_post_send()
3177 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16); in mlx4_ib_post_send()
3188 for (i = wr->num_sge - 1; i >= 0; --i, --dseg) in mlx4_ib_post_send()
3189 set_data_seg(dseg, wr->sg_list + i); in mlx4_ib_post_send()
3199 ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ? in mlx4_ib_post_send()
3209 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) { in mlx4_ib_post_send()
3210 *bad_wr = wr; in mlx4_ib_post_send()
3215 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] | in mlx4_ib_post_send()
3230 if (wr->next) { in mlx4_ib_post_send()
3266 int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, in mlx4_ib_post_recv() argument
3284 *bad_wr = wr; in mlx4_ib_post_recv()
3291 for (nreq = 0; wr; ++nreq, wr = wr->next) { in mlx4_ib_post_recv()
3294 *bad_wr = wr; in mlx4_ib_post_recv()
3298 if (unlikely(wr->num_sge > qp->rq.max_gs)) { in mlx4_ib_post_recv()
3300 *bad_wr = wr; in mlx4_ib_post_recv()
3315 scat->lkey = cpu_to_be32(wr->sg_list->lkey); in mlx4_ib_post_recv()
3321 for (i = 0; i < wr->num_sge; ++i) in mlx4_ib_post_recv()
3322 __set_data_seg(scat + i, wr->sg_list + i); in mlx4_ib_post_recv()
3330 qp->rq.wrid[ind] = wr->wr_id; in mlx4_ib_post_recv()