1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 #if HAVE_CONFIG_H
37 # include <config.h>
38 #endif /* HAVE_CONFIG_H */
39
40 #ifdef OSM_VENDOR_INTF_MTL
41
42 #include <stdlib.h>
43 #include <string.h>
44 #include <opensm/osm_helper.h>
45 #include <opensm/osm_log.h>
46 /* HACK - I do not know how to prevent complib from loading kernel H files */
47 #undef __init
48 #include <vendor/osm_vendor_mtl.h>
49 #include <vendor/osm_vendor_api.h>
50 #include <opensm/osm_subnet.h>
51 #include <opensm/osm_opensm.h>
52 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
53 #include <vendor/osm_mtl_bind.h>
54
55 /*
56 Since a race can accure on requests. Meaning - a response is received before
57 the send_callback is called - we will save both the madw_p and the fact
58 whether or not it is a response. A race can occure only on requests that did
59 not fail, and then the madw_p will be put back in the pool before the callback.
60 */
__osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)61 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
62 {
63 uint64_t wrid = 0;
64
65 CL_ASSERT(p_madw->p_mad);
66
67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
68 wrid = (wrid << 1) |
69 ib_mad_is_response(p_madw->p_mad) |
70 (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS);
71 return wrid;
72 }
73
74 void
__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,OUT uint8_t * is_resp,OUT osm_madw_t ** pp_madw)75 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
76 OUT uint8_t * is_resp,
77 OUT osm_madw_t ** pp_madw)
78 {
79 *is_resp = wrid & 0x0000000000000001;
80 wrid = wrid >> 1;
81 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
82 }
83
84 /**********************************************************************
85 * IB_MGT to OSM ADDRESS VECTOR
86 **********************************************************************/
87 void
__osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)88 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
89 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
90 IN uint8_t is_smi,
91 OUT osm_mad_addr_t * p_mad_addr)
92 {
93 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
94 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
95 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
96 p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
97 if (is_smi) {
98 /* SMI */
99 p_mad_addr->addr_type.smi.source_lid =
100 cl_hton16(p_rcv_desc->remote_lid);
101 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */
102 } else {
103 /* GSI */
104 /* seems to me there is a IBMGT bug reversing the QPN ... */
105 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
106 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
107
108 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
109 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
110 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
111 /* the full PKey table - than go by the index. */
112 /* since this does not seem reasonable to me I simply use the default */
113 /* There is a TAVOR limitation that only one P_KEY is supported per */
114 /* QP - so QP1 must use IB_DEFAULT_PKEY */
115 p_mad_addr->addr_type.gsi.pkey_ix = 0;
116 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
117
118 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
119 /* copy the GRH data if relevant */
120 if (p_mad_addr->addr_type.gsi.global_route) {
121 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
122 ib_grh_set_ver_class_flow(p_rcv_desc->grh.
123 IP_version,
124 p_rcv_desc->grh.
125 traffic_class,
126 p_rcv_desc->grh.
127 flow_label);
128 p_mad_addr->addr_type.gsi.grh_info.hop_limit =
129 p_rcv_desc->grh.hop_limit;
130 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
131 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
132 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
133 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
134 }
135 }
136 }
137
138 /**********************************************************************
139 * OSM ADDR VECTOR TO IB_MGT
140 **********************************************************************/
141 void
__osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT IB_ud_av_t * p_av)142 __osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr,
143 IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
144 {
145
146 /* For global destination or Multicast address: */
147 u_int8_t ver;
148
149 memset(p_av, 0, sizeof(IB_ud_av_t));
150
151 p_av->src_path_bits = p_mad_addr->path_bits;
152 p_av->static_rate = p_mad_addr->static_rate;
153 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
154
155 if (is_smi) {
156 p_av->sl = 0; /* Just to note we use 0 here. */
157 } else {
158 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
159 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
160
161 if (p_mad_addr->addr_type.gsi.global_route) {
162 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
163 grh_info.ver_class_flow, &ver,
164 &p_av->traffic_class,
165 &p_av->flow_label);
166 p_av->hop_limit =
167 p_mad_addr->addr_type.gsi.grh_info.hop_limit;
168 p_av->sgid_index = 0; /* we always use source GID 0 */
169 memcpy(&p_av->dgid,
170 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
171 sizeof(ib_net64_t));
172
173 }
174 }
175 }
176
177 /**********************************************************************
178 **********************************************************************/
__osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)179 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
180 {
181 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
182 osm_vendor_t *p_vend = p_bind->p_vend;
183 VAPI_ret_t status;
184 VAPI_hca_attr_t attr_mod;
185 VAPI_hca_attr_mask_t attr_mask;
186
187 OSM_LOG_ENTER(p_vend->p_log);
188
189 memset(&attr_mod, 0, sizeof(attr_mod));
190 memset(&attr_mask, 0, sizeof(attr_mask));
191
192 attr_mod.is_sm = FALSE;
193 attr_mask = HCA_ATTR_IS_SM;
194
195 status =
196 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
197 &attr_mask);
198 if (status != VAPI_OK) {
199 osm_log(p_vend->p_log, OSM_LOG_ERROR,
200 "__osm_vendor_clear_sm: ERR 3C21: "
201 "Unable set 'IS_SM' bit in port attributes (%d).\n",
202 status);
203 }
204
205 OSM_LOG_EXIT(p_vend->p_log);
206 }
207
208 /**********************************************************************
209 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
210 **********************************************************************/
osm_vendor_construct(IN osm_vendor_t * const p_vend)211 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
212 {
213 memset(p_vend, 0, sizeof(*p_vend));
214 }
215
216 /**********************************************************************
217 * DEALOCATE osm_vendor_t
218 **********************************************************************/
osm_vendor_destroy(IN osm_vendor_t * const p_vend)219 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
220 {
221 osm_vendor_mgt_bind_t *vendor_mgt_bind_p;
222 IB_MGT_ret_t mgt_ret;
223 OSM_LOG_ENTER(p_vend->p_log);
224
225 if (p_vend->h_al != NULL) {
226 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
227 if (vendor_mgt_bind_p->gsi_init) {
228
229 /* un register the class */
230 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */
231 mgt_ret =
232 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p->
233 gsi_mads_hdl,
234 IB_MCLASS_SUBN_ADM);
235 if (mgt_ret != IB_MGT_OK) {
236 osm_log(p_vend->p_log, OSM_LOG_ERROR,
237 "osm_vendor_destroy: ERR 3C03: "
238 "Fail to unbind the SA class.\n");
239 }
240
241 /* un bind the handle */
242 if (IB_MGT_release_handle
243 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) {
244 osm_log(p_vend->p_log, OSM_LOG_ERROR,
245 "osm_vendor_destroy: ERR 3C02: "
246 "Fail to unbind the SA GSI handle.\n");
247 }
248 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
249 "osm_vendor_destroy: DBG 1002: "
250 "Unbind the GSI handles.\n");
251 }
252 if (vendor_mgt_bind_p->smi_init) {
253 /* first - clear the IS_SM in the capability mask */
254 __osm_vendor_clear_sm((osm_bind_handle_t)
255 (vendor_mgt_bind_p->smi_p_bind));
256
257 /* un register the class */
258 mgt_ret =
259 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl);
260 if (mgt_ret != IB_MGT_OK) {
261 osm_log(p_vend->p_log, OSM_LOG_ERROR,
262 "osm_vendor_destroy: ERR 3C04: "
263 "Fail to unbind the SM class.\n");
264 }
265
266 /* un bind the handle */
267 if (IB_MGT_release_handle
268 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) {
269 osm_log(p_vend->p_log, OSM_LOG_ERROR,
270 "osm_vendor_destroy: ERR 3C05: "
271 "Fail to unbind the SMI handle.\n");
272 }
273 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
274 "osm_vendor_destroy: DBG 1003: "
275 "Unbind the SMI handles.\n");
276
277 }
278 }
279 osm_transaction_mgr_destroy(p_vend);
280 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */
281 OSM_LOG_EXIT(p_vend->p_log);
282 }
283
284 /**********************************************************************
285 DEALLOCATE A POINTER TO osm_vendor_t
286 **********************************************************************/
osm_vendor_delete(IN osm_vendor_t ** const pp_vend)287 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
288 {
289 CL_ASSERT(pp_vend);
290
291 osm_vendor_destroy(*pp_vend);
292 free(*pp_vend);
293 *pp_vend = NULL;
294 }
295
296 /**********************************************************************
297 * This proc actuall binds the handle to the lower level.
298 *
299 * We might have here as a result a casting of our struct to the ib_al_handle_t
300 *
301 * Q: Do we need 2 of those - one for MSI and one for GSI ?
302 * A: Yes! We should be able to do the SA too. So we need a struct!
303 *
304 **********************************************************************/
305
306 ib_api_status_t
osm_vendor_init(IN osm_vendor_t * const p_vend,IN osm_log_t * const p_log,IN const uint32_t timeout)307 osm_vendor_init(IN osm_vendor_t * const p_vend,
308 IN osm_log_t * const p_log, IN const uint32_t timeout)
309 {
310 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
311 ib_api_status_t status = IB_SUCCESS;
312
313 OSM_LOG_ENTER(p_log);
314
315 p_vend->p_log = p_log;
316
317 /*
318 * HACK: We need no handle. Assuming the driver is up.
319 */
320 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *)
321 malloc(sizeof(osm_vendor_mgt_bind_t));
322 if (ib_mgt_hdl_p == NULL) {
323 osm_log(p_vend->p_log, OSM_LOG_ERROR,
324 "osm_vendor_init: ERR 3C06: "
325 "Fail to allocate vendor mgt handle.\n");
326 goto Exit;
327 }
328
329 ib_mgt_hdl_p->smi_init = FALSE;
330 ib_mgt_hdl_p->gsi_init = FALSE;
331 /* cast it into the ib_al_handle_t h_al */
332 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p;
333 p_vend->p_transaction_mgr = NULL;
334 osm_transaction_mgr_init(p_vend);
335 /* p_vend->madw_by_tid_map_p = NULL; */
336 /* __osm_mtl_init_tid_mad_map( p_vend ); */
337 p_vend->timeout = timeout;
338
339 Exit:
340 OSM_LOG_EXIT(p_log);
341 return (status);
342 }
343
344 /**********************************************************************
345 * Create and Initialize osm_vendor_t Object
346 **********************************************************************/
osm_vendor_new(IN osm_log_t * const p_log,IN const uint32_t timeout)347 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
348 IN const uint32_t timeout)
349 {
350 ib_api_status_t status;
351 osm_vendor_t *p_vend;
352
353 OSM_LOG_ENTER(p_log);
354
355 CL_ASSERT(p_log);
356
357 p_vend = malloc(sizeof(*p_vend));
358 if (p_vend != NULL) {
359 memset(p_vend, 0, sizeof(*p_vend));
360 status = osm_vendor_init(p_vend, p_log, timeout);
361 if (status != IB_SUCCESS) {
362 osm_vendor_delete(&p_vend);
363 }
364 } else {
365 osm_log(p_vend->p_log, OSM_LOG_ERROR,
366 "osm_vendor_new: ERR 3C07: "
367 "Fail to allocate vendor object.\n");
368 }
369
370 OSM_LOG_EXIT(p_log);
371 return (p_vend);
372 }
373
374 /**********************************************************************
375 * IB_MGT RCV callback
376 *
377 **********************************************************************/
378 void
__osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,IN void * private_ctx_p,IN void * payload_p,IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)379 __osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl,
380 IN void *private_ctx_p,
381 IN void *payload_p,
382 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
383 {
384 IB_MGT_ret_t status;
385 osm_mtl_bind_info_t *bind_info_p = private_ctx_p;
386 osm_madw_t *req_madw_p = NULL;
387 osm_madw_t *madw_p;
388 osm_vend_wrap_t *p_new_vw;
389 osm_mad_addr_t mad_addr;
390 ib_mad_t *mad_buf_p;
391 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
392
393 OSM_LOG_ENTER(p_log);
394
395 /* if it is a response MAD we mustbe able to get the request */
396 if (ib_mad_is_response((ib_mad_t *) payload_p)) {
397 /* can we find a matching madw by this payload TID */
398 status =
399 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend,
400 (ib_mad_t *) payload_p,
401 &req_madw_p);
402 if (status != IB_MGT_OK) {
403 osm_log(p_log, OSM_LOG_ERROR,
404 "__osm_mtl_rcv_callback: ERR 3C08: "
405 "Error obtaining request madw by TID (%d).\n",
406 status);
407 req_madw_p = NULL;
408 }
409
410 if (req_madw_p == NULL) {
411 osm_log(p_log, OSM_LOG_ERROR,
412 "__osm_mtl_rcv_callback: ERR 3C09: "
413 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n",
414 ((ib_mad_t *) payload_p)->method,
415 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id)
416
417 );
418 goto Exit;
419 }
420 }
421
422 /* do we have a request ??? */
423 if (req_madw_p == NULL) {
424
425 /* first arrange an address */
426 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend,
427 rcv_remote_info_p,
428 (((ib_mad_t *)
429 payload_p)->
430 mgmt_class ==
431 IB_MCLASS_SUBN_LID)
432 || (((ib_mad_t *)
433 payload_p)->
434 mgmt_class ==
435 IB_MCLASS_SUBN_DIR),
436 &mad_addr);
437
438 osm_log(p_log, OSM_LOG_ERROR,
439 "__osm_mtl_rcv_callback: : "
440 "Received MAD from QP:%X.\n",
441 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp)
442 );
443
444 /* if not - get new osm_madw and arrange it. */
445 /* create the new madw in the pool */
446 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool,
447 (osm_bind_handle_t) bind_info_p,
448 MAD_BLOCK_SIZE, &mad_addr);
449 if (madw_p == NULL) {
450 osm_log(p_log, OSM_LOG_ERROR,
451 "__osm_mtl_rcv_callback: ERR 3C10: "
452 "Error request for a new madw.\n");
453 goto Exit;
454 }
455 /* HACK: we cust to avoid the const ??? */
456 mad_buf_p = (void *)madw_p->p_mad;
457 } else {
458 /* we have the madw defined during the send and stored in the vend_wrap */
459 /* we need to make sure the wrapper is correctly init there */
460 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0);
461 madw_p = req_madw_p->vend_wrap.p_resp_madw;
462
463 /* HACK: we do not Support RMPP */
464 CL_ASSERT(madw_p->h_bind);
465 mad_buf_p =
466 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE,
467 &madw_p->vend_wrap);
468
469 if (mad_buf_p == NULL) {
470 osm_log(p_log, OSM_LOG_ERROR,
471 "__osm_mtl_rcv_callback: ERR 3C11: "
472 "Unable to acquire wire MAD.\n");
473
474 goto Exit;
475 }
476
477 /*
478 Finally, attach the wire MAD to this wrapper.
479 */
480 osm_madw_set_mad(madw_p, mad_buf_p);
481
482 /* also we need to handle the size of the mad since we did not init ... */
483 madw_p->mad_size = MAD_BLOCK_SIZE;
484 }
485
486 /* init some fields of the vendor wrapper */
487 p_new_vw = osm_madw_get_vend_ptr(madw_p);
488 p_new_vw->h_bind = bind_info_p;
489 p_new_vw->size = MAD_BLOCK_SIZE;
490 p_new_vw->p_resp_madw = NULL;
491 p_new_vw->mad_buf_p = mad_buf_p;
492
493 /* HACK: We do not support RMPP in receiving MADS */
494 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE);
495
496 /* attach the buffer to the wrapper */
497 madw_p->p_mad = mad_buf_p;
498
499 /* we can also make sure we marked the size and bind on the returned madw */
500 madw_p->h_bind = p_new_vw->h_bind;
501
502 /* call the CB */
503 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context,
504 req_madw_p);
505
506 Exit:
507 OSM_LOG_EXIT(p_log);
508 }
509
510 /**********************************************************************
511 * IB_MGT Send callback : invoked after each send
512 *
513 **********************************************************************/
514 void
__osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,IN u_int64_t wrid,IN IB_comp_status_t status,IN void * private_ctx_p)515 __osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl,
516 IN u_int64_t wrid,
517 IN IB_comp_status_t status, IN void *private_ctx_p)
518 {
519 osm_madw_t *madw_p;
520 osm_mtl_bind_info_t *bind_info_p =
521 (osm_mtl_bind_info_t *) private_ctx_p;
522 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
523 osm_vend_wrap_t *p_vw;
524 uint8_t is_resp;
525
526 OSM_LOG_ENTER(p_log);
527
528 /* obtain the madp from the wrid */
529 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p);
530
531 osm_log(p_log, OSM_LOG_DEBUG,
532 "__osm_mtl_send_callback: INFO 1008: "
533 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
534
535 /* we need to handle requests and responses differently */
536 if (is_resp) {
537 if (status != IB_COMP_SUCCESS) {
538 osm_log(p_log, OSM_LOG_ERROR,
539 "__osm_mtl_send_callback: ERR 3C12: "
540 "Error Sending Response MADW:%p.\n", madw_p);
541 } else {
542 osm_log(p_log, OSM_LOG_DEBUG,
543 "__osm_mtl_send_callback: DBG 1008: "
544 "Completed Sending Response MADW:%p.\n",
545 madw_p);
546 }
547
548 /* if we are a response - we need to clean it up */
549 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
550 } else {
551
552 /* this call back is invoked on completion of send - error or not */
553 if (status != IB_COMP_SUCCESS) {
554
555 osm_log(p_log, OSM_LOG_ERROR,
556 "__osm_mtl_send_callback: ERR 3C13: "
557 "Received an Error from IB_MGT Send (%d).\n",
558 status);
559
560 p_vw = osm_madw_get_vend_ptr(madw_p);
561 CL_ASSERT(p_vw);
562
563 /*
564 Return any wrappers to the pool that may have been
565 pre-emptively allocated to handle a receive.
566 */
567 if (p_vw->p_resp_madw) {
568 osm_mad_pool_put(bind_info_p->p_osm_pool,
569 p_vw->p_resp_madw);
570 p_vw->p_resp_madw = NULL;
571 }
572
573 /* invoke the CB */
574 (*bind_info_p->send_err_callback) (bind_info_p->
575 client_context,
576 madw_p);
577 } else {
578 /* successful request send - do nothing - the response will need the
579 out mad */
580 osm_log(p_log, OSM_LOG_DEBUG,
581 "__osm_mtl_send_callback: DBG 1008: "
582 "Completed Sending Request MADW:%p.\n", madw_p);
583 }
584 }
585
586 OSM_LOG_EXIT(p_log);
587 }
588
589 /**********************************************************************
590 * BINDs a callback (rcv and send error) for a given class and method
591 * defined by the given: osm_bind_info_t
592 **********************************************************************/
593 osm_bind_handle_t
osm_vendor_bind(IN osm_vendor_t * const p_vend,IN osm_bind_info_t * const p_user_bind,IN osm_mad_pool_t * const p_mad_pool,IN osm_vend_mad_recv_callback_t mad_recv_callback,IN osm_vend_mad_send_err_callback_t send_err_callback,IN void * context)594 osm_vendor_bind(IN osm_vendor_t * const p_vend,
595 IN osm_bind_info_t * const p_user_bind,
596 IN osm_mad_pool_t * const p_mad_pool,
597 IN osm_vend_mad_recv_callback_t mad_recv_callback,
598 IN osm_vend_mad_send_err_callback_t send_err_callback,
599 IN void *context)
600 {
601 ib_net64_t port_guid;
602 osm_mtl_bind_info_t *p_bind = NULL;
603 VAPI_hca_hndl_t hca_hndl;
604 VAPI_hca_id_t hca_id;
605 IB_MGT_mad_type_t mad_type;
606 uint32_t port_num;
607 osm_vendor_mgt_bind_t *ib_mgt_hdl_p;
608 IB_MGT_ret_t mgt_ret;
609
610 OSM_LOG_ENTER(p_vend->p_log);
611
612 CL_ASSERT(p_user_bind);
613 CL_ASSERT(p_mad_pool);
614 CL_ASSERT(mad_recv_callback);
615 CL_ASSERT(send_err_callback);
616
617 /* cast back the AL handle to vendor mgt bind */
618 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al;
619
620 port_guid = p_user_bind->port_guid;
621
622 osm_log(p_vend->p_log, OSM_LOG_INFO,
623 "osm_vendor_bind: "
624 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
625
626 /* obtain the hca name and port num from the guid */
627 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
628 "osm_vendor_bind: "
629 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
630 port_guid);
631
632 mgt_ret =
633 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
634 &hca_id, &port_num);
635 if (mgt_ret != IB_MGT_OK) {
636 osm_log(p_vend->p_log, OSM_LOG_ERROR,
637 "osm_vendor_bind: ERR 3C14: "
638 "Unable to obtain CA and port (%d).\n");
639 goto Exit;
640 }
641
642 /* create the bind object tracking this binding */
643 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t));
644 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t));
645 if (p_bind == NULL) {
646 osm_log(p_vend->p_log, OSM_LOG_ERROR,
647 "osm_vendor_bind: ERR 3C15: "
648 "Unable to allocate internal bind object.\n");
649 goto Exit;
650 }
651
652 /* track this bind request info */
653 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t));
654 p_bind->port_num = port_num;
655 p_bind->p_vend = p_vend;
656 p_bind->client_context = context;
657 p_bind->rcv_callback = mad_recv_callback;
658 p_bind->send_err_callback = send_err_callback;
659 p_bind->p_osm_pool = p_mad_pool;
660
661 CL_ASSERT(p_bind->port_num);
662
663 /*
664 * Get the proper CLASS
665 */
666
667 switch (p_user_bind->mad_class) {
668 case IB_MCLASS_SUBN_LID:
669 case IB_MCLASS_SUBN_DIR:
670 mad_type = IB_MGT_SMI;
671 break;
672
673 case IB_MCLASS_SUBN_ADM:
674 default:
675 mad_type = IB_MGT_GSI;
676 break;
677 }
678
679 /* we split here - based on the type of MADS GSI / SMI */
680 /* HACK: we only support one class registration per SMI/GSI !!! */
681 if (mad_type == IB_MGT_SMI) {
682 /*
683 * SMI CASE
684 */
685
686 /* we do not need to bind the handle if already available */
687 if (ib_mgt_hdl_p->smi_init == FALSE) {
688
689 /* First we have to reg and get the handle for the mad */
690 osm_log(p_vend->p_log, OSM_LOG_ERROR,
691 "osm_vendor_bind: "
692 "Binding to IB_MGT SMI of %s port %u\n", hca_id,
693 port_num);
694
695 mgt_ret =
696 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI,
697 &(ib_mgt_hdl_p->smi_mads_hdl));
698 if (IB_MGT_OK != mgt_ret) {
699 free(p_bind);
700 p_bind = NULL;
701 osm_log(p_vend->p_log, OSM_LOG_ERROR,
702 "osm_vendor_bind: ERR 3C16: "
703 "Error obtaining IB_MGT handle to SMI.\n");
704 goto Exit;
705 }
706
707 /* bind it */
708 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl);
709 if (IB_MGT_OK != mgt_ret) {
710 free(p_bind);
711 p_bind = NULL;
712 osm_log(p_vend->p_log, OSM_LOG_ERROR,
713 "osm_vendor_bind: ERR 3C17: "
714 "Error binding IB_MGT handle to SM.\n");
715 goto Exit;
716 }
717
718 ib_mgt_hdl_p->smi_init = TRUE;
719
720 }
721
722 /* attach to this bind info */
723 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl;
724 ib_mgt_hdl_p->smi_p_bind = p_bind;
725
726 /* now register the callback */
727 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
728 &__osm_mtl_rcv_callback,
729 p_bind,
730 &__osm_mtl_send_callback,
731 p_bind,
732 IB_MGT_RCV_CB_MASK |
733 IB_MGT_SEND_CB_MASK);
734
735 } else {
736 /*
737 * GSI CASE
738 */
739
740 if (ib_mgt_hdl_p->gsi_init == FALSE) {
741 osm_log(p_vend->p_log, OSM_LOG_ERROR,
742 "osm_vendor_bind: " "Binding to IB_MGT GSI\n");
743
744 /* First we have to reg and get the handle for the mad */
745 mgt_ret =
746 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI,
747 &(ib_mgt_hdl_p->gsi_mads_hdl));
748 if (IB_MGT_OK != mgt_ret) {
749 free(p_bind);
750 p_bind = NULL;
751 osm_log(p_vend->p_log, OSM_LOG_ERROR,
752 "osm_vendor_bind: ERR 3C20: "
753 "Error obtaining IB_MGT handle to GSI.\n");
754 goto Exit;
755 }
756
757 /* bind it */
758 mgt_ret =
759 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl,
760 p_user_bind->mad_class);
761 if (IB_MGT_OK != mgt_ret) {
762 free(p_bind);
763 p_bind = NULL;
764 osm_log(p_vend->p_log, OSM_LOG_ERROR,
765 "osm_vendor_bind: ERR 3C22: "
766 "Error binding IB_MGT handle to GSI.\n");
767 goto Exit;
768 }
769
770 ib_mgt_hdl_p->gsi_init = TRUE;
771
772 /* attach to this bind info */
773 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
774
775 /* now register the callback */
776 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl,
777 &__osm_mtl_rcv_callback,
778 p_bind,
779 &__osm_mtl_send_callback,
780 p_bind,
781 IB_MGT_RCV_CB_MASK |
782 IB_MGT_SEND_CB_MASK);
783
784 } else {
785 /* we can use the existing handle */
786 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl;
787 mgt_ret = IB_MGT_OK;
788 }
789
790 }
791
792 if (IB_MGT_OK != mgt_ret) {
793 free(p_bind);
794 p_bind = NULL;
795 osm_log(p_vend->p_log, OSM_LOG_ERROR,
796 "osm_vendor_bind: ERR 3C23: "
797 "Error binding IB_MGT CB (%d).\n", mgt_ret);
798 goto Exit;
799 }
800
801 /* HACK: Do we need to initialize an address vector ???? */
802
803 Exit:
804 OSM_LOG_EXIT(p_vend->p_log);
805 return ((osm_bind_handle_t) p_bind);
806 }
807
808 /**********************************************************************
809 Get a mad from the lower level.
810 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
811 **********************************************************************/
osm_vendor_get(IN osm_bind_handle_t h_bind,IN const uint32_t mad_size,IN osm_vend_wrap_t * const p_vw)812 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
813 IN const uint32_t mad_size,
814 IN osm_vend_wrap_t * const p_vw)
815 {
816 ib_mad_t *mad_p;
817 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
818 osm_vendor_t *p_vend = p_bind->p_vend;
819
820 OSM_LOG_ENTER(p_vend->p_log);
821
822 CL_ASSERT(p_vw);
823 /* HACK: We know we can not send through IB_MGT */
824 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE);
825
826 /* IB_MGT assumes it is 256 - we must follow */
827 p_vw->size = MAD_BLOCK_SIZE;
828
829 /* allocate it */
830 mad_p = (ib_mad_t *) malloc(p_vw->size);
831 if (mad_p == NULL) {
832 osm_log(p_vend->p_log, OSM_LOG_ERROR,
833 "osm_vendor_get: ERR 3C24: "
834 "Error Obtaining MAD buffer.\n");
835 goto Exit;
836 }
837
838 memset(mad_p, 0, p_vw->size);
839
840 /* track locally */
841 p_vw->mad_buf_p = mad_p;
842 p_vw->h_bind = h_bind;
843 p_vw->p_resp_madw = NULL;
844
845 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
846 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
847 "osm_vendor_get: "
848 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size);
849 }
850
851 Exit:
852 OSM_LOG_EXIT(p_vend->p_log);
853 return (mad_p);
854 }
855
856 /**********************************************************************
857 * Return a MAD by providing it's wrapper object.
858 **********************************************************************/
859 void
osm_vendor_put(IN osm_bind_handle_t h_bind,IN osm_vend_wrap_t * const p_vw)860 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
861 {
862 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
863 osm_vendor_t *p_vend = p_bind->p_vend;
864 osm_madw_t *p_madw;
865
866 OSM_LOG_ENTER(p_vend->p_log);
867
868 CL_ASSERT(p_vw);
869 CL_ASSERT(p_vw->mad_buf_p);
870
871 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
872 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
873 "osm_vendor_put: " "Retiring MAD %p.\n",
874 p_vw->mad_buf_p);
875 }
876
877 /*
878 * We moved the removal of the transaction to immediatly after
879 * it was looked up.
880 */
881
882 /* free the mad but the wrapper is part of the madw object */
883 free(p_vw->mad_buf_p);
884 p_vw->mad_buf_p = NULL;
885 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
886 p_madw->p_mad = NULL;
887
888 OSM_LOG_EXIT(p_vend->p_log);
889 }
890
891 /**********************************************************************
892 Actually Send a MAD
893
894 This is for internal use by osm_vendor_send and the transaction mgr
895 retry too.
896 **********************************************************************/
897 ib_api_status_t
osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind,IN osm_madw_t * const p_madw)898 osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
899 {
900 osm_vendor_t *const p_vend = p_bind->p_vend;
901 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
902 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
903 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
904 ib_api_status_t status;
905 IB_MGT_ret_t mgt_res;
906 IB_ud_av_t av;
907 uint64_t wrid;
908 uint32_t qpn;
909
910 OSM_LOG_ENTER(p_vend->p_log);
911
912 /*
913 * For all sends other than directed route SM MADs,
914 * acquire an address vector for the destination.
915 */
916 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
917 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr,
918 p_mad->mgmt_class ==
919 IB_MCLASS_SUBN_LID, &av);
920 } else {
921 /* is a directed route - we need to construct a permissive address */
922 memset(&av, 0, sizeof(av));
923 /* we do not need port number since it is part of the mad_hndl */
924 av.dlid = IB_LID_PERMISSIVE;
925 }
926
927 wrid = __osm_set_wrid_by_p_madw(p_madw);
928
929 /* send it */
930 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
931 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
932
933 /* SMI CASE */
934 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
935 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
936 "osm_mtl_send_mad: "
937 "av.dlid 0x%X, "
938 "av.static_rate %d, "
939 "av.path_bits %d.\n",
940 cl_ntoh16(av.dlid), av.static_rate,
941 av.src_path_bits);
942 }
943
944 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */
945 &av, /* address vector */
946 wrid, /* casting the mad wrapper pointer for err cb */
947 p_vend->timeout);
948
949 } else {
950 /* GSI CASE - Support Remote QP */
951 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
952 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
953 "osm_mtl_send_mad: "
954 "av.dlid 0x%X, av.static_rate %d, "
955 "av.path_bits %d, remote qp: 0x%06X \n",
956 av.dlid,
957 av.static_rate,
958 av.src_path_bits,
959 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
960 );
961 }
962
963 /* IBMGT have a bug sending to a QP not 1 -
964 the QPN must be in network order except when it qpn 1 ... */
965 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp);
966
967 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */
968 &av, /* address vector */
969 wrid, /* casting the mad wrapper pointer for err cb */
970 p_vend->timeout, qpn);
971 }
972
973 if (mgt_res != IB_MGT_OK) {
974 osm_log(p_vend->p_log, OSM_LOG_ERROR,
975 "osm_mtl_send_mad: ERR 3C26: "
976 "Error sending mad (%d).\n", mgt_res);
977 if (p_vw->p_resp_madw)
978 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw);
979 status = IB_ERROR;
980 goto Exit;
981 }
982
983 status = IB_SUCCESS;
984
985 Exit:
986 OSM_LOG_EXIT(p_vend->p_log);
987 return (status);
988 }
989
990 /**********************************************************************
991 Send a MAD through.
992
993 What is unclear to me is the need for the setting of all the MAD Wrapper
994 fields. Seems like the OSM uses these values during it's processing...
995 **********************************************************************/
996 ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,IN osm_madw_t * const p_madw,IN boolean_t const resp_expected)997 osm_vendor_send(IN osm_bind_handle_t h_bind,
998 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
999 {
1000 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind;
1001 osm_vendor_t *const p_vend = p_bind->p_vend;
1002 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1003 ib_api_status_t status;
1004
1005 OSM_LOG_ENTER(p_vend->p_log);
1006
1007 /*
1008 * If a response is expected to this MAD, then preallocate
1009 * a mad wrapper to contain the wire MAD received in the
1010 * response. Allocating a wrapper here allows for easier
1011 * failure paths than after we already received the wire mad.
1012 */
1013 if (resp_expected == TRUE) {
1014 /* we track it in the vendor wrapper */
1015 p_vw->p_resp_madw =
1016 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
1017 if (p_vw->p_resp_madw == NULL) {
1018 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1019 "osm_vendor_send: ERR 3C27: "
1020 "Unable to allocate MAD wrapper.\n");
1021 status = IB_INSUFFICIENT_RESOURCES;
1022 goto Exit;
1023 }
1024
1025 /* put some minimal info on that wrapper */
1026 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
1027
1028 /* we also want to track it in the TID based map */
1029 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t)
1030 p_bind, p_madw);
1031 if (status != IB_SUCCESS) {
1032 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1033 "osm_vendor_send: ERR 3C25: "
1034 "Error inserting request madw by TID (%d).\n",
1035 status);
1036 }
1037
1038 } else
1039 p_vw->p_resp_madw = NULL;
1040
1041 /* do the actual send */
1042 status = osm_mtl_send_mad(p_bind, p_madw);
1043
1044 Exit:
1045 OSM_LOG_EXIT(p_vend->p_log);
1046 return (status);
1047 }
1048
1049 /**********************************************************************
1050 * the idea here is to change the content of the bind such that it
1051 * will hold the local address used for sending directed route by the SMA.
1052 **********************************************************************/
osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)1053 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1054 {
1055 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend;
1056
1057 OSM_LOG_ENTER(p_vend->p_log);
1058
1059 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
1060 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
1061
1062 OSM_LOG_EXIT(p_vend->p_log);
1063
1064 return (IB_SUCCESS);
1065 }
1066
1067 /**********************************************************************
1068 **********************************************************************/
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)1069 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1070 {
1071 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind;
1072 osm_vendor_t *p_vend = p_bind->p_vend;
1073 VAPI_ret_t status;
1074 VAPI_hca_attr_t attr_mod;
1075 VAPI_hca_attr_mask_t attr_mask;
1076
1077 OSM_LOG_ENTER(p_vend->p_log);
1078
1079 memset(&attr_mod, 0, sizeof(attr_mod));
1080 memset(&attr_mask, 0, sizeof(attr_mask));
1081
1082 attr_mod.is_sm = is_sm_val;
1083 attr_mask = HCA_ATTR_IS_SM;
1084
1085 status =
1086 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
1087 &attr_mask);
1088 if (status != VAPI_OK) {
1089 osm_log(p_vend->p_log, OSM_LOG_ERROR,
1090 "osm_vendor_set_sm: ERR 3C28: "
1091 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
1092 is_sm_val, status);
1093 }
1094
1095 OSM_LOG_EXIT(p_vend->p_log);
1096 }
1097
1098 /**********************************************************************
1099 **********************************************************************/
osm_vendor_set_debug(IN osm_vendor_t * const p_vend,IN int32_t level)1100 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1101 {
1102
1103 }
1104
1105 #endif /* OSM_VENDOR_INTF_TEST */
1106