1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 #undef __init
37 #if HAVE_CONFIG_H
38 # include <config.h>
39 #endif /* HAVE_CONFIG_H */
40
41 #include <stdlib.h>
42 #include <string.h>
43 #include <vendor/osm_vendor_ts.h>
44 #include <vendor/osm_vendor_api.h>
45 #include <vendor/osm_ts_useraccess.h>
46 #include <opensm/osm_subnet.h>
47 #include <opensm/osm_opensm.h>
48
49 /*
50 Since a race can accure on requests. Meaning - a response is received before
51 the send_callback is called - we will save both the madw_p and the fact
52 whether or not it is a response. A race can occure only on requests that did
53 not fail, and then the madw_p will be put back in the pool before the
54 callback.
55 */
__osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)56 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw)
57 {
58 uint64_t wrid = 0;
59
60 CL_ASSERT(p_madw->p_mad);
61
62 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *));
63 wrid = (wrid << 1) |
64 ib_mad_is_response(p_madw->p_mad) |
65 (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS);
66 return wrid;
67 }
68
69 void
__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,OUT uint8_t * is_resp,OUT osm_madw_t ** pp_madw)70 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid,
71 OUT uint8_t * is_resp,
72 OUT osm_madw_t ** pp_madw)
73 {
74 *is_resp = wrid & 0x0000000000000001;
75 wrid = wrid >> 1;
76 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *));
77 }
78
79 /**********************************************************************
80 * TS MAD to OSM ADDRESS VECTOR
81 **********************************************************************/
82 void
__osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,IN struct ib_mad * p_mad,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)83 __osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend,
84 IN struct ib_mad *p_mad,
85 IN uint8_t is_smi,
86 OUT osm_mad_addr_t * p_mad_addr)
87 {
88 p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
89 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
90 p_mad_addr->path_bits = 0; /* HACK - no way to know in TS */
91 if (is_smi) {
92 /* SMI */
93 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
94 p_mad_addr->addr_type.smi.port_num = p_mad->port;
95 } else {
96 /* GSI */
97 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
98 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
99 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
100 p_mad_addr->addr_type.gsi.service_level = 0; /* HACK no way to know */
101
102 p_mad_addr->addr_type.gsi.global_route = FALSE; /* HACK no way to know */
103 /* copy the GRH data if relevant */
104 /*
105 if (p_mad_addr->addr_type.gsi.global_route)
106 {
107 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
108 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
109 p_rcv_desc->grh.traffic_class,
110 p_rcv_desc->grh.flow_label);
111 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
112 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
113 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
114 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
115 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
116 }
117 */
118 }
119 }
120
121 /**********************************************************************
122 * OSM ADDR VECTOR TO TS MAD:
123 **********************************************************************/
124 void
__osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT struct ib_mad * p_mad)125 __osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr,
126 IN uint8_t is_smi, OUT struct ib_mad *p_mad)
127 {
128
129 /* For global destination or Multicast address: */
130 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
131 p_mad->sl = 0;
132 if (is_smi) {
133 p_mad->sqpn = 0;
134 p_mad->dqpn = 0;
135 } else {
136 p_mad->sqpn = 1;
137 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
138 }
139 }
140
141 /**********************************************************************
142 **********************************************************************/
__osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)143 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind)
144 {
145 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
146 osm_vendor_t *p_vend = p_bind->p_vend;
147 VAPI_ret_t status;
148 VAPI_hca_attr_t attr_mod;
149 VAPI_hca_attr_mask_t attr_mask;
150
151 OSM_LOG_ENTER(p_vend->p_log);
152
153 memset(&attr_mod, 0, sizeof(attr_mod));
154 memset(&attr_mask, 0, sizeof(attr_mask));
155
156 attr_mod.is_sm = FALSE;
157 attr_mask = HCA_ATTR_IS_SM;
158
159 status =
160 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
161 &attr_mask);
162 if (status != VAPI_OK) {
163 osm_log(p_vend->p_log, OSM_LOG_ERROR,
164 "__osm_vendor_clear_sm: ERR 5021: "
165 "Unable set 'IS_SM' bit in port attributes (%d).\n",
166 status);
167 }
168
169 OSM_LOG_EXIT(p_vend->p_log);
170 }
171
172 /**********************************************************************
173 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT
174 **********************************************************************/
osm_vendor_construct(IN osm_vendor_t * const p_vend)175 void osm_vendor_construct(IN osm_vendor_t * const p_vend)
176 {
177 memset(p_vend, 0, sizeof(*p_vend));
178 cl_thread_construct(&(p_vend->smi_bind.poller));
179 cl_thread_construct(&(p_vend->gsi_bind.poller));
180 }
181
182 /**********************************************************************
183 * DEALOCATE osm_vendor_t
184 **********************************************************************/
osm_vendor_destroy(IN osm_vendor_t * const p_vend)185 void osm_vendor_destroy(IN osm_vendor_t * const p_vend)
186 {
187 OSM_LOG_ENTER(p_vend->p_log);
188 osm_transaction_mgr_destroy(p_vend);
189
190 /* Destroy the poller threads */
191 /* HACK: can you destroy an un-initialized thread ? */
192 pthread_cancel(p_vend->smi_bind.poller.osd.id);
193 pthread_cancel(p_vend->gsi_bind.poller.osd.id);
194 cl_thread_destroy(&(p_vend->smi_bind.poller));
195 cl_thread_destroy(&(p_vend->gsi_bind.poller));
196 OSM_LOG_EXIT(p_vend->p_log);
197 }
198
199 /**********************************************************************
200 DEALLOCATE A POINTER TO osm_vendor_t
201 **********************************************************************/
osm_vendor_delete(IN osm_vendor_t ** const pp_vend)202 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
203 {
204 CL_ASSERT(pp_vend);
205
206 osm_vendor_destroy(*pp_vend);
207 free(*pp_vend);
208 *pp_vend = NULL;
209 }
210
211 /**********************************************************************
212 Initializes the vendor:
213 **********************************************************************/
214
215 ib_api_status_t
osm_vendor_init(IN osm_vendor_t * const p_vend,IN osm_log_t * const p_log,IN const uint32_t timeout)216 osm_vendor_init(IN osm_vendor_t * const p_vend,
217 IN osm_log_t * const p_log, IN const uint32_t timeout)
218 {
219 ib_api_status_t status = IB_SUCCESS;
220
221 OSM_LOG_ENTER(p_log);
222
223 p_vend->p_log = p_log;
224 p_vend->p_transaction_mgr = NULL;
225 osm_transaction_mgr_init(p_vend);
226 p_vend->timeout = timeout;
227
228 /* we use the file handle to track the binding */
229 p_vend->smi_bind.ul_dev_fd = -1;
230 p_vend->gsi_bind.ul_dev_fd = -1;
231
232 OSM_LOG_EXIT(p_log);
233 return (status);
234 }
235
236 /**********************************************************************
237 * Create and Initialize osm_vendor_t Object
238 **********************************************************************/
osm_vendor_new(IN osm_log_t * const p_log,IN const uint32_t timeout)239 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
240 IN const uint32_t timeout)
241 {
242 ib_api_status_t status;
243 osm_vendor_t *p_vend;
244
245 OSM_LOG_ENTER(p_log);
246
247 CL_ASSERT(p_log);
248
249 p_vend = malloc(sizeof(*p_vend));
250 if (p_vend != NULL) {
251 memset(p_vend, 0, sizeof(*p_vend));
252
253 status = osm_vendor_init(p_vend, p_log, timeout);
254 if (status != IB_SUCCESS) {
255 osm_vendor_delete(&p_vend);
256 }
257 } else {
258 osm_log(p_vend->p_log, OSM_LOG_ERROR,
259 "osm_vendor_new: ERR 5007: "
260 "Fail to allocate vendor object.\n");
261 }
262
263 OSM_LOG_EXIT(p_log);
264 return (p_vend);
265 }
266
267 /**********************************************************************
268 * TS RCV Thread callback
269 * HACK: - we need to make this support arbitrary size mads.
270 **********************************************************************/
271 void
__osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind,IN osm_mad_addr_t * p_mad_addr,IN uint32_t mad_size,IN void * p_mad)272 __osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind,
273 IN osm_mad_addr_t * p_mad_addr,
274 IN uint32_t mad_size, IN void *p_mad)
275 {
276 ib_api_status_t status;
277 osm_madw_t *p_req_madw = NULL;
278 osm_madw_t *p_madw;
279 osm_vend_wrap_t *p_new_vw;
280 ib_mad_t *p_mad_buf;
281 osm_log_t *const p_log = p_bind->p_vend->p_log;
282
283 OSM_LOG_ENTER(p_log);
284
285 /* if it is a response MAD we mustbe able to get the request */
286 if (ib_mad_is_response((ib_mad_t *) p_mad)) {
287 /* can we find a matching madw by this payload TID */
288 status =
289 osm_transaction_mgr_get_madw_for_tid(p_bind->p_vend,
290 (ib_mad_t *) p_mad,
291 &p_req_madw);
292 if (status != IB_SUCCESS) {
293 osm_log(p_log, OSM_LOG_ERROR,
294 "__osm_ts_rcv_callback: ERR 5008: "
295 "Error obtaining request madw by TID (%d).\n",
296 status);
297 p_req_madw = NULL;
298 }
299
300 if (p_req_madw == NULL) {
301 osm_log(p_log, OSM_LOG_ERROR,
302 "__osm_ts_rcv_callback: ERR 5009: "
303 "Fail to obtain request madw for receined MAD. Aborting CB.\n");
304 goto Exit;
305 }
306 }
307
308 /* do we have a request ??? */
309 if (p_req_madw == NULL) {
310
311 /* if not - get new osm_madw and arrange it. */
312 /* create the new madw in the pool */
313 p_madw = osm_mad_pool_get(p_bind->p_osm_pool,
314 (osm_bind_handle_t) p_bind,
315 mad_size, p_mad_addr);
316 if (p_madw == NULL) {
317 osm_log(p_log, OSM_LOG_ERROR,
318 "__osm_ts_rcv_callback: ERR 5010: "
319 "Error request for a new madw.\n");
320 goto Exit;
321 }
322 /* HACK: we cust to avoid the const ??? */
323 p_mad_buf = (void *)p_madw->p_mad;
324 } else {
325 /* we have the madw defined during the send and stored in the vend_wrap */
326 /* we need to make sure the wrapper is correctly init there */
327 CL_ASSERT(p_req_madw->vend_wrap.p_resp_madw != 0);
328 p_madw = p_req_madw->vend_wrap.p_resp_madw;
329
330 CL_ASSERT(p_madw->h_bind);
331 p_mad_buf =
332 osm_vendor_get(p_madw->h_bind, mad_size,
333 &p_madw->vend_wrap);
334
335 if (p_mad_buf == NULL) {
336 osm_log(p_log, OSM_LOG_ERROR,
337 "__osm_ts_rcv_callback: ERR 5011: "
338 "Unable to acquire wire MAD.\n");
339
340 goto Exit;
341 }
342
343 /*
344 Finally, attach the wire MAD to this wrapper.
345 */
346 osm_madw_set_mad(p_madw, p_mad_buf);
347 }
348
349 /* init some fields of the vendor wrapper */
350 p_new_vw = osm_madw_get_vend_ptr(p_madw);
351 p_new_vw->h_bind = p_bind;
352 p_new_vw->size = mad_size;
353 p_new_vw->p_resp_madw = NULL;
354 p_new_vw->p_mad_buf = p_mad_buf;
355
356 memcpy(p_new_vw->p_mad_buf, p_mad, mad_size);
357
358 /* attach the buffer to the wrapper */
359 p_madw->p_mad = p_mad_buf;
360
361 /* we can also make sure we marked the size and bind on the returned madw */
362 p_madw->h_bind = p_new_vw->h_bind;
363
364 /* call the CB */
365 (*(osm_vend_mad_recv_callback_t) p_bind->rcv_callback)
366 (p_madw, p_bind->client_context, p_req_madw);
367
368 Exit:
369 OSM_LOG_EXIT(p_log);
370 }
371
372 /**********************************************************************
373 * TS Send callback : invoked after each send
374 *
375 **********************************************************************/
376 void
__osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p,IN boolean_t is_resp,IN osm_madw_t * madw_p,IN IB_comp_status_t status)377 __osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p,
378 IN boolean_t is_resp,
379 IN osm_madw_t * madw_p, IN IB_comp_status_t status)
380 {
381 osm_log_t *const p_log = bind_info_p->p_vend->p_log;
382 osm_vend_wrap_t *p_vw;
383
384 OSM_LOG_ENTER(p_log);
385
386 osm_log(p_log, OSM_LOG_DEBUG,
387 "__osm_ts_send_callback: INFO 1008: "
388 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp);
389
390 /* we need to handle requests and responses differently */
391 if (is_resp) {
392 if (status != IB_COMP_SUCCESS) {
393 osm_log(p_log, OSM_LOG_ERROR,
394 "__osm_ts_send_callback: ERR 5012: "
395 "Error Sending Response MADW:%p.\n", madw_p);
396 } else {
397 osm_log(p_log, OSM_LOG_DEBUG,
398 "__osm_ts_send_callback: DBG 1008: "
399 "Completed Sending Response MADW:%p.\n",
400 madw_p);
401 }
402
403 /* if we are a response - we need to clean it up */
404 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p);
405 } else {
406
407 /* this call back is invoked on completion of send - error or not */
408 if (status != IB_COMP_SUCCESS) {
409
410 osm_log(p_log, OSM_LOG_ERROR,
411 "__osm_ts_send_callback: ERR 5013: "
412 "Received an Error from IB_MGT Send (%d).\n",
413 status);
414
415 p_vw = osm_madw_get_vend_ptr(madw_p);
416 CL_ASSERT(p_vw);
417
418 /*
419 Return any wrappers to the pool that may have been
420 pre-emptively allocated to handle a receive.
421 */
422 if (p_vw->p_resp_madw) {
423 osm_mad_pool_put(bind_info_p->p_osm_pool,
424 p_vw->p_resp_madw);
425 p_vw->p_resp_madw = NULL;
426 }
427
428 /* invoke the CB */
429 (*(osm_vend_mad_send_err_callback_t) bind_info_p->
430 send_err_callback)
431 (bind_info_p->client_context, madw_p);
432 } else {
433 /* successful request send - do nothing - the response will need the
434 out mad */
435 osm_log(p_log, OSM_LOG_DEBUG,
436 "__osm_ts_send_callback: DBG 1008: "
437 "Completed Sending Request MADW:%p.\n", madw_p);
438 }
439 }
440
441 OSM_LOG_EXIT(p_log);
442 }
443
444 /**********************************************************************
445 * Poller thread:
446 * Always receive 256byte mads from the devcie file
447 **********************************************************************/
__osm_vendor_ts_poller(IN void * p_ptr)448 void __osm_vendor_ts_poller(IN void *p_ptr)
449 {
450 int ts_ret_code;
451 struct ib_mad mad;
452 osm_mad_addr_t mad_addr;
453 osm_ts_bind_info_t *const p_bind = (osm_ts_bind_info_t *) p_ptr;
454
455 OSM_LOG_ENTER(p_bind->p_vend->p_log);
456 /* we set the type of cancelation for this thread */
457 pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
458
459 while (1) {
460 /* we read one mad at a time and pass it to the read callback function */
461 ts_ret_code = read(p_bind->ul_dev_fd, &mad, sizeof(mad));
462 if (ts_ret_code != sizeof(mad)) {
463 osm_log(p_bind->p_vend->p_log, OSM_LOG_ERROR,
464 "__osm_vendor_ts_poller: ERR 5003: "
465 "error with read, bytes = %d, errno = %d\n",
466 ts_ret_code, errno);
467 } else {
468 osm_log(p_bind->p_vend->p_log, OSM_LOG_DEBUG,
469 "__osm_vendor_ts_poller: "
470 "MAD QPN:%d SLID:0x%04x class:0x%02x "
471 "__osm_vendor_ts_poller:0x%02x attr:0x%04x status:0x%04x "
472 "__osm_vendor_ts_poller:0x%016" PRIx64 "\n",
473 cl_ntoh32(mad.dqpn),
474 cl_ntoh16(mad.slid),
475 mad.mgmt_class,
476 mad.r_method,
477 cl_ntoh16(mad.attribute_id),
478 cl_ntoh16(mad.status),
479 cl_ntoh64(mad.transaction_id));
480
481 /* first arrange an address */
482 __osm_ts_conv_mad_rcv_desc_to_osm_addr(p_bind->p_vend,
483 &mad,
484 (((ib_mad_t *) &
485 mad)->
486 mgmt_class ==
487 IB_MCLASS_SUBN_LID)
488 ||
489 (((ib_mad_t *) &
490 mad)->
491 mgmt_class ==
492 IB_MCLASS_SUBN_DIR),
493 &mad_addr);
494
495 /* call the receiver callback */
496 /* HACK: this should be replaced with a call to the RMPP Assembly ... */
497 __osm_ts_rcv_callback(p_bind, &mad_addr, 256, &mad);
498 }
499 }
500
501 OSM_LOG_EXIT(p_bind->p_vend->p_log);
502 }
503
504 /**********************************************************************
505 * BINDs a callback (rcv and send error) for a given class and method
506 * defined by the given: osm_bind_info_t
507 **********************************************************************/
508 osm_bind_handle_t
osm_vendor_bind(IN osm_vendor_t * const p_vend,IN osm_bind_info_t * const p_user_bind,IN osm_mad_pool_t * const p_mad_pool,IN osm_vend_mad_recv_callback_t mad_recv_callback,IN osm_vend_mad_send_err_callback_t send_err_callback,IN void * context)509 osm_vendor_bind(IN osm_vendor_t * const p_vend,
510 IN osm_bind_info_t * const p_user_bind,
511 IN osm_mad_pool_t * const p_mad_pool,
512 IN osm_vend_mad_recv_callback_t mad_recv_callback,
513 IN osm_vend_mad_send_err_callback_t send_err_callback,
514 IN void *context)
515 {
516 ib_net64_t port_guid;
517 osm_ts_bind_info_t *p_bind = NULL;
518 VAPI_hca_hndl_t hca_hndl;
519 VAPI_hca_id_t hca_id;
520 uint32_t port_num;
521 ib_api_status_t status;
522 int device_fd;
523 char device_file[16];
524 osm_ts_user_mad_filter filter;
525 int ts_ioctl_ret;
526 int qpn;
527
528 OSM_LOG_ENTER(p_vend->p_log);
529
530 CL_ASSERT(p_mad_pool);
531
532 port_guid = p_user_bind->port_guid;
533
534 osm_log(p_vend->p_log, OSM_LOG_INFO,
535 "osm_vendor_bind: "
536 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid));
537
538 switch (p_user_bind->mad_class) {
539 case IB_MCLASS_SUBN_LID:
540 case IB_MCLASS_SUBN_DIR:
541 p_bind = &(p_vend->smi_bind);
542 qpn = 0;
543 break;
544
545 case IB_MCLASS_SUBN_ADM:
546 default:
547 p_bind = &(p_vend->gsi_bind);
548 qpn = 1;
549 break;
550 }
551
552 /* Make sure we did not previously opened the file */
553 if (p_bind->ul_dev_fd >= 0) {
554 osm_log(p_vend->p_log, OSM_LOG_ERROR,
555 "osm_vendor_bind: ERR 5004: "
556 "Already binded to port %u\n", p_bind->port_num);
557 goto Exit;
558 }
559
560 /*
561 We need to figure out what is the TS file name to attach to.
562 I guess it is following the index of the port in the table of
563 ports.
564 */
565
566 /* obtain the hca name and port num from the guid */
567 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
568 "osm_vendor_bind: "
569 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n",
570 cl_ntoh64(port_guid));
571 status =
572 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl,
573 &hca_id, &port_num);
574 if (status != IB_SUCCESS) {
575 osm_log(p_vend->p_log, OSM_LOG_ERROR,
576 "osm_vendor_bind: ERR 5005: "
577 "Fail to find port number of port guid:0x%016" PRIx64
578 "\n", port_guid);
579 goto Exit;
580 }
581
582 /* the file name is just /dev/ts_ua0: */
583 strcpy(device_file, "/dev/ts_ua0");
584
585 osm_log(p_vend->p_log, OSM_LOG_ERROR,
586 "osm_vendor_bind: " "Opening TS UL dev file:%s\n", device_file);
587
588 /* Open the file ... */
589 device_fd = open(device_file, O_RDWR);
590 if (device_fd < 0) {
591 osm_log(p_vend->p_log, OSM_LOG_ERROR,
592 "osm_vendor_bind: ERR 5006: "
593 "Fail to open TS UL dev file:%s\n", device_file);
594 goto Exit;
595 }
596
597 /* track this bind request info */
598 p_bind->ul_dev_fd = device_fd;
599 p_bind->port_num = port_num;
600 p_bind->p_vend = p_vend;
601 p_bind->client_context = context;
602 p_bind->rcv_callback = mad_recv_callback;
603 p_bind->send_err_callback = send_err_callback;
604 p_bind->p_osm_pool = p_mad_pool;
605 p_bind->hca_hndl = hca_hndl;
606
607 /*
608 * Create the MAD filter on this file handle.
609 */
610 filter.port = port_num;
611
612 filter.qpn = qpn;
613 filter.mgmt_class = p_user_bind->mad_class;
614 filter.direction = TS_IB_MAD_DIRECTION_IN;
615 filter.mask =
616 TS_IB_MAD_FILTER_DIRECTION |
617 TS_IB_MAD_FILTER_PORT |
618 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
619
620 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
621 if (ts_ioctl_ret < 0) {
622 osm_log(p_vend->p_log, OSM_LOG_ERROR,
623 "osm_vendor_bind: ERR 5014: "
624 "Fail to register MAD filter with err:%u\n",
625 ts_ioctl_ret);
626 goto Exit;
627 }
628
629 /* Initialize the listener thread for this port */
630 status = cl_thread_init(&p_bind->poller,
631 __osm_vendor_ts_poller, p_bind,
632 "osm ts poller");
633 if (status != IB_SUCCESS)
634 goto Exit;
635
636 Exit:
637 OSM_LOG_EXIT(p_vend->p_log);
638 return ((osm_bind_handle_t) p_bind);
639 }
640
641 /**********************************************************************
642 Get a mad from the lower level.
643 The osm_vend_wrap_t is a wrapper used to connect the mad to the response.
644 **********************************************************************/
osm_vendor_get(IN osm_bind_handle_t h_bind,IN const uint32_t mad_size,IN osm_vend_wrap_t * const p_vw)645 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
646 IN const uint32_t mad_size,
647 IN osm_vend_wrap_t * const p_vw)
648 {
649 ib_mad_t *p_mad;
650 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
651 osm_vendor_t *p_vend = p_bind->p_vend;
652
653 OSM_LOG_ENTER(p_vend->p_log);
654
655 CL_ASSERT(p_vw);
656
657 p_vw->size = mad_size;
658
659 /* allocate it */
660 p_mad = (ib_mad_t *) malloc(p_vw->size);
661 if (p_mad == NULL) {
662 osm_log(p_vend->p_log, OSM_LOG_ERROR,
663 "osm_vendor_get: ERR 5022: "
664 "Error Obtaining MAD buffer.\n");
665 goto Exit;
666 }
667
668 memset(p_mad, 0, p_vw->size);
669
670 /* track locally */
671 p_vw->p_mad_buf = p_mad;
672 p_vw->h_bind = h_bind;
673 p_vw->p_resp_madw = NULL;
674
675 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
676 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
677 "osm_vendor_get: "
678 "Acquired MAD %p, size = %u.\n", p_mad, p_vw->size);
679 }
680
681 Exit:
682 OSM_LOG_EXIT(p_vend->p_log);
683 return (p_mad);
684 }
685
686 /**********************************************************************
687 * Return a MAD by providing it's wrapper object.
688 **********************************************************************/
689 void
osm_vendor_put(IN osm_bind_handle_t h_bind,IN osm_vend_wrap_t * const p_vw)690 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
691 {
692 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
693 osm_vendor_t *p_vend = p_bind->p_vend;
694 osm_madw_t *p_madw;
695
696 OSM_LOG_ENTER(p_vend->p_log);
697
698 CL_ASSERT(p_vw);
699 CL_ASSERT(p_vw->p_mad_buf);
700
701 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) {
702 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
703 "osm_vendor_put: " "Retiring MAD %p.\n",
704 p_vw->p_mad_buf);
705 }
706
707 /*
708 * We moved the removal of the transaction to immediatly after
709 * it was looked up.
710 */
711
712 /* free the mad but the wrapper is part of the madw object */
713 free(p_vw->p_mad_buf);
714 p_vw->p_mad_buf = NULL;
715 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
716 p_madw->p_mad = NULL;
717
718 OSM_LOG_EXIT(p_vend->p_log);
719 }
720
721 /**********************************************************************
722 Actually Send a MAD
723
724 MADs are buffers of type: struct ib_mad - so they are limited by size.
725 This is for internal use by osm_vendor_send and the transaction mgr
726 retry too.
727 **********************************************************************/
728 ib_api_status_t
osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind,IN osm_madw_t * const p_madw)729 osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind, IN osm_madw_t * const p_madw)
730 {
731 osm_vendor_t *const p_vend = p_bind->p_vend;
732 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
733 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
734 struct ib_mad ts_mad;
735 int ret;
736 ib_api_status_t status;
737
738 OSM_LOG_ENTER(p_vend->p_log);
739
740 /*
741 * Copy the MAD over to the sent mad
742 */
743 memcpy(&ts_mad, p_mad, 256);
744
745 /*
746 * For all sends other than directed route SM MADs,
747 * acquire an address vector for the destination.
748 */
749 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
750 __osm_ts_conv_osm_addr_to_ts_addr(p_mad_addr,
751 p_mad->mgmt_class ==
752 IB_MCLASS_SUBN_LID, &ts_mad);
753 } else {
754 /* is a directed route - we need to construct a permissive address */
755 /* we do not need port number since it is part of the mad_hndl */
756 ts_mad.dlid = IB_LID_PERMISSIVE;
757 ts_mad.slid = IB_LID_PERMISSIVE;
758 }
759 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
760 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
761 ts_mad.sqpn = 0;
762 ts_mad.dqpn = 0;
763 } else {
764 ts_mad.sqpn = 1;
765 ts_mad.dqpn = 1;
766 }
767 ts_mad.port = p_bind->port_num;
768
769 /* send it */
770 ret = write(p_bind->ul_dev_fd, &ts_mad, sizeof(ts_mad));
771
772 if (ret != sizeof(ts_mad)) {
773 osm_log(p_vend->p_log, OSM_LOG_ERROR,
774 "osm_ts_send_mad: ERR 5026: "
775 "Error sending mad (%d).\n", ret);
776 status = IB_ERROR;
777 goto Exit;
778 }
779
780 status = IB_SUCCESS;
781
782 Exit:
783 OSM_LOG_EXIT(p_vend->p_log);
784 return (status);
785 }
786
787 /**********************************************************************
788 Send a MAD through.
789
790 What is unclear to me is the need for the setting of all the MAD Wrapper
791 fields. Seems like the OSM uses these values during it's processing...
792 **********************************************************************/
793 ib_api_status_t
osm_vendor_send(IN osm_bind_handle_t h_bind,IN osm_madw_t * const p_madw,IN boolean_t const resp_expected)794 osm_vendor_send(IN osm_bind_handle_t h_bind,
795 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
796 {
797 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
798 osm_vendor_t *const p_vend = p_bind->p_vend;
799 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
800 ib_api_status_t status;
801
802 OSM_LOG_ENTER(p_vend->p_log);
803
804 /*
805 * If a response is expected to this MAD, then preallocate
806 * a mad wrapper to contain the wire MAD received in the
807 * response. Allocating a wrapper here allows for easier
808 * failure paths than after we already received the wire mad.
809 */
810 if (resp_expected == TRUE) {
811 /* we track it in the vendor wrapper */
812 p_vw->p_resp_madw =
813 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool);
814 if (p_vw->p_resp_madw == NULL) {
815 osm_log(p_vend->p_log, OSM_LOG_ERROR,
816 "osm_vendor_send: ERR 5024: "
817 "Unable to allocate MAD wrapper.\n");
818 status = IB_INSUFFICIENT_RESOURCES;
819 goto Exit;
820 }
821
822 /* put some minimal info on that wrapper */
823 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind;
824
825 /* we also want to track it in the TID based map */
826 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *)
827 p_bind, p_madw);
828 if (status != IB_SUCCESS) {
829 osm_log(p_vend->p_log, OSM_LOG_ERROR,
830 "osm_vendor_send: ERR 5025: "
831 "Error inserting request madw by TID (%d).\n",
832 status);
833 }
834 } else
835 p_vw->p_resp_madw = NULL;
836
837 /* do the actual send */
838 /* HACK: to be replaced by call to RMPP Segmentation */
839 status = osm_ts_send_mad(p_bind, p_madw);
840
841 /* we do not get an asycn callback so call it ourselves */
842 /* this will handle all cleanup if neccessary */
843 __osm_ts_send_callback(p_bind, !resp_expected, p_madw, status);
844
845 Exit:
846 OSM_LOG_EXIT(p_vend->p_log);
847 return (status);
848 }
849
850 /**********************************************************************
851 * the idea here is to change the content of the bind such that it
852 * will hold the local address used for sending directed route by the SMA.
853 **********************************************************************/
osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)854 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
855 {
856 osm_vendor_t *p_vend = ((osm_ts_bind_info_t *) h_bind)->p_vend;
857
858 OSM_LOG_ENTER(p_vend->p_log);
859
860 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
861 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n");
862
863 OSM_LOG_EXIT(p_vend->p_log);
864
865 return (IB_SUCCESS);
866 }
867
868 /**********************************************************************
869 **********************************************************************/
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)870 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
871 {
872 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind;
873 osm_vendor_t *p_vend = p_bind->p_vend;
874 VAPI_ret_t status;
875 VAPI_hca_attr_t attr_mod;
876 VAPI_hca_attr_mask_t attr_mask;
877
878 OSM_LOG_ENTER(p_vend->p_log);
879
880 memset(&attr_mod, 0, sizeof(attr_mod));
881 memset(&attr_mask, 0, sizeof(attr_mask));
882
883 attr_mod.is_sm = is_sm_val;
884 attr_mask = HCA_ATTR_IS_SM;
885
886 status =
887 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod,
888 &attr_mask);
889 if (status != VAPI_OK) {
890 osm_log(p_vend->p_log, OSM_LOG_ERROR,
891 "osm_vendor_set_sm: ERR 5027: "
892 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n",
893 is_sm_val, status);
894 }
895
896 OSM_LOG_EXIT(p_vend->p_log);
897 }
898
899 /**********************************************************************
900 **********************************************************************/
osm_vendor_set_debug(IN osm_vendor_t * const p_vend,IN int32_t level)901 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
902 {
903
904 }
905