1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 /*
37 * Abstract:
38 * Implementation of osm_state_mgr_t.
39 * This file implements the State Manager object.
40 */
41
42 #if HAVE_CONFIG_H
43 # include <config.h>
44 #endif /* HAVE_CONFIG_H */
45
46 #include <unistd.h>
47 #include <stdlib.h>
48 #include <string.h>
49 #include <iba/ib_types.h>
50 #include <complib/cl_passivelock.h>
51 #include <complib/cl_debug.h>
52 #include <complib/cl_qmap.h>
53 #include <opensm/osm_sm.h>
54 #include <opensm/osm_madw.h>
55 #include <opensm/osm_switch.h>
56 #include <opensm/osm_log.h>
57 #include <opensm/osm_subnet.h>
58 #include <opensm/osm_helper.h>
59 #include <opensm/osm_msgdef.h>
60 #include <opensm/osm_node.h>
61 #include <opensm/osm_port.h>
62 #include <vendor/osm_vendor_api.h>
63 #include <opensm/osm_inform.h>
64 #include <opensm/osm_opensm.h>
65
66 extern void osm_drop_mgr_process(IN osm_sm_t * sm);
67 extern osm_signal_t osm_qos_setup(IN osm_opensm_t * p_osm);
68 extern osm_signal_t osm_pkey_mgr_process(IN osm_opensm_t * p_osm);
69 extern osm_signal_t osm_mcast_mgr_process(IN osm_sm_t * sm);
70 extern osm_signal_t osm_mcast_mgr_process_mgroups(IN osm_sm_t * sm);
71 extern osm_signal_t osm_link_mgr_process(IN osm_sm_t * sm, IN uint8_t state);
72
73 /**********************************************************************
74 **********************************************************************/
__osm_state_mgr_up_msg(IN const osm_sm_t * sm)75 static void __osm_state_mgr_up_msg(IN const osm_sm_t * sm)
76 {
77 /*
78 * This message should be written only once - when the
79 * SM moves to Master state and the subnet is up for
80 * the first time.
81 */
82 osm_log(sm->p_log, sm->p_subn->first_time_master_sweep ?
83 OSM_LOG_SYS : OSM_LOG_INFO, "SUBNET UP\n");
84
85 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
86 sm->p_subn->opt.sweep_interval ?
87 "SUBNET UP" : "SUBNET UP (sweep disabled)");
88 }
89
90 /**********************************************************************
91 **********************************************************************/
__osm_state_mgr_reset_node_count(IN cl_map_item_t * const p_map_item,IN void * context)92 static void __osm_state_mgr_reset_node_count(IN cl_map_item_t *
93 const p_map_item, IN void *context)
94 {
95 osm_node_t *p_node = (osm_node_t *) p_map_item;
96
97 p_node->discovery_count = 0;
98 }
99
100 /**********************************************************************
101 **********************************************************************/
__osm_state_mgr_reset_port_count(IN cl_map_item_t * const p_map_item,IN void * context)102 static void __osm_state_mgr_reset_port_count(IN cl_map_item_t *
103 const p_map_item, IN void *context)
104 {
105 osm_port_t *p_port = (osm_port_t *) p_map_item;
106
107 p_port->discovery_count = 0;
108 }
109
110 /**********************************************************************
111 **********************************************************************/
112 static void
__osm_state_mgr_reset_switch_count(IN cl_map_item_t * const p_map_item,IN void * context)113 __osm_state_mgr_reset_switch_count(IN cl_map_item_t * const p_map_item,
114 IN void *context)
115 {
116 osm_switch_t *p_sw = (osm_switch_t *) p_map_item;
117
118 p_sw->discovery_count = 0;
119 p_sw->need_update = 1;
120 }
121
122 /**********************************************************************
123 **********************************************************************/
__osm_state_mgr_get_sw_info(IN cl_map_item_t * const p_object,IN void * context)124 static void __osm_state_mgr_get_sw_info(IN cl_map_item_t * const p_object,
125 IN void *context)
126 {
127 osm_node_t *p_node;
128 osm_dr_path_t *p_dr_path;
129 osm_madw_context_t mad_context;
130 osm_switch_t *const p_sw = (osm_switch_t *) p_object;
131 osm_sm_t *sm = context;
132 ib_api_status_t status;
133
134 OSM_LOG_ENTER(sm->p_log);
135
136 p_node = p_sw->p_node;
137 p_dr_path = osm_physp_get_dr_path_ptr(osm_node_get_physp_ptr(p_node, 0));
138
139 memset(&mad_context, 0, sizeof(mad_context));
140
141 mad_context.si_context.node_guid = osm_node_get_node_guid(p_node);
142 mad_context.si_context.set_method = FALSE;
143 mad_context.si_context.light_sweep = TRUE;
144
145 status = osm_req_get(sm, p_dr_path, IB_MAD_ATTR_SWITCH_INFO, 0,
146 OSM_MSG_LIGHT_SWEEP_FAIL, &mad_context);
147
148 if (status != IB_SUCCESS)
149 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3304: "
150 "Request for SwitchInfo failed\n");
151
152 OSM_LOG_EXIT(sm->p_log);
153 }
154
155 /**********************************************************************
156 Initiate a remote port info request for the given physical port
157 **********************************************************************/
158 static void
__osm_state_mgr_get_remote_port_info(IN osm_sm_t * sm,IN osm_physp_t * const p_physp)159 __osm_state_mgr_get_remote_port_info(IN osm_sm_t * sm,
160 IN osm_physp_t * const p_physp)
161 {
162 osm_dr_path_t *p_dr_path;
163 osm_dr_path_t rem_node_dr_path;
164 osm_madw_context_t mad_context;
165 ib_api_status_t status;
166
167 OSM_LOG_ENTER(sm->p_log);
168
169 /* generate a dr path leaving on the physp to the remote node */
170 p_dr_path = osm_physp_get_dr_path_ptr(p_physp);
171 memcpy(&rem_node_dr_path, p_dr_path, sizeof(osm_dr_path_t));
172 osm_dr_path_extend(&rem_node_dr_path, osm_physp_get_port_num(p_physp));
173
174 memset(&mad_context, 0, sizeof(mad_context));
175
176 mad_context.pi_context.node_guid =
177 osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp));
178 mad_context.pi_context.port_guid = p_physp->port_guid;
179 mad_context.pi_context.set_method = FALSE;
180 mad_context.pi_context.light_sweep = TRUE;
181 mad_context.pi_context.active_transition = FALSE;
182
183 /* note that with some negative logic - if the query failed it means that
184 * there is no point in going to heavy sweep */
185 status = osm_req_get(sm, &rem_node_dr_path,
186 IB_MAD_ATTR_PORT_INFO, 0, CL_DISP_MSGID_NONE,
187 &mad_context);
188
189 if (status != IB_SUCCESS)
190 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 332E: "
191 "Request for PortInfo failed\n");
192
193 OSM_LOG_EXIT(sm->p_log);
194 }
195
196 /**********************************************************************
197 Initiates a thorough sweep of the subnet.
198 Used when there is suspicion that something on the subnet has changed.
199 **********************************************************************/
__osm_state_mgr_sweep_hop_0(IN osm_sm_t * sm)200 static ib_api_status_t __osm_state_mgr_sweep_hop_0(IN osm_sm_t * sm)
201 {
202 ib_api_status_t status;
203 osm_dr_path_t dr_path;
204 osm_bind_handle_t h_bind;
205 uint8_t path_array[IB_SUBNET_PATH_HOPS_MAX];
206
207 OSM_LOG_ENTER(sm->p_log);
208
209 memset(path_array, 0, sizeof(path_array));
210
211 /*
212 * First, get the bind handle.
213 */
214 h_bind = osm_sm_mad_ctrl_get_bind_handle(&sm->mad_ctrl);
215 if (h_bind != OSM_BIND_INVALID_HANDLE) {
216 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
217 "INITIATING HEAVY SWEEP");
218 /*
219 * Start the sweep by clearing the port counts, then
220 * get our own NodeInfo at 0 hops.
221 */
222 CL_PLOCK_ACQUIRE(sm->p_lock);
223
224 cl_qmap_apply_func(&sm->p_subn->node_guid_tbl,
225 __osm_state_mgr_reset_node_count, sm);
226
227 cl_qmap_apply_func(&sm->p_subn->port_guid_tbl,
228 __osm_state_mgr_reset_port_count, sm);
229
230 cl_qmap_apply_func(&sm->p_subn->sw_guid_tbl,
231 __osm_state_mgr_reset_switch_count, sm);
232
233 /* Set the in_sweep_hop_0 flag in subn to be TRUE.
234 * This will indicate the sweeping not to continue beyond the
235 * the current node.
236 * This is relevant for the case of SM on switch, since in the
237 * switch info we need to signal somehow not to continue
238 * the sweeping. */
239 sm->p_subn->in_sweep_hop_0 = TRUE;
240
241 CL_PLOCK_RELEASE(sm->p_lock);
242
243 osm_dr_path_init(&dr_path, h_bind, 0, path_array);
244 status = osm_req_get(sm, &dr_path, IB_MAD_ATTR_NODE_INFO, 0,
245 CL_DISP_MSGID_NONE, NULL);
246
247 if (status != IB_SUCCESS)
248 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3305: "
249 "Request for NodeInfo failed\n");
250 } else {
251 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
252 "No bound ports. Deferring sweep...\n");
253 status = IB_INVALID_STATE;
254 }
255
256 OSM_LOG_EXIT(sm->p_log);
257 return (status);
258 }
259
260 /**********************************************************************
261 Clear out all existing port lid assignments
262 **********************************************************************/
__osm_state_mgr_clean_known_lids(IN osm_sm_t * sm)263 static ib_api_status_t __osm_state_mgr_clean_known_lids(IN osm_sm_t * sm)
264 {
265 ib_api_status_t status = IB_SUCCESS;
266 cl_ptr_vector_t *p_vec = &(sm->p_subn->port_lid_tbl);
267 uint32_t i;
268
269 OSM_LOG_ENTER(sm->p_log);
270
271 /* we need a lock here! */
272 CL_PLOCK_ACQUIRE(sm->p_lock);
273
274 for (i = 0; i < cl_ptr_vector_get_size(p_vec); i++)
275 cl_ptr_vector_set(p_vec, i, NULL);
276
277 CL_PLOCK_RELEASE(sm->p_lock);
278
279 OSM_LOG_EXIT(sm->p_log);
280 return (status);
281 }
282
283 /**********************************************************************
284 Notifies the transport layer that the local LID has changed,
285 which give it a chance to update address vectors, etc..
286 **********************************************************************/
__osm_state_mgr_notify_lid_change(IN osm_sm_t * sm)287 static ib_api_status_t __osm_state_mgr_notify_lid_change(IN osm_sm_t * sm)
288 {
289 ib_api_status_t status;
290 osm_bind_handle_t h_bind;
291
292 OSM_LOG_ENTER(sm->p_log);
293
294 /*
295 * First, get the bind handle.
296 */
297 h_bind = osm_sm_mad_ctrl_get_bind_handle(&sm->mad_ctrl);
298 if (h_bind == OSM_BIND_INVALID_HANDLE) {
299 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3306: "
300 "No bound ports\n");
301 status = IB_ERROR;
302 goto Exit;
303 }
304
305 /*
306 * Notify the transport layer that we changed the local LID.
307 */
308 status = osm_vendor_local_lid_change(h_bind);
309 if (status != IB_SUCCESS)
310 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3307: "
311 "Vendor LID update failed (%s)\n",
312 ib_get_err_str(status));
313
314 Exit:
315 OSM_LOG_EXIT(sm->p_log);
316 return (status);
317 }
318
319 /**********************************************************************
320 Returns true if the SM port is down.
321 The SM's port object must exist in the port_guid table.
322 **********************************************************************/
__osm_state_mgr_is_sm_port_down(IN osm_sm_t * sm)323 static boolean_t __osm_state_mgr_is_sm_port_down(IN osm_sm_t * sm)
324 {
325 ib_net64_t port_guid;
326 osm_port_t *p_port;
327 osm_physp_t *p_physp;
328 uint8_t state;
329
330 OSM_LOG_ENTER(sm->p_log);
331
332 port_guid = sm->p_subn->sm_port_guid;
333
334 /*
335 * If we don't know our own port guid yet, assume the port is down.
336 */
337 if (port_guid == 0) {
338 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3308: "
339 "SM port GUID unknown\n");
340 state = IB_LINK_DOWN;
341 goto Exit;
342 }
343
344 CL_ASSERT(port_guid);
345
346 CL_PLOCK_ACQUIRE(sm->p_lock);
347 p_port = osm_get_port_by_guid(sm->p_subn, port_guid);
348 if (!p_port) {
349 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3309: "
350 "SM port with GUID:%016" PRIx64 " is unknown\n",
351 cl_ntoh64(port_guid));
352 state = IB_LINK_DOWN;
353 CL_PLOCK_RELEASE(sm->p_lock);
354 goto Exit;
355 }
356
357 p_physp = p_port->p_physp;
358
359 CL_ASSERT(p_physp);
360
361 state = osm_physp_get_port_state(p_physp);
362 CL_PLOCK_RELEASE(sm->p_lock);
363
364 Exit:
365 OSM_LOG_EXIT(sm->p_log);
366 return (state == IB_LINK_DOWN);
367 }
368
369 /**********************************************************************
370 Sweeps the node 1 hop away.
371 This sets off a "chain reaction" that causes discovery of the subnet.
372 Used when there is suspicion that something on the subnet has changed.
373 **********************************************************************/
__osm_state_mgr_sweep_hop_1(IN osm_sm_t * sm)374 static ib_api_status_t __osm_state_mgr_sweep_hop_1(IN osm_sm_t * sm)
375 {
376 ib_api_status_t status = IB_SUCCESS;
377 osm_bind_handle_t h_bind;
378 osm_madw_context_t context;
379 osm_node_t *p_node;
380 osm_port_t *p_port;
381 osm_physp_t *p_physp;
382 osm_dr_path_t *p_dr_path;
383 osm_dr_path_t hop_1_path;
384 ib_net64_t port_guid;
385 uint8_t port_num;
386 uint8_t path_array[IB_SUBNET_PATH_HOPS_MAX];
387 uint8_t num_ports;
388 osm_physp_t *p_ext_physp;
389
390 OSM_LOG_ENTER(sm->p_log);
391
392 /*
393 * First, get our own port and node objects.
394 */
395 port_guid = sm->p_subn->sm_port_guid;
396
397 CL_ASSERT(port_guid);
398
399 /* Set the in_sweep_hop_0 flag in subn to be FALSE.
400 * This will indicate the sweeping to continue beyond the
401 * the current node.
402 * This is relevant for the case of SM on switch, since in the
403 * switch info we need to signal that the sweeping should
404 * continue through the switch. */
405 sm->p_subn->in_sweep_hop_0 = FALSE;
406
407 p_port = osm_get_port_by_guid(sm->p_subn, port_guid);
408 if (!p_port) {
409 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3310: "
410 "No SM port object\n");
411 status = IB_ERROR;
412 goto Exit;
413 }
414
415 p_node = p_port->p_node;
416 CL_ASSERT(p_node);
417
418 port_num = ib_node_info_get_local_port_num(&p_node->node_info);
419
420 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
421 "Probing hop 1 on local port %u\n", port_num);
422
423 p_physp = osm_node_get_physp_ptr(p_node, port_num);
424
425 CL_ASSERT(p_physp);
426
427 p_dr_path = osm_physp_get_dr_path_ptr(p_physp);
428 h_bind = osm_dr_path_get_bind_handle(p_dr_path);
429
430 CL_ASSERT(h_bind != OSM_BIND_INVALID_HANDLE);
431
432 memset(path_array, 0, sizeof(path_array));
433 /* the hop_1 operations depend on the type of our node.
434 * Currently - legal nodes that can host SM are SW and CA */
435 switch (osm_node_get_type(p_node)) {
436 case IB_NODE_TYPE_CA:
437 case IB_NODE_TYPE_ROUTER:
438 memset(&context, 0, sizeof(context));
439 context.ni_context.node_guid = osm_node_get_node_guid(p_node);
440 context.ni_context.port_num = port_num;
441
442 path_array[1] = port_num;
443
444 osm_dr_path_init(&hop_1_path, h_bind, 1, path_array);
445 status = osm_req_get(sm, &hop_1_path, IB_MAD_ATTR_NODE_INFO, 0,
446 CL_DISP_MSGID_NONE, &context);
447 if (status != IB_SUCCESS)
448 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3311: "
449 "Request for NodeInfo failed\n");
450 break;
451
452 case IB_NODE_TYPE_SWITCH:
453 /* Need to go over all the ports of the switch, and send a node_info
454 * from them. This doesn't include the port 0 of the switch, which
455 * hosts the SM.
456 * Note: We'll send another switchInfo on port 0, since if no ports
457 * are connected, we still want to get some response, and have the
458 * subnet come up.
459 */
460 num_ports = osm_node_get_num_physp(p_node);
461 for (port_num = 0; port_num < num_ports; port_num++) {
462 /* go through the port only if the port is not DOWN */
463 p_ext_physp = osm_node_get_physp_ptr(p_node, port_num);
464 if (p_ext_physp && ib_port_info_get_port_state
465 (&(p_ext_physp->port_info)) > IB_LINK_DOWN) {
466 memset(&context, 0, sizeof(context));
467 context.ni_context.node_guid =
468 osm_node_get_node_guid(p_node);
469 context.ni_context.port_num = port_num;
470
471 path_array[1] = port_num;
472 osm_dr_path_init(&hop_1_path, h_bind, 1,
473 path_array);
474 status = osm_req_get(sm, &hop_1_path,
475 IB_MAD_ATTR_NODE_INFO, 0,
476 CL_DISP_MSGID_NONE,
477 &context);
478
479 if (status != IB_SUCCESS)
480 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3312: "
481 "Request for NodeInfo failed\n");
482 }
483 }
484 break;
485
486 default:
487 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
488 "ERR 3313: Unknown node type %d (%s)\n",
489 osm_node_get_type(p_node), p_node->print_desc);
490 }
491
492 Exit:
493 OSM_LOG_EXIT(sm->p_log);
494 return (status);
495 }
496
query_sm_info(cl_map_item_t * item,void * cxt)497 static void query_sm_info(cl_map_item_t *item, void *cxt)
498 {
499 osm_madw_context_t context;
500 osm_remote_sm_t *r_sm = cl_item_obj(item, r_sm, map_item);
501 osm_sm_t *sm = cxt;
502 ib_api_status_t ret;
503
504 context.smi_context.port_guid = r_sm->p_port->guid;
505 context.smi_context.set_method = FALSE;
506 context.smi_context.light_sweep = TRUE;
507
508 ret = osm_req_get(sm, osm_physp_get_dr_path_ptr(r_sm->p_port->p_physp),
509 IB_MAD_ATTR_SM_INFO, 0, CL_DISP_MSGID_NONE, &context);
510 if (ret != IB_SUCCESS)
511 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3314: "
512 "Failure requesting SMInfo (%s)\n",
513 ib_get_err_str(ret));
514 }
515
516 /**********************************************************************
517 During a light sweep check each node to see if the node descriptor is valid
518 if not issue a ND query.
519 **********************************************************************/
__osm_state_mgr_get_node_desc(IN cl_map_item_t * const p_object,IN void * context)520 static void __osm_state_mgr_get_node_desc(IN cl_map_item_t * const p_object,
521 IN void *context)
522 {
523 osm_madw_context_t mad_context;
524 osm_node_t *const p_node = (osm_node_t *) p_object;
525 osm_sm_t *sm = context;
526 osm_physp_t *p_physp = NULL;
527 unsigned i, num_ports;
528 ib_api_status_t status;
529
530 OSM_LOG_ENTER(sm->p_log);
531
532 CL_ASSERT(p_node);
533
534 if (p_node->print_desc && strcmp(p_node->print_desc, OSM_NODE_DESC_UNKNOWN))
535 /* if ND is valid, do nothing */
536 goto exit;
537
538 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
539 "ERR 3319: Unknown node description for node GUID "
540 "0x%016" PRIx64 ". Reissuing ND query\n",
541 cl_ntoh64(osm_node_get_node_guid (p_node)));
542
543 /* get a physp to request from. */
544 num_ports = osm_node_get_num_physp(p_node);
545 for (i = 0; i < num_ports; i++)
546 if ((p_physp = osm_node_get_physp_ptr(p_node, i)))
547 break;
548
549 if (!p_physp) {
550 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 331C: "
551 "Failed to find any valid physical port object.\n");
552 goto exit;
553 }
554
555 mad_context.nd_context.node_guid = osm_node_get_node_guid(p_node);
556
557 status = osm_req_get(sm, osm_physp_get_dr_path_ptr(p_physp),
558 IB_MAD_ATTR_NODE_DESC, 0, CL_DISP_MSGID_NONE,
559 &mad_context);
560 if (status != IB_SUCCESS)
561 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
562 "ERR 331B: Failure initiating NodeDescription request "
563 "(%s)\n", ib_get_err_str(status));
564
565 exit:
566 OSM_LOG_EXIT(sm->p_log);
567 }
568
569 /**********************************************************************
570 Initiates a lightweight sweep of the subnet.
571 Used during normal sweeps after the subnet is up.
572 **********************************************************************/
__osm_state_mgr_light_sweep_start(IN osm_sm_t * sm)573 static ib_api_status_t __osm_state_mgr_light_sweep_start(IN osm_sm_t * sm)
574 {
575 ib_api_status_t status = IB_SUCCESS;
576 osm_bind_handle_t h_bind;
577 cl_qmap_t *p_sw_tbl;
578 cl_map_item_t *p_next;
579 osm_node_t *p_node;
580 osm_physp_t *p_physp;
581 uint8_t port_num;
582
583 OSM_LOG_ENTER(sm->p_log);
584
585 p_sw_tbl = &sm->p_subn->sw_guid_tbl;
586
587 /*
588 * First, get the bind handle.
589 */
590 h_bind = osm_sm_mad_ctrl_get_bind_handle(&sm->mad_ctrl);
591 if (h_bind == OSM_BIND_INVALID_HANDLE) {
592 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
593 "No bound ports. Deferring sweep...\n");
594 status = IB_INVALID_STATE;
595 goto _exit;
596 }
597
598 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, "INITIATING LIGHT SWEEP");
599 CL_PLOCK_ACQUIRE(sm->p_lock);
600 cl_qmap_apply_func(p_sw_tbl, __osm_state_mgr_get_sw_info, sm);
601 CL_PLOCK_RELEASE(sm->p_lock);
602
603 CL_PLOCK_ACQUIRE(sm->p_lock);
604 cl_qmap_apply_func(&sm->p_subn->node_guid_tbl, __osm_state_mgr_get_node_desc, sm);
605 CL_PLOCK_RELEASE(sm->p_lock);
606
607 /* now scan the list of physical ports that were not down but have no remote port */
608 CL_PLOCK_ACQUIRE(sm->p_lock);
609 p_next = cl_qmap_head(&sm->p_subn->node_guid_tbl);
610 while (p_next != cl_qmap_end(&sm->p_subn->node_guid_tbl)) {
611 p_node = (osm_node_t *) p_next;
612 p_next = cl_qmap_next(p_next);
613
614 for (port_num = 1; port_num < osm_node_get_num_physp(p_node);
615 port_num++) {
616 p_physp = osm_node_get_physp_ptr(p_node, port_num);
617 if (p_physp && (osm_physp_get_port_state(p_physp) !=
618 IB_LINK_DOWN)
619 && !osm_physp_get_remote(p_physp)) {
620 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3315: "
621 "Unknown remote side for node 0x%016"
622 PRIx64
623 "(%s) port %u. Adding to light sweep sampling list\n",
624 cl_ntoh64(osm_node_get_node_guid
625 (p_node)),
626 p_node->print_desc, port_num);
627
628 osm_dump_dr_path(sm->p_log,
629 osm_physp_get_dr_path_ptr
630 (p_physp), OSM_LOG_ERROR);
631
632 __osm_state_mgr_get_remote_port_info(sm,
633 p_physp);
634 }
635 }
636 }
637
638 cl_qmap_apply_func(&sm->p_subn->sm_guid_tbl, query_sm_info, sm);
639
640 CL_PLOCK_RELEASE(sm->p_lock);
641
642 _exit:
643 OSM_LOG_EXIT(sm->p_log);
644 return (status);
645 }
646
647 /**********************************************************************
648 * Go over all the remote SMs (as updated in the sm_guid_tbl).
649 * Find if there is a remote sm that is a master SM.
650 * If there is a remote master SM - return a pointer to it,
651 * else - return NULL.
652 **********************************************************************/
__osm_state_mgr_exists_other_master_sm(IN osm_sm_t * sm)653 static osm_remote_sm_t *__osm_state_mgr_exists_other_master_sm(IN osm_sm_t * sm)
654 {
655 cl_qmap_t *p_sm_tbl;
656 osm_remote_sm_t *p_sm;
657 osm_remote_sm_t *p_sm_res = NULL;
658
659 OSM_LOG_ENTER(sm->p_log);
660
661 p_sm_tbl = &sm->p_subn->sm_guid_tbl;
662
663 /* go over all the remote SMs */
664 for (p_sm = (osm_remote_sm_t *) cl_qmap_head(p_sm_tbl);
665 p_sm != (osm_remote_sm_t *) cl_qmap_end(p_sm_tbl);
666 p_sm = (osm_remote_sm_t *) cl_qmap_next(&p_sm->map_item)) {
667 /* If the sm is in MASTER state - return a pointer to it */
668 if (ib_sminfo_get_state(&p_sm->smi) == IB_SMINFO_STATE_MASTER) {
669 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
670 "Found remote master SM with guid:0x%016" PRIx64
671 " (node %s)\n", cl_ntoh64(p_sm->smi.guid),
672 p_sm->p_port->p_node ? p_sm->p_port->p_node->
673 print_desc : "UNKNOWN");
674 p_sm_res = p_sm;
675 goto Exit;
676 }
677 }
678
679 Exit:
680 OSM_LOG_EXIT(sm->p_log);
681 return (p_sm_res);
682 }
683
684 /**********************************************************************
685 * Go over all remote SMs (as updated in the sm_guid_tbl).
686 * Find the one with the highest priority and lowest guid.
687 * Compare this SM to the local SM. If the local SM is higher -
688 * return NULL, if the remote SM is higher - return a pointer to it.
689 **********************************************************************/
__osm_state_mgr_get_highest_sm(IN osm_sm_t * sm)690 static osm_remote_sm_t *__osm_state_mgr_get_highest_sm(IN osm_sm_t * sm)
691 {
692 cl_qmap_t *p_sm_tbl;
693 osm_remote_sm_t *p_sm = NULL;
694 osm_remote_sm_t *p_highest_sm;
695 uint8_t highest_sm_priority;
696 ib_net64_t highest_sm_guid;
697
698 OSM_LOG_ENTER(sm->p_log);
699
700 p_sm_tbl = &sm->p_subn->sm_guid_tbl;
701
702 /* Start with the local sm as the standard */
703 p_highest_sm = NULL;
704 highest_sm_priority = sm->p_subn->opt.sm_priority;
705 highest_sm_guid = sm->p_subn->sm_port_guid;
706
707 /* go over all the remote SMs */
708 for (p_sm = (osm_remote_sm_t *) cl_qmap_head(p_sm_tbl);
709 p_sm != (osm_remote_sm_t *) cl_qmap_end(p_sm_tbl);
710 p_sm = (osm_remote_sm_t *) cl_qmap_next(&p_sm->map_item)) {
711
712 /* If the sm is in NOTACTIVE state - continue */
713 if (ib_sminfo_get_state(&p_sm->smi) ==
714 IB_SMINFO_STATE_NOTACTIVE)
715 continue;
716
717 if (osm_sm_is_greater_than(ib_sminfo_get_priority(&p_sm->smi),
718 p_sm->smi.guid, highest_sm_priority,
719 highest_sm_guid)) {
720 /* the new p_sm is with higher priority - update the highest_sm */
721 /* to this sm */
722 p_highest_sm = p_sm;
723 highest_sm_priority =
724 ib_sminfo_get_priority(&p_sm->smi);
725 highest_sm_guid = p_sm->smi.guid;
726 }
727 }
728
729 if (p_highest_sm != NULL)
730 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
731 "Found higher SM with guid: %016" PRIx64 " (node %s)\n",
732 cl_ntoh64(p_highest_sm->smi.guid),
733 p_highest_sm->p_port->p_node ?
734 p_highest_sm->p_port->p_node->print_desc : "UNKNOWN");
735
736 OSM_LOG_EXIT(sm->p_log);
737 return (p_highest_sm);
738 }
739
740 /**********************************************************************
741 * Send SubnSet(SMInfo) SMP with HANDOVER attribute to the
742 * remote_sm indicated.
743 **********************************************************************/
744 static void
__osm_state_mgr_send_handover(IN osm_sm_t * const sm,IN osm_remote_sm_t * const p_sm)745 __osm_state_mgr_send_handover(IN osm_sm_t * const sm,
746 IN osm_remote_sm_t * const p_sm)
747 {
748 uint8_t payload[IB_SMP_DATA_SIZE];
749 ib_sm_info_t *p_smi = (ib_sm_info_t *) payload;
750 osm_madw_context_t context;
751 const osm_port_t *p_port;
752 ib_api_status_t status;
753
754 OSM_LOG_ENTER(sm->p_log);
755
756 /*
757 * Send a query of SubnSet(SMInfo) HANDOVER to the remote sm given.
758 */
759
760 memset(&context, 0, sizeof(context));
761 p_port = p_sm->p_port;
762 if (p_port == NULL) {
763 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3316: "
764 "No port object on given remote_sm object\n");
765 goto Exit;
766 }
767
768 /* update the master_guid in the sm_state_mgr object according to */
769 /* the guid of the port where the new Master SM should reside. */
770 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
771 "Handing over mastership. Updating sm_state_mgr master_guid: %016"
772 PRIx64 " (node %s)\n", cl_ntoh64(p_port->guid),
773 p_port->p_node ? p_port->p_node->print_desc : "UNKNOWN");
774 sm->master_sm_guid = p_port->guid;
775
776 context.smi_context.port_guid = p_port->guid;
777 context.smi_context.set_method = TRUE;
778
779 p_smi->guid = sm->p_subn->sm_port_guid;
780 p_smi->act_count = cl_hton32(sm->p_subn->p_osm->stats.qp0_mads_sent);
781 p_smi->pri_state = (uint8_t) (sm->p_subn->sm_state |
782 sm->p_subn->opt.sm_priority << 4);
783 /*
784 * Return 0 for the SM key unless we authenticate the requester
785 * as the master SM.
786 */
787 if (ib_sminfo_get_state(&p_sm->smi) == IB_SMINFO_STATE_MASTER) {
788 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
789 "Responding to master SM with real sm_key\n");
790 p_smi->sm_key = sm->p_subn->opt.sm_key;
791 } else {
792 /* The requester is not authenticated as master - set sm_key to zero */
793 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
794 "Responding to SM not master with zero sm_key\n");
795 p_smi->sm_key = 0;
796 }
797
798 status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_port->p_physp),
799 payload, sizeof(payload), IB_MAD_ATTR_SM_INFO,
800 IB_SMINFO_ATTR_MOD_HANDOVER, CL_DISP_MSGID_NONE,
801 &context);
802
803 if (status != IB_SUCCESS)
804 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3317: "
805 "Failure requesting SMInfo (%s)\n",
806 ib_get_err_str(status));
807
808 Exit:
809 OSM_LOG_EXIT(sm->p_log);
810 }
811
812 /**********************************************************************
813 * Send Trap 64 on all new ports.
814 **********************************************************************/
__osm_state_mgr_report_new_ports(IN osm_sm_t * sm)815 static void __osm_state_mgr_report_new_ports(IN osm_sm_t * sm)
816 {
817 ib_gid_t port_gid;
818 ib_mad_notice_attr_t notice;
819 ib_api_status_t status;
820 ib_net64_t port_guid;
821 cl_map_item_t *p_next;
822 osm_port_t *p_port;
823 uint16_t min_lid_ho;
824 uint16_t max_lid_ho;
825
826 OSM_LOG_ENTER(sm->p_log);
827
828 CL_PLOCK_ACQUIRE(sm->p_lock);
829 p_next = cl_qmap_head(&sm->p_subn->port_guid_tbl);
830 while (p_next != cl_qmap_end(&sm->p_subn->port_guid_tbl)) {
831 p_port = (osm_port_t *) p_next;
832 p_next = cl_qmap_next(p_next);
833
834 if (!p_port->is_new)
835 continue;
836
837 port_guid = osm_port_get_guid(p_port);
838 /* issue a notice - trap 64 */
839
840 /* details of the notice */
841 notice.generic_type = 0x83; /* is generic subn mgt type */
842 ib_notice_set_prod_type_ho(¬ice, 4); /* A Class Manager generator */
843 /* endport becomes to be reachable */
844 notice.g_or_v.generic.trap_num = CL_HTON16(64);
845 /* The sm_base_lid is saved in network order already. */
846 notice.issuer_lid = sm->p_subn->sm_base_lid;
847 /* following C14-72.1.1 and table 119 p739 */
848 /* we need to provide the GID */
849 port_gid.unicast.prefix = sm->p_subn->opt.subnet_prefix;
850 port_gid.unicast.interface_id = port_guid;
851 memcpy(&(notice.data_details.ntc_64_67.gid), &(port_gid),
852 sizeof(ib_gid_t));
853
854 /* According to page 653 - the issuer gid in this case of trap
855 * is the SM gid, since the SM is the initiator of this trap. */
856 notice.issuer_gid.unicast.prefix =
857 sm->p_subn->opt.subnet_prefix;
858 notice.issuer_gid.unicast.interface_id =
859 sm->p_subn->sm_port_guid;
860
861 status = osm_report_notice(sm->p_log, sm->p_subn, ¬ice);
862 if (status != IB_SUCCESS)
863 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3318: "
864 "Error sending trap reports on GUID:0x%016"
865 PRIx64 " (%s)\n", port_gid.unicast.interface_id,
866 ib_get_err_str(status));
867 osm_port_get_lid_range_ho(p_port, &min_lid_ho, &max_lid_ho);
868 OSM_LOG(sm->p_log, OSM_LOG_INFO,
869 "Discovered new port with GUID:0x%016" PRIx64
870 " LID range [%u,%u] of node:%s\n",
871 cl_ntoh64(port_gid.unicast.interface_id),
872 min_lid_ho, max_lid_ho,
873 p_port->p_node ? p_port->p_node->
874 print_desc : "UNKNOWN");
875
876 p_port->is_new = 0;
877 }
878 CL_PLOCK_RELEASE(sm->p_lock);
879
880 OSM_LOG_EXIT(sm->p_log);
881 }
882
883 /**********************************************************************
884 * Make sure that the lid_port_tbl of the subnet has only the ports
885 * that are recognized, and in the correct lid place. There could be
886 * errors if we wanted to assign a certain port with lid X, but that
887 * request didn't reach the port. In this case port_lid_tbl will have
888 * the port under lid X, though the port isn't updated with this lid.
889 * We will run a new heavy sweep (since there were errors in the
890 * initialization), but here we'll clean the database from incorrect
891 * information.
892 **********************************************************************/
__osm_state_mgr_check_tbl_consistency(IN osm_sm_t * sm)893 static void __osm_state_mgr_check_tbl_consistency(IN osm_sm_t * sm)
894 {
895 cl_qmap_t *p_port_guid_tbl;
896 osm_port_t *p_port;
897 osm_port_t *p_next_port;
898 cl_ptr_vector_t *p_port_lid_tbl;
899 size_t max_lid, ref_size, curr_size, lid;
900 osm_port_t *p_port_ref, *p_port_stored;
901 cl_ptr_vector_t ref_port_lid_tbl;
902 uint16_t min_lid_ho;
903 uint16_t max_lid_ho;
904 uint16_t lid_ho;
905
906 OSM_LOG_ENTER(sm->p_log);
907
908 cl_ptr_vector_construct(&ref_port_lid_tbl);
909 cl_ptr_vector_init(&ref_port_lid_tbl,
910 cl_ptr_vector_get_size(&sm->p_subn->port_lid_tbl),
911 OSM_SUBNET_VECTOR_GROW_SIZE);
912
913 p_port_guid_tbl = &sm->p_subn->port_guid_tbl;
914
915 /* Let's go over all the ports according to port_guid_tbl,
916 * and add the port to a reference port_lid_tbl. */
917 p_next_port = (osm_port_t *) cl_qmap_head(p_port_guid_tbl);
918 while (p_next_port != (osm_port_t *) cl_qmap_end(p_port_guid_tbl)) {
919 p_port = p_next_port;
920 p_next_port =
921 (osm_port_t *) cl_qmap_next(&p_next_port->map_item);
922
923 osm_port_get_lid_range_ho(p_port, &min_lid_ho, &max_lid_ho);
924 for (lid_ho = min_lid_ho; lid_ho <= max_lid_ho; lid_ho++)
925 cl_ptr_vector_set(&ref_port_lid_tbl, lid_ho, p_port);
926 }
927
928 p_port_lid_tbl = &sm->p_subn->port_lid_tbl;
929
930 ref_size = cl_ptr_vector_get_size(&ref_port_lid_tbl);
931 curr_size = cl_ptr_vector_get_size(p_port_lid_tbl);
932 /* They should be the same, but compare it anyway */
933 max_lid = (ref_size > curr_size) ? ref_size : curr_size;
934
935 for (lid = 1; lid <= max_lid; lid++) {
936 p_port_ref = NULL;
937 p_port_stored = NULL;
938 cl_ptr_vector_at(p_port_lid_tbl, lid, (void *)&p_port_stored);
939 cl_ptr_vector_at(&ref_port_lid_tbl, lid, (void *)&p_port_ref);
940
941 if (p_port_stored == p_port_ref)
942 /* This is the "good" case - both entries are the
943 * same for this lid. Nothing to do. */
944 continue;
945
946 if (p_port_ref == NULL)
947 /* There is an object in the subnet database for this
948 * lid, but no such object exists in the reference
949 * port_list_tbl. This can occur if we wanted to assign
950 * a certain port with some lid (different than the one
951 * pre-assigned to it), and the port didn't get the
952 * PortInfo Set request. Due to this, the port is
953 * updated with its original lid in our database, but
954 * with the new lid we wanted to give it in our
955 * port_lid_tbl. */
956 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3322: "
957 "lid %zu is wrongly assigned to port 0x%016"
958 PRIx64 " (\'%s\' port %u) in port_lid_tbl\n",
959 lid,
960 cl_ntoh64(osm_port_get_guid(p_port_stored)),
961 p_port_stored->p_node->print_desc,
962 p_port_stored->p_physp->port_num);
963 else if (p_port_stored == NULL)
964 /* There is an object in the new database, but no
965 * object in our subnet database. This is the matching
966 * case of the prior check - the port still has its
967 * original lid. */
968 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3323: "
969 "port 0x%016" PRIx64 " (\'%s\' port %u)"
970 " exists in new port_lid_tbl under lid %zu,"
971 " but missing in subnet port_lid_tbl db\n",
972 cl_ntoh64(osm_port_get_guid(p_port_ref)),
973 p_port_ref->p_node->print_desc,
974 p_port_ref->p_physp->port_num, lid);
975 else
976 /* if we reached here then p_port_stored != p_port_ref.
977 * We were trying to set a lid to p_port_stored, but
978 * it didn't reach it, and p_port_ref also didn't get
979 * the lid update. */
980 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3324: "
981 "lid %zu has port 0x%016" PRIx64
982 " (\'%s\' port %u) in new port_lid_tbl db, "
983 "and port 0x%016" PRIx64 " (\'%s\' port %u)"
984 " in subnet port_lid_tbl db\n", lid,
985 cl_ntoh64(osm_port_get_guid(p_port_ref)),
986 p_port_ref->p_node->print_desc,
987 p_port_ref->p_physp->port_num,
988 cl_ntoh64(osm_port_get_guid(p_port_stored)),
989 p_port_ref->p_node->print_desc,
990 p_port_ref->p_physp->port_num);
991
992 /* In any of these cases we want to set NULL in the
993 * port_lid_tbl, since this entry is invalid. Also, make sure
994 * we'll do another heavy sweep. */
995 cl_ptr_vector_set(p_port_lid_tbl, lid, NULL);
996 sm->p_subn->subnet_initialization_error = TRUE;
997 }
998
999 cl_ptr_vector_destroy(&ref_port_lid_tbl);
1000 OSM_LOG_EXIT(sm->p_log);
1001 }
1002
cleanup_switch(cl_map_item_t * item,void * log)1003 static void cleanup_switch(cl_map_item_t *item, void *log)
1004 {
1005 osm_switch_t *sw = (osm_switch_t *)item;
1006
1007 if (!sw->new_lft)
1008 return;
1009
1010 if (memcmp(sw->lft, sw->new_lft, IB_LID_UCAST_END_HO + 1))
1011 osm_log(log, OSM_LOG_ERROR, "ERR 331D: "
1012 "LFT of switch 0x%016" PRIx64 " is not up to date.\n",
1013 cl_ntoh64(sw->p_node->node_info.node_guid));
1014 else {
1015 free(sw->new_lft);
1016 sw->new_lft = NULL;
1017 }
1018 }
1019
1020 /**********************************************************************
1021 **********************************************************************/
wait_for_pending_transactions(osm_stats_t * stats)1022 int wait_for_pending_transactions(osm_stats_t * stats)
1023 {
1024 #ifdef HAVE_LIBPTHREAD
1025 pthread_mutex_lock(&stats->mutex);
1026 while (stats->qp0_mads_outstanding && !osm_exit_flag)
1027 pthread_cond_wait(&stats->cond, &stats->mutex);
1028 pthread_mutex_unlock(&stats->mutex);
1029 #else
1030 while (1) {
1031 unsigned count = stats->qp0_mads_outstanding;
1032 if (!count || osm_exit_flag)
1033 break;
1034 cl_event_wait_on(&stats->event, EVENT_NO_TIMEOUT, TRUE);
1035 }
1036 #endif
1037 return osm_exit_flag;
1038 }
1039
do_sweep(osm_sm_t * sm)1040 static void do_sweep(osm_sm_t * sm)
1041 {
1042 ib_api_status_t status;
1043 osm_remote_sm_t *p_remote_sm;
1044
1045 if (sm->p_subn->sm_state != IB_SMINFO_STATE_MASTER &&
1046 sm->p_subn->sm_state != IB_SMINFO_STATE_DISCOVERING)
1047 return;
1048
1049 if (sm->p_subn->coming_out_of_standby)
1050 /*
1051 * Need to force re-write of sm_base_lid to all ports
1052 * to do that we want all the ports to be considered
1053 * foreign
1054 */
1055 __osm_state_mgr_clean_known_lids(sm);
1056
1057 sm->master_sm_found = 0;
1058
1059 /*
1060 * If we already have switches, then try a light sweep.
1061 * Otherwise, this is probably our first discovery pass
1062 * or we are connected in loopback. In both cases do a
1063 * heavy sweep.
1064 * Note: If we are connected in loopback we want a heavy
1065 * sweep, since we will not be getting any traps if there is
1066 * a lost connection.
1067 */
1068 /* if we are in DISCOVERING state - this means it is either in
1069 * initializing or wake up from STANDBY - run the heavy sweep */
1070 if (cl_qmap_count(&sm->p_subn->sw_guid_tbl)
1071 && sm->p_subn->sm_state != IB_SMINFO_STATE_DISCOVERING
1072 && sm->p_subn->opt.force_heavy_sweep == FALSE
1073 && sm->p_subn->force_heavy_sweep == FALSE
1074 && sm->p_subn->force_reroute == FALSE
1075 && sm->p_subn->subnet_initialization_error == FALSE
1076 && (__osm_state_mgr_light_sweep_start(sm) == IB_SUCCESS)) {
1077 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1078 return;
1079 if (!sm->p_subn->force_heavy_sweep) {
1080 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1081 "LIGHT SWEEP COMPLETE");
1082 return;
1083 }
1084 }
1085
1086 /*
1087 * Unicast cache should be invalidated if there were errors
1088 * during initialization or if subnet re-route is requested.
1089 */
1090 if (sm->p_subn->opt.use_ucast_cache &&
1091 (sm->p_subn->subnet_initialization_error ||
1092 sm->p_subn->force_reroute))
1093 osm_ucast_cache_invalidate(&sm->ucast_mgr);
1094
1095 /*
1096 * If we don't need to do a heavy sweep and we want to do a reroute,
1097 * just reroute only.
1098 */
1099 if (cl_qmap_count(&sm->p_subn->sw_guid_tbl)
1100 && sm->p_subn->sm_state != IB_SMINFO_STATE_DISCOVERING
1101 && sm->p_subn->opt.force_heavy_sweep == FALSE
1102 && sm->p_subn->force_heavy_sweep == FALSE
1103 && sm->p_subn->force_reroute == TRUE
1104 && sm->p_subn->subnet_initialization_error == FALSE) {
1105 /* Reset flag */
1106 sm->p_subn->force_reroute = FALSE;
1107
1108 /* Re-program the switches fully */
1109 sm->p_subn->ignore_existing_lfts = TRUE;
1110
1111 osm_ucast_mgr_process(&sm->ucast_mgr);
1112
1113 /* Reset flag */
1114 sm->p_subn->ignore_existing_lfts = FALSE;
1115
1116 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1117 return;
1118
1119 if (!sm->p_subn->subnet_initialization_error) {
1120 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1121 "REROUTE COMPLETE");
1122 return;
1123 }
1124 }
1125
1126 /* go to heavy sweep */
1127 _repeat_discovery:
1128
1129 /* First of all - unset all flags */
1130 sm->p_subn->force_heavy_sweep = FALSE;
1131 sm->p_subn->force_reroute = FALSE;
1132 sm->p_subn->subnet_initialization_error = FALSE;
1133
1134 /* rescan configuration updates */
1135 if (osm_subn_rescan_conf_files(sm->p_subn) < 0)
1136 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 331A: "
1137 "osm_subn_rescan_conf_file failed\n");
1138
1139 if (sm->p_subn->sm_state != IB_SMINFO_STATE_MASTER)
1140 sm->p_subn->need_update = 1;
1141
1142 status = __osm_state_mgr_sweep_hop_0(sm);
1143 if (status != IB_SUCCESS ||
1144 wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1145 return;
1146
1147 if (__osm_state_mgr_is_sm_port_down(sm) == TRUE) {
1148 osm_log(sm->p_log, OSM_LOG_SYS, "SM port is down\n");
1149 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, "SM PORT DOWN");
1150
1151 /* Run the drop manager - we want to clear all records */
1152 osm_drop_mgr_process(sm);
1153
1154 /* Move to DISCOVERING state */
1155 osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_DISCOVER);
1156 return;
1157 }
1158
1159 status = __osm_state_mgr_sweep_hop_1(sm);
1160 if (status != IB_SUCCESS ||
1161 wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1162 return;
1163
1164 /* discovery completed - check other sm presense */
1165 if (sm->master_sm_found) {
1166 /*
1167 * Call the sm_state_mgr with signal
1168 * MASTER_OR_HIGHER_SM_DETECTED_DONE
1169 */
1170 osm_sm_state_mgr_process(sm,
1171 OSM_SM_SIGNAL_MASTER_OR_HIGHER_SM_DETECTED);
1172 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1173 "ENTERING STANDBY STATE");
1174 /* notify master SM about us */
1175 osm_send_trap144(sm, 0);
1176 return;
1177 }
1178
1179 /* if new sweep requested - don't bother with the rest */
1180 if (sm->p_subn->force_heavy_sweep)
1181 goto _repeat_discovery;
1182
1183 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, "HEAVY SWEEP COMPLETE");
1184
1185 /* If we are MASTER - get the highest remote_sm, and
1186 * see if it is higher than our local sm.
1187 */
1188 if (sm->p_subn->sm_state == IB_SMINFO_STATE_MASTER) {
1189 p_remote_sm = __osm_state_mgr_get_highest_sm(sm);
1190 if (p_remote_sm != NULL) {
1191 /* report new ports (trap 64) before leaving MASTER */
1192 __osm_state_mgr_report_new_ports(sm);
1193
1194 /* need to handover the mastership
1195 * to the remote sm, and move to standby */
1196 __osm_state_mgr_send_handover(sm, p_remote_sm);
1197 osm_sm_state_mgr_process(sm,
1198 OSM_SM_SIGNAL_HANDOVER_SENT);
1199 return;
1200 } else {
1201 /* We are the highest sm - check to see if there is
1202 * a remote SM that is in master state. */
1203 p_remote_sm =
1204 __osm_state_mgr_exists_other_master_sm(sm);
1205 if (p_remote_sm != NULL) {
1206 /* There is a remote SM that is master.
1207 * need to wait for that SM to relinquish control
1208 * of its portion of the subnet. C14-60.2.1.
1209 * Also - need to start polling on that SM. */
1210 sm->p_polling_sm = p_remote_sm;
1211 osm_sm_state_mgr_process(sm,
1212 OSM_SM_SIGNAL_WAIT_FOR_HANDOVER);
1213 return;
1214 }
1215 }
1216 }
1217
1218 /* Need to continue with lid assignment */
1219 osm_drop_mgr_process(sm);
1220
1221 /*
1222 * If we are not MASTER already - this means that we are
1223 * in discovery state. call osm_sm_state_mgr with signal
1224 * DISCOVERY_COMPLETED
1225 */
1226 if (sm->p_subn->sm_state == IB_SMINFO_STATE_DISCOVERING)
1227 osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_DISCOVERY_COMPLETED);
1228
1229 osm_pkey_mgr_process(sm->p_subn->p_osm);
1230
1231 osm_qos_setup(sm->p_subn->p_osm);
1232
1233 /* try to restore SA DB (this should be before lid_mgr
1234 because we may want to disable clients reregistration
1235 when SA DB is restored) */
1236 osm_sa_db_file_load(sm->p_subn->p_osm);
1237
1238 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1239 return;
1240
1241 osm_lid_mgr_process_sm(&sm->lid_mgr);
1242 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1243 return;
1244
1245 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1246 "SM LID ASSIGNMENT COMPLETE - STARTING SUBNET LID CONFIG");
1247 __osm_state_mgr_notify_lid_change(sm);
1248
1249 osm_lid_mgr_process_subnet(&sm->lid_mgr);
1250 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1251 return;
1252
1253 /* At this point we need to check the consistency of
1254 * the port_lid_tbl under the subnet. There might be
1255 * errors in it if PortInfo Set requests didn't reach
1256 * their destination. */
1257 __osm_state_mgr_check_tbl_consistency(sm);
1258
1259 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1260 "LID ASSIGNMENT COMPLETE - STARTING SWITCH TABLE CONFIG");
1261
1262 /*
1263 * Proceed with unicast forwarding table configuration.
1264 */
1265
1266 if (!sm->ucast_mgr.cache_valid ||
1267 osm_ucast_cache_process(&sm->ucast_mgr))
1268 osm_ucast_mgr_process(&sm->ucast_mgr);
1269
1270 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1271 return;
1272
1273 /* cleanup switch lft buffers */
1274 cl_qmap_apply_func(&sm->p_subn->sw_guid_tbl, cleanup_switch, sm->p_log);
1275
1276 /* We are done setting all LFTs so clear the ignore existing.
1277 * From now on, as long as we are still master, we want to
1278 * take into account these lfts. */
1279 sm->p_subn->ignore_existing_lfts = FALSE;
1280
1281 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1282 "SWITCHES CONFIGURED FOR UNICAST");
1283
1284 if (!sm->p_subn->opt.disable_multicast) {
1285 osm_mcast_mgr_process(sm);
1286 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1287 return;
1288 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1289 "SWITCHES CONFIGURED FOR MULTICAST");
1290 }
1291
1292 /*
1293 * The LINK_PORTS state is required since we cannot count on
1294 * the port state change MADs to succeed. This is an artifact
1295 * of the spec defining state change from state X to state X
1296 * as an error. The hardware then is not required to process
1297 * other parameters provided by the Set(PortInfo) Packet.
1298 */
1299
1300 osm_link_mgr_process(sm, IB_LINK_NO_CHANGE);
1301 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1302 return;
1303
1304 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1305 "LINKS PORTS CONFIGURED - SET LINKS TO ARMED STATE");
1306
1307 osm_link_mgr_process(sm, IB_LINK_ARMED);
1308 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1309 return;
1310
1311 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE,
1312 "LINKS ARMED - SET LINKS TO ACTIVE STATE");
1313
1314 osm_link_mgr_process(sm, IB_LINK_ACTIVE);
1315 if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
1316 return;
1317
1318 /*
1319 * The sweep completed!
1320 */
1321
1322 /*
1323 * Send trap 64 on newly discovered endports
1324 */
1325 __osm_state_mgr_report_new_ports(sm);
1326
1327 /* in any case we zero this flag */
1328 sm->p_subn->coming_out_of_standby = FALSE;
1329
1330 /* If there were errors - then the subnet is not really up */
1331 if (sm->p_subn->subnet_initialization_error == TRUE) {
1332 osm_log(sm->p_log, OSM_LOG_SYS,
1333 "Errors during initialization\n");
1334 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_ERROR,
1335 "ERRORS DURING INITIALIZATION");
1336 } else {
1337 sm->p_subn->need_update = 0;
1338 osm_dump_all(sm->p_subn->p_osm);
1339 __osm_state_mgr_up_msg(sm);
1340 sm->p_subn->first_time_master_sweep = FALSE;
1341
1342 if (osm_log_is_active(sm->p_log, OSM_LOG_VERBOSE))
1343 osm_sa_db_file_dump(sm->p_subn->p_osm);
1344 }
1345
1346 /*
1347 * Finally signal the subnet up event
1348 */
1349 cl_event_signal(&sm->subnet_up_event);
1350
1351 osm_opensm_report_event(sm->p_subn->p_osm, OSM_EVENT_ID_SUBNET_UP, NULL);
1352
1353 /* if we got a signal to force heavy sweep or errors
1354 * in the middle of the sweep - try another sweep. */
1355 if (sm->p_subn->force_heavy_sweep
1356 || sm->p_subn->subnet_initialization_error)
1357 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
1358 }
1359
do_process_mgrp_queue(osm_sm_t * sm)1360 static void do_process_mgrp_queue(osm_sm_t * sm)
1361 {
1362 if (sm->p_subn->sm_state != IB_SMINFO_STATE_MASTER)
1363 return;
1364 osm_mcast_mgr_process_mgroups(sm);
1365 wait_for_pending_transactions(&sm->p_subn->p_osm->stats);
1366 }
1367
osm_state_mgr_process(IN osm_sm_t * sm,IN osm_signal_t signal)1368 void osm_state_mgr_process(IN osm_sm_t * sm, IN osm_signal_t signal)
1369 {
1370 CL_ASSERT(sm);
1371
1372 OSM_LOG_ENTER(sm->p_log);
1373
1374 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
1375 "Received signal %s in state %s\n",
1376 osm_get_sm_signal_str(signal),
1377 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
1378
1379 switch (signal) {
1380 case OSM_SIGNAL_SWEEP:
1381 do_sweep(sm);
1382 break;
1383
1384 case OSM_SIGNAL_IDLE_TIME_PROCESS_REQUEST:
1385 do_process_mgrp_queue(sm);
1386 break;
1387
1388 default:
1389 CL_ASSERT(FALSE);
1390 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3320: "
1391 "Invalid SM signal %u\n", signal);
1392 break;
1393 }
1394
1395 OSM_LOG_EXIT(sm->p_log);
1396 }
1397