1 /*
2 * Copyright (c) 2004-2006 Voltaire Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #if HAVE_CONFIG_H
35 # include <config.h>
36 #endif /* HAVE_CONFIG_H */
37
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <unistd.h>
41 #include <pthread.h>
42 #include <sys/time.h>
43 #include <string.h>
44 #include <errno.h>
45
46 #include <infiniband/umad.h>
47 #include "mad.h"
48
49 #define MAX_CLASS 256
50
51 struct ibmad_port {
52 int port_id; /* file descriptor returned by umad_open() */
53 int class_agents[MAX_CLASS]; /* class2agent mapper */
54 };
55
56 int ibdebug;
57
58 static int mad_portid = -1;
59 static int iberrs;
60
61 static int madrpc_retries = MAD_DEF_RETRIES;
62 static int def_madrpc_timeout = MAD_DEF_TIMEOUT_MS;
63 static void *save_mad;
64 static int save_mad_len = 256;
65
66 #undef DEBUG
67 #define DEBUG if (ibdebug) IBWARN
68 #define ERRS if (iberrs || ibdebug) IBWARN
69
70 #define MAD_TID(mad) (*((uint64_t *)((char *)(mad) + 8)))
71
72 void
madrpc_show_errors(int set)73 madrpc_show_errors(int set)
74 {
75 iberrs = set;
76 }
77
78 void
madrpc_save_mad(void * madbuf,int len)79 madrpc_save_mad(void *madbuf, int len)
80 {
81 save_mad = madbuf;
82 save_mad_len = len;
83 }
84
85 int
madrpc_set_retries(int retries)86 madrpc_set_retries(int retries)
87 {
88 if (retries > 0)
89 madrpc_retries = retries;
90 return madrpc_retries;
91 }
92
93 int
madrpc_set_timeout(int timeout)94 madrpc_set_timeout(int timeout)
95 {
96 def_madrpc_timeout = timeout;
97 return 0;
98 }
99
100 int
madrpc_def_timeout(void)101 madrpc_def_timeout(void)
102 {
103 return def_madrpc_timeout;
104 }
105
106 int
madrpc_portid(void)107 madrpc_portid(void)
108 {
109 return mad_portid;
110 }
111
112 static int
_do_madrpc(int port_id,void * sndbuf,void * rcvbuf,int agentid,int len,int timeout)113 _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
114 int timeout)
115 {
116 uint32_t trid; /* only low 32 bits */
117 int retries;
118 int length, status;
119
120 if (!timeout)
121 timeout = def_madrpc_timeout;
122
123 if (ibdebug > 1) {
124 IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
125 xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
126 }
127
128 if (save_mad) {
129 memcpy(save_mad, umad_get_mad(sndbuf),
130 save_mad_len < len ? save_mad_len : len);
131 save_mad = 0;
132 }
133
134 trid = mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
135
136 for (retries = 0; retries < madrpc_retries; retries++) {
137 if (retries) {
138 ERRS("retry %d (timeout %d ms)", retries, timeout);
139 }
140
141 length = len;
142 if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
143 IBWARN("send failed; %s", strerror(errno));
144 return -1;
145 }
146
147 /* Use same timeout on receive side just in case */
148 /* send packet is lost somewhere. */
149 do {
150 if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
151 IBWARN("recv failed: %s", strerror(errno));
152 return -1;
153 }
154
155 if (ibdebug > 1) {
156 IBWARN("rcv buf:");
157 xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE);
158 }
159 } while ((uint32_t)mad_get_field64(umad_get_mad(rcvbuf), 0, IB_MAD_TRID_F) != trid);
160
161 status = umad_status(rcvbuf);
162 if (!status)
163 return length; /* done */
164 if (status == ENOMEM)
165 return length;
166 }
167
168 ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
169 return -1;
170 }
171
172 void *
mad_rpc(const void * port_id,ib_rpc_t * rpc,ib_portid_t * dport,void * payload,void * rcvdata)173 mad_rpc(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport, void *payload,
174 void *rcvdata)
175 {
176 const struct ibmad_port *p = port_id;
177 int status, len;
178 uint8_t sndbuf[1024], rcvbuf[1024], *mad;
179
180 len = 0;
181 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
182
183 if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
184 return 0;
185
186 if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
187 p->class_agents[rpc->mgtclass],
188 len, rpc->timeout)) < 0) {
189 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
190 return 0;
191 }
192
193 mad = umad_get_mad(rcvbuf);
194
195 if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) {
196 ERRS("MAD completed with error status 0x%x; dport (%s)",
197 status, portid2str(dport));
198 return 0;
199 }
200
201 if (ibdebug) {
202 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
203 xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
204 }
205
206 if (rcvdata)
207 memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
208
209 return rcvdata;
210 }
211
212 void *
mad_rpc_rmpp(const void * port_id,ib_rpc_t * rpc,ib_portid_t * dport,ib_rmpp_hdr_t * rmpp,void * data)213 mad_rpc_rmpp(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport,
214 ib_rmpp_hdr_t *rmpp, void *data)
215 {
216 const struct ibmad_port *p = port_id;
217 int status, len;
218 uint8_t sndbuf[1024], rcvbuf[1024], *mad;
219
220 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
221
222 DEBUG("rmpp %p data %p", rmpp, data);
223
224 if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
225 return 0;
226
227 if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
228 p->class_agents[rpc->mgtclass],
229 len, rpc->timeout)) < 0) {
230 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
231 return 0;
232 }
233
234 mad = umad_get_mad(rcvbuf);
235
236 if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
237 ERRS("MAD completed with error status 0x%x; dport (%s)",
238 status, portid2str(dport));
239 return 0;
240 }
241
242 if (ibdebug) {
243 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
244 xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
245 rpc->datasz);
246 }
247
248 if (rmpp) {
249 rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
250 if ((rmpp->flags & 0x3) &&
251 mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
252 IBWARN("bad rmpp version");
253 return 0;
254 }
255 rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
256 rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
257 DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
258 rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
259 rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
260 }
261
262 if (data)
263 memcpy(data, mad + rpc->dataoffs, rpc->datasz);
264
265 rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
266
267 return data;
268 }
269
270 void *
madrpc(ib_rpc_t * rpc,ib_portid_t * dport,void * payload,void * rcvdata)271 madrpc(ib_rpc_t *rpc, ib_portid_t *dport, void *payload, void *rcvdata)
272 {
273 struct ibmad_port port;
274
275 port.port_id = mad_portid;
276 port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass);
277 return mad_rpc(&port, rpc, dport, payload, rcvdata);
278 }
279
280 void *
madrpc_rmpp(ib_rpc_t * rpc,ib_portid_t * dport,ib_rmpp_hdr_t * rmpp,void * data)281 madrpc_rmpp(ib_rpc_t *rpc, ib_portid_t *dport, ib_rmpp_hdr_t *rmpp, void *data)
282 {
283 struct ibmad_port port;
284
285 port.port_id = mad_portid;
286 port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass);
287 return mad_rpc_rmpp(&port, rpc, dport, rmpp, data);
288 }
289
290 static pthread_mutex_t rpclock = PTHREAD_MUTEX_INITIALIZER;
291
292 void
madrpc_lock(void)293 madrpc_lock(void)
294 {
295 pthread_mutex_lock(&rpclock);
296 }
297
298 void
madrpc_unlock(void)299 madrpc_unlock(void)
300 {
301 pthread_mutex_unlock(&rpclock);
302 }
303
304 void
madrpc_init(char * dev_name,int dev_port,int * mgmt_classes,int num_classes)305 madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
306 {
307 if (umad_init() < 0)
308 IBPANIC("can't init UMAD library");
309
310 if ((mad_portid = umad_open_port(dev_name, dev_port)) < 0)
311 IBPANIC("can't open UMAD port (%s:%d)", dev_name, dev_port);
312
313 if (num_classes >= MAX_CLASS)
314 IBPANIC("too many classes %d requested", num_classes);
315
316 while (num_classes--) {
317 int rmpp_version = 0;
318 int mgmt = *mgmt_classes++;
319
320 if (mgmt == IB_SA_CLASS)
321 rmpp_version = 1;
322 if (mad_register_client(mgmt, rmpp_version) < 0)
323 IBPANIC("client_register for mgmt class %d failed", mgmt);
324 }
325 }
326
327 void *
mad_rpc_open_port(char * dev_name,int dev_port,int * mgmt_classes,int num_classes)328 mad_rpc_open_port(char *dev_name, int dev_port,
329 int *mgmt_classes, int num_classes)
330 {
331 struct ibmad_port *p;
332 int port_id;
333
334 if (num_classes >= MAX_CLASS) {
335 IBWARN("too many classes %d requested", num_classes);
336 errno = EINVAL;
337 return NULL;
338 }
339
340 if (umad_init() < 0) {
341 IBWARN("can't init UMAD library");
342 errno = ENODEV;
343 return NULL;
344 }
345
346 p = malloc(sizeof(*p));
347 if (!p) {
348 errno = ENOMEM;
349 return NULL;
350 }
351 memset(p, 0, sizeof(*p));
352
353 if ((port_id = umad_open_port(dev_name, dev_port)) < 0) {
354 IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
355 if (!errno)
356 errno = EIO;
357 free(p);
358 return NULL;
359 }
360
361 while (num_classes--) {
362 int rmpp_version = 0;
363 int mgmt = *mgmt_classes++;
364 int agent;
365
366 if (mgmt == IB_SA_CLASS)
367 rmpp_version = 1;
368 if (mgmt < 0 || mgmt >= MAX_CLASS ||
369 (agent = mad_register_port_client(port_id, mgmt,
370 rmpp_version)) < 0) {
371 IBWARN("client_register for mgmt %d failed", mgmt);
372 if(!errno)
373 errno = EINVAL;
374 umad_close_port(port_id);
375 free(p);
376 return NULL;
377 }
378 p->class_agents[mgmt] = agent;
379 }
380
381 p->port_id = port_id;
382 return p;
383 }
384
385 void
mad_rpc_close_port(void * port_id)386 mad_rpc_close_port(void *port_id)
387 {
388 struct ibmad_port *p = port_id;
389
390 umad_close_port(p->port_id);
391 free(p);
392 }
393
394 uint8_t *
sa_call(void * rcvbuf,ib_portid_t * portid,ib_sa_call_t * sa,unsigned timeout)395 sa_call(void *rcvbuf, ib_portid_t *portid, ib_sa_call_t *sa, unsigned timeout)
396 {
397 struct ibmad_port port;
398
399 port.port_id = mad_portid;
400 port.class_agents[IB_SA_CLASS] = mad_class_agent(IB_SA_CLASS);
401 return sa_rpc_call(&port, rcvbuf, portid, sa, timeout);
402 }
403